diff --git a/.gitignore b/.gitignore
index e706a14847..9679195d83 100644
--- a/.gitignore
+++ b/.gitignore
@@ -25,12 +25,18 @@
 /tests/cuda_long_context_smoke
 /tests/test_layer_pack
 /tests/test_metal_session_batch
+/tests/test_metal_indexer_q4
+/tests/test_metal_q4_qb_f16_cache
+/tests/test_metal_q4_prefill_pair
+/tests/test_metal_q4_streams
 /tests/test_mxfp4_cuda
 /tests/test_mxfp4_dot
 /tests/test_mxfp4_metal
 /tests/test_q4k_dot
 /tests/test_sampling
 /tests/test_glm53_kda
+/tests/test_quantizer_indexer_q4
+/tests/test_rocm_q4_dense_pair
 /tests/test_deepseek4_vision_image
 *.o
 *.dSYM/
@@ -41,3 +47,8 @@ __pycache__/
 /misc/
 .*.swp
 .DS_Store
+
+# Local speed benchmark outputs
+/speed-bench/*.csv
+/speed-bench/*_logprobs.json
+/speed-bench/*_logits/
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index f9bb07168b..f915b92629 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -25,6 +25,14 @@ The C test runner is `ds4_test`. Running it without arguments is equivalent to
 make test
 ```
 
+If a change adds, renames, removes, or changes the parsing/default/effect of an
+environment variable, update `scripts/environment_variables.tsv`, regenerate
+the complete reference, and verify that it is current:
+
+```sh
+make environment-docs
+```
+
 Useful narrower checks:
 
 ```sh
diff --git a/ENVIRONMENT_VARIABLES.md b/ENVIRONMENT_VARIABLES.md
new file mode 100644
index 0000000000..f169704b60
--- /dev/null
+++ b/ENVIRONMENT_VARIABLES.md
@@ -0,0 +1,1481 @@
+# Environment variables
+
+Command-line options are the supported interface for normal inference.  The
+first part of this document is the curated, user-facing reference for switches
+used to isolate optimized paths, require test coverage, or collect diagnostics.
+The generated inventory below it lists every environment name consumed by the
+runtime, tests, and maintained tools.  Inclusion in that complete inventory does
+not make an internal tuning knob a stable API.
+
+Run `make check-environment-docs` after adding, renaming, or removing an
+environment variable.  Regenerate the inventory with
+`python3 scripts/generate_environment_variables.py`.
+
+Most backend switches are cached on first use.  Start a new process when
+changing them.  Unless a row says otherwise:
+
+- use the documented value `1`; many rollback switches are presence-based, so
+  setting them to `0` still enables the rollback and they must instead be unset;
+- `DISABLE` or `NO` is the rollback switch and takes precedence;
+- `REQUIRE` turns an eligible silent fallback into an error, so tests cannot
+  pass without exercising the intended path;
+- `STATS`, `PROFILE`, and `ORACLE` are diagnostic and may perturb timing;
+- benchmark controls and candidates in separate processes.
+
+## Greedy top-1 readback
+
+| Variable | Default behavior and purpose |
+| --- | --- |
+| `DS4_DISABLE_GREEDY_TOP1_READBACK=1` | Restore the legacy full-logits readback and CPU argmax. Unset uses device top-1 plus a four-byte readback for eligible single-tier greedy generation, including SSD streaming, on Metal, CUDA, and ROCm. |
+
+## Metal
+
+| Variable | Default behavior and purpose |
+| --- | --- |
+| `DS4_METAL_PREFILL_CHUNK=N` | Set the prefill cap when `--prefill-chunk` is absent; the CLI option takes precedence. This historical name is consumed by the shared graph planner rather than by a Metal kernel alone. |
+| `DS4_METAL_NO_RESIDENCY=1` | Skip creation and residency requests for the model-view residency set. Diagnostic rollback for resident, non-streaming models. |
+| `DS4_METAL_DISABLE_QUEUE_RESIDENCY_SET=1` | Still create, commit, and request the model residency set, but do not attach it to Metal command queues. This isolates queue-residency behavior without disabling the complete residency policy. |
+| `DS4_METAL_DISABLE_DECODE_ARGMAX_TOP1=1` | Restore the generic full argsort used by Metal argmax. Unset uses the dedicated two-stage top-1 reduction for eligible large decode rows. |
+| `DS4_METAL_REQUIRE_DECODE_ARGMAX_TOP1=1` | Fail closed instead of using generic argsort when a row of at least 4096 logits cannot execute the dedicated top-1 path. Intended for correctness and performance oracles. |
+| `DS4_METAL_STREAMING_EXPERT_NOCACHE=1` | Reopen the Metal SSD expert file with `F_NOCACHE` so streamed experts do not displace the dense working set from the page cache. Leave unset for cached `pread`. |
+| `DS4_METAL_STREAMING_EXPERT_PREAD_SPLIT=N` | Split each expert read into 1–8 aligned requests. The automatic value is 1 below 64 configured cache experts and 4 at 64 or more. |
+| `DS4_METAL_DISABLE_Q4_DENSE_PAIR=1` | Split the default Metal Q-A/KV Q4 pair back into two standalone projections. |
+| `DS4_METAL_ENABLE_Q4_PREFILL_PAIR_F16_RHS=1` | On Apple M1–M4, opt into the Q-A/KV prefill pair that materializes their shared F32 activation as F16 once for exact 32-token tiles through N=128. |
+| `DS4_METAL_DISABLE_Q4_PREFILL_PAIR_F16_RHS=1` | Dominant rollback to two standalone Q4_K/F32-RHS prefill projections. |
+| `DS4_METAL_REQUIRE_Q4_PREFILL_PAIR_F16_RHS=1` | Request the shared-F16-RHS pair and fail closed if its device, shape, storage, or buffer contract is unavailable. Intended for strict A/B oracles. |
+| `DS4_METAL_DISABLE_Q4_ATTN_OUT_A_DIRECT=1` | Restore the generic route-map/work-list Q4 attention output-A path. Unset uses the bit-identical fixed-route direct kernel by default for eligible Apple M1–M4 `4096 -> 1024`, eight-group prefills at N=512–4096. Any defined value, including `0`, disables the specialization. |
+| `DS4_METAL_REQUIRE_Q4_ATTN_OUT_A_DIRECT=1` | Require the fixed-route Q4 attention output-A kernel and fail closed before dispatch if its device, shape, concurrency, pipeline, or memory contract is unavailable. Intended for production-shape correctness and performance oracles; `DISABLE` wins. |
+| `DS4_METAL_DISABLE_M1_IQ2_MID_ONLY=1` | Restore the canonical IQ2 address-table gate/up producer on the exact M1 SSD-streaming decode shape. The specialization is automatic by default. |
+| `DS4_METAL_REQUIRE_M1_IQ2_MID_ONLY=1` | Fail closed when an otherwise eligible M1 IQ2 mid-only dispatch cannot use the specialization. |
+| `DS4_METAL_ENABLE_M1_IQ2_MID_ONLY=1` | Legacy spelling retained for migration notes only. The runtime does not read it; the path is automatic and this setting is ignored. |
+| `DS4_METAL_DISABLE_IQ2_XXS_SSD_PREFILL_MM=1` | Restore sparse matvec for an eligible grouped IQ2 SSD-prefill chunk. |
+| `DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM=1` | Require grouped IQ2 SSD-prefill MM for eligible chunks and reject insufficient cache instead of silently falling back. |
+| `DS4_METAL_DISABLE_Q4_SSD_PREFILL_ATTN_OUT_SCALE_META=1` | Restore per-SIMDgroup Q4_K scale/min unpacking inside the opt-in SSD-prefill attention-output exact-N path. Shared scale metadata is automatic when that path is enabled and the PSO and threadgroup memory are available. |
+| `DS4_METAL_REQUIRE_Q4_SSD_PREFILL_ATTN_OUT_SCALE_META=1` | Enable the Q4_K SSD-prefill attention-output exact-N path and fail closed unless its shared scale/min metadata PSO is used. Intended for oracles and A/B tests. |
+| `DS4_METAL_ENABLE_STREAMING_PREFILL_EXPERT_READAHEAD=1` | Restore the historical `F_RDADVISE` plus parallel-`pread` sequence for cold-storage A/B tests. Normal grouped prefill skips the redundant hint. |
+
+The detailed Metal A/B contracts and expected oracle counters live in
+[`QA_BEFORE_RELEASES.md`](QA_BEFORE_RELEASES.md).
+
+## CUDA Q4 and Q8 diagnostics
+
+| Variable | Default behavior and purpose |
+| --- | --- |
+| `DS4_CUDA_DECODE_GRAPHS=0` | Disable CUDA decode graph capture. Unset, `1`, `on`, `yes`, or `true` enables capture; `0`, `off`, `no`, or `false` disables it. Oracle modes may also suppress capture. |
+| `DS4_CUDA_DISABLE_Q4_ATTN_Q_B_TRANSIENT_F16=1` | Restore native Q4_K `attn_q_b` for automatic long resident prefills. The transient path is otherwise eligible from 4096 tokens when the single-GPU model image is physically device-resident and retains only one reusable expanded matrix; explicit persistent-cache controls remain independent. |
+| `DS4_CUDA_Q4_ATTN_Q_B_TRANSIENT_F16_MIN_TOKENS=N` | Override the automatic transient Q4_K-to-F16 crossover (default 4096 tokens) for single-GPU, device-image-resident, non-SSD, non-quality prefills. |
+| `DS4_CUDA_DISABLE_Q4_DENSE_PAIR=1` | Split the Q-A/KV Q4 pair back into two standalone projections. |
+| `DS4_CUDA_NO_Q4_GB10_FAST=1` | Umbrella rollback for the GB10-specific Q4 choices; it does not disable the older cross-CUDA dense pair. |
+| `DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_BATCH=1` | Enable grouped attention-A for two-to-eight-token GB10 verifier batches. |
+| `DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_BATCH=1` | Fail closed if that grouped batch path is unavailable. |
+| `DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_PREFILL=0` | Compatibility opt-out for the default GB10 Q4 attention-A grouped prefill path above eight tokens; any other nonempty value explicitly requests it. |
+| `DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL=1` | Dominant rollback from the default grouped Q4 attention-A prefill path to eight pack/MMQ/unpack projections. Any defined value disables the path. |
+| `DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_PREFILL=1` | Request the grouped Q4 attention-A prefill candidate and fail before enqueue when its GB10, shape, residency, or buffer contract is unavailable. |
+| `DS4_CUDA_NO_Q4_GROUPED_ATTN_A_Q81=1` | Narrow rollback from the default fixed-layout `K=4096`, `groups=8` Q8_1 producer to the canonical strided producer while retaining grouped prefill. Any defined value disables the specialized kernel. |
+| `DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_Q81=1` | Request the fixed-layout eight-warp Q8_1 producer and fail before enqueue on rollback or ineligibility. |
+| `DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_SINGLE_GRID=1` | On eligible GB10 prefills, submit the eight grouped attention-A MMQs as one grid.z launch while retaining a separate stream-K coordinate and fixup slice per group. |
+| `DS4_CUDA_DISABLE_Q4_GROUPED_ATTN_A_SINGLE_GRID=1` | Dominant rollback from the single-grid experiment to the established one-MMQ-grid-per-group path. |
+| `DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_SINGLE_GRID=1` | Require the single-grid grouped attention-A submission and fail closed on rollback, ineligibility, or dispatch failure. |
+| `DS4_CUDA_Q4_GROUPED_ATTN_A_ORACLE=1` | Compare grouped attention-A with the canonical result and retain the canonical output. Disable graph capture for this diagnostic. |
+| `DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT=1` | Enable the experimental persistent-CTA kernel for the exact `32768x1024` Q4 shape. |
+| `DS4_CUDA_NO_Q4_K1024_PERSISTENT=1` | Roll back the persistent K1024 experiment. |
+| `DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT=1` | Require the K1024 candidate before enqueue instead of silently using canonical MMVQ. |
+| `DS4_CUDA_Q4_MMQ_16WARP=1` | Request the experimental exact-integer m128n128 16-warp Q4_K kernel for eligible CUDA prefills. Standalone dense admits `M>=1024` and `K<=8192`, including attention output-B; dense-pair shares one Q8_1 activation, admits legs down to `M=512`, and remains bounded to `K<=4096`. Candidate grids require at least 80% whole-tile SM-wave efficiency; below the canonical 90% cutoff the kernel mirrors canonical Stream-K partitioning and fixup so the FP32 reduction tree is unchanged. Ineligible optional shapes fall back. |
+| `DS4_CUDA_NO_Q4_MMQ_16WARP=1` | Roll back the 16-warp Q4_K prefill experiment. This value-aware switch overrides request and require. |
+| `DS4_CUDA_REQUIRE_Q4_MMQ_16WARP=1` | Request the 16-warp Q4_K prefill kernel and fail closed instead of silently measuring another MMQ/Q8_K path; a dense-pair requires both legs to be eligible. Decode/speculative batches of at most eight tokens remain on MMVQ. |
+| `DS4_CUDA_ENABLE_Q8_FOLD=1` | Enable the experimental one-shot Q8_1 producer-to-consumer fold. |
+| `DS4_CUDA_NO_Q8_FOLD=1` | Dominant rollback for the Q8_1 fold. |
+| `DS4_CUDA_Q8_FOLD_ORACLE=1` | Compare fresh canonical Q8_1 bytes and consumer outputs. Use with `DS4_CUDA_DECODE_GRAPHS=0`; require nonzero calls and zero mismatches/skips. |
+| `DS4_CUDA_MOE_PROFILE=1` | Print `cudaEvent` stage timings for routed-MoE launches, including resident IQ2 pair/SwiGLU/Q2-down/sum and aligned SoA/direct-D2R fast paths. Profiling is diagnostic and synchronizes the measured stream; the added aligned-IQ2 recorder deliberately excludes one-token graph-captured decode. |
+
+## ROCm Q4 and IQ2/Q2 diagnostics
+
+| Variable | Default behavior and purpose |
+| --- | --- |
+| `DS4_ROCM_DISABLE_Q4_ATTN_Q_B_TRANSIENT_F16=1` | Restore native Q4_K `attn_q_b` for automatic long resident prefills. The transient path is otherwise eligible from 4096 tokens when every `attn_q_b` source is already in a device image or device-backed resident range and retains only one reusable expanded matrix; explicit persistent-cache controls remain independent. |
+| `DS4_ROCM_Q4_ATTN_Q_B_TRANSIENT_F16_MIN_TOKENS=N` | Override the automatic transient Q4_K-to-F16 crossover (default 4096 tokens) for device-resident, non-SSD, non-quality prefills. |
+| `DS4_ROCM_DISABLE_Q4_PREFILL_TILE8=1` | Restore the legacy Q4 prefill kernel. TILE8 is automatic for validated chunks of 9 through 4096 tokens. |
+| `DS4_ROCM_REQUIRE_Q4_PREFILL_TILE8=1` | Fail closed when an eligible Q4 prefill call cannot use TILE8. |
+| `DS4_ROCM_ENABLE_Q4_PREFILL_TILE8=1` | Legacy spelling retained for migration notes only. The runtime does not read it; TILE8 is automatic and this setting is ignored. |
+| `DS4_ROCM_ENABLE_Q4_PREFILL_Q8_K_WAVE32=1` | On gfx1151 wave32, opt into the no-LDS Q8_K activation quantizer that assigns one 256-value block to each wave before the exact Q4 prefill matmul. This exact path takes precedence over automatic direct-Q4 WMMA; `REQUIRE_Q4_PREFILL_WMMA` overrides an optional request, while dual REQUIRE fails closed. |
+| `DS4_ROCM_DISABLE_Q4_PREFILL_Q8_K_WAVE32=1` | Dominant rollback to the canonical one-workgroup-per-Q8_K-block quantizer. |
+| `DS4_ROCM_REQUIRE_Q4_PREFILL_Q8_K_WAVE32=1` | Require the wave32 quantizer for strict prefill A/B runs; unsupported scope/device, rollback, or an incompatible required F16/WMMA path fails closed. |
+| `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA=0/1` | Compatibility control for the resident gfx1151 wave32 direct-Q4 WMMA prefill kernel at 256–4096 tokens. Unset keeps the automatic standalone and attention-output-A paths, while attention-output B stays on Q8_K+TILE8. A true value explicitly retains those eligible A paths but no longer opts B into direct WMMA; an explicit false value opts out unless `REQUIRE` is also set. Use `DISABLE=1` for an authoritative rollback. The kernel uses transient F16 register dequantization and F32 accumulation, with 64 rows below output dimension 1024, 128 rows below 8192, and 256 rows otherwise; aligned 256-row launches use K128/P144 and float4 activation staging by default. |
+| `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_K64=0/1` | Base staging control for an otherwise eligible direct-Q4 WMMA launch. Unset or true uses K64/P80 on 64/128-row or K128-incompatible launches and permits the K128/P144 default on aligned 256-row launches. `0`/`false`/`no`/`off` suppresses both wider stages and rolls back selectively to K32, while `DS4_ROCM_DISABLE_Q4_PREFILL_WMMA=1` rolls back direct WMMA entirely. |
+| `DS4_ROCM_DISABLE_Q4_PREFILL_WMMA_K128=1` | Value-aware opt-out for the default K128/P144 stage on aligned 256-row direct-Q4 WMMA launches, targeting resident `attn_q_b`. Unset or `0`/`false`/`no`/`off` keeps K128; empty or any other value rolls the same launch back to K64/P80. Incompatible alignment or row geometry also uses K64, while `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_K64=0` still rolls back to K32. This setting has no effect when the persistent or automatic transient F16 `attn_q_b` path owns the projection. |
+| `DS4_ROCM_Q4_PREFILL_WMMA_ROW_TILE=64|128|256` | Override the direct-Q4 WMMA output-row tile. The default uses 64 rows below output dimension 1024, 128 below 8192, and 256 otherwise; `64` also retains the prior kernel geometry as an A/B control. |
+| `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_SSD=1` | Explicitly allow direct-Q4 WMMA during SSD streaming only when each complete projection weight range is already backed by physical device storage. It opts attention-output A into WMMA while B remains on Q8_K+TILE8, and never treats mapped/registered host memory as resident. |
+| `DS4_ROCM_DISABLE_Q4_PREFILL_WMMA=1` | Dominant value-aware opt-out for the automatic direct-Q4 WMMA path, including explicit resident or SSD requests. |
+| `DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA=1` | Require direct-Q4 WMMA for every selected projection and fail closed on an unsupported device/shape, quality mode, rollback, or SSD weight range that is not physically device-resident. In an all-Q4 attention-output batch this is the only control that selects the numerically compounded direct-WMMA B stage; it is a diagnostic assertion for strict kernel A/B oracles, not a quality-sensitive runtime setting. |
+| `DS4_ROCM_Q4_PREFILL_TILE8_STATS=1` | Report dense, pair, attention-batch, token, and direct-WMMA K32/K64/K128 dispatch counters at process exit. |
+| `DS4_ROCM_ENABLE_Q4_DENSE_PAIR=1` | Share one Q8_K activation quantization between the two Q4 dense projections. This pair remains opt-in. |
+| `DS4_ROCM_DISABLE_Q4_DENSE_PAIR=1` | Dominant rollback for the ROCm Q4 dense pair. |
+| `DS4_ROCM_ENABLE_Q4_GROUPED_ATTN_A=1` | Extend the two-launch grouped attention-A path outside its default scope. The exact caller-marked resident decode shape `groups=8, N=1, K=4096, M=1024` is automatic; row-at-a-time batch fallbacks are not. |
+| `DS4_ROCM_DISABLE_Q4_GROUPED_ATTN_A=1` | Dominant rollback to eight standalone Q4 attention-A projections, including for the resident default. |
+| `DS4_ROCM_REQUIRE_Q4_GROUPED_ATTN_A=1` | Request grouped attention-A for eligible non-default shapes and fail closed on fallback; `DISABLE` remains authoritative. |
+| `DS4_ROCM_Q4_GROUPED_ATTN_A_STATS=1` | Report grouped calls, dispatches, groups, fallbacks, and failures. |
+| `DS4_ROCM_ENABLE_IQ2_MOE_WMMA_TAIL_CULL=1` | Opt into inactive-tail wave culling in the resident IQ2 gate/up and Q2 down hot-list rocWMMA prefill kernels. Requires a runtime wave width of 32; default is off pending hardware benchmarks. |
+| `DS4_ROCM_DISABLE_IQ2_MOE_WMMA_TAIL_CULL=1` | Dominant value-aware rollback for the IQ2/Q2 rocWMMA tail-wave candidate. Use as the baseline arm even though the candidate is currently off by default. |
+| `DS4_ROCM_IQ2_MOE_WMMA_PROFILE=1` | Measure only the affected IQ2 gate/up and Q2 down rocWMMA kernels with GPU events. A profiled eligible call fails rather than reporting a partial or scalar-fallback measurement. |
+
+Run `make test-strix-rocm-q4-parity` and
+`make test-strix-rocm-q4-prefill` on a `gfx1151` Strix Halo host before making
+performance claims.  The synthetic oracle proves layout and numerical parity;
+it does not by itself prove that a complete Q4 model fits safely in GTT.
+
+## Historical `DS4_METAL_*` graph controls
+
+The shared graph implementation predates the CUDA backend and retained a few
+`DS4_METAL_*` names.  In a non-ROCm GPU build, these controls affect the shared
+Metal/CUDA graph policy; ROCm explicitly ignores them:
+
+- `DS4_METAL_DISABLE_HC_FUSION=1`
+- `DS4_METAL_DISABLE_HC_NORM_FUSION=1`
+- `DS4_METAL_DISABLE_KV_FUSION=1`
+- `DS4_METAL_DISABLE_QKV_NORM_FUSION=1`
+- `DS4_METAL_DISABLE_QKV_PAIR_PROJ=1`
+- `DS4_METAL_DISABLE_COMPRESSOR_PAIR_PROJ=1`
+- `DS4_METAL_DISABLE_ATTN_OUT_HC_FUSION=1`
+- `DS4_METAL_DISABLE_SHARED_DOWN_HC_FUSION=1`
+
+The prefix is historical rather than an indication that these particular
+switches are always Metal-only.  New backend-specific controls should use the
+backend they actually configure (`DS4_CUDA_*` or `DS4_ROCM_*`) instead of
+extending this legacy naming.
+
+<!-- BEGIN GENERATED ENVIRONMENT VARIABLE INVENTORY -->
+## Complete implementation inventory
+
+This section is generated by `scripts/generate_environment_variables.py`; do not edit it by hand.
+Human-reviewed value/default and purpose metadata lives in
+`scripts/environment_variables.tsv`; the generator verifies it against the source tree.
+It lists every `DS4_*` string consumed by production C/C++/Objective-C/CUDA/ROCm
+sources, including names passed indirectly through helper functions, macros, and
+source-specification arrays. Unless a variable appears in the user-facing reference
+above, it is an unstable internal diagnostic or tuning interface. The linked source
+remains normative for exact eligibility
+gates, bounds, and architecture-specific defaults.
+
+Inventory totals: **1083 `DS4_*` runtime variables** and
+**6 external runtime variables**.
+The auxiliary inventories contain **118 test/test-fixture entries**
+and **19 tool/wrapper entries**.
+
+<details>
+<summary><strong>Metal (447)</strong></summary>
+
+| Variable | Accepted value and default | Effect | Source |
+| --- | --- | --- | --- |
+| `DS4_METAL_ARGSORT_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the argsort Metal kernel source file loaded at runtime. | [ds4_metal.m:4943](ds4_metal.m#L4943) |
+| `DS4_METAL_ATTN_OUT_STAGE_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for attn out stage. | [ds4.c:65151](ds4.c#L65151) |
+| `DS4_METAL_BIN_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the binary operations Metal kernel source file loaded at runtime. | [ds4_metal.m:4952](ds4_metal.m#L4952) |
+| `DS4_METAL_COMPRESSOR_PAIR_NR4` | presence control; unset: off/default; any value including 0 enables | Selects the NR4 compressor-pair variant. | [ds4_metal.m:2651](ds4_metal.m#L2651) |
+| `DS4_METAL_CONCAT_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the concatenation Metal kernel source file loaded at runtime. | [ds4_metal.m:4945](ds4_metal.m#L4945) |
+| `DS4_METAL_CPY_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the copy Metal kernel source file loaded at runtime. | [ds4_metal.m:4944](ds4_metal.m#L4944) |
+| `DS4_METAL_DECODE_INDEXER_SPARSE_THRESHOLD` | integer in {64,128,256,512,1024,2048,4096}; default 1024; invalid restores default | Sets the compressed-row crossover from dense to sparse indexed attention. | [ds4.c:20352](ds4.c#L20352) |
+| `DS4_METAL_DECODE_STAGE_PROFILE` | unset: off; 1/true/yes/on/all enables all layers; a layer index selects one; 0/false/no/off disables | Prints timing/profile diagnostics for decode stage. | [ds4.c:17568](ds4.c#L17568) |
+| `DS4_METAL_DECODE_STAGE_PROFILE_LAYER` | single unsigned layer index; unset/empty: all layers enabled by the parent profile; invalid matches no layer | Restricts the corresponding shared graph stage profiler to one layer. | [ds4.c:29120](ds4.c#L29120) |
+| `DS4_METAL_DENSE_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the dense matmul Metal kernel source file loaded at runtime. | [ds4_metal.m:4936](ds4_metal.m#L4936) |
+| `DS4_METAL_DISABLE_AFFINE_ROPE_PAIR` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables affine RoPE pair. | [ds4_metal.m:25171](ds4_metal.m#L25171) |
+| `DS4_METAL_DISABLE_ATTN_OUT_HC_FUSION` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Disables attn out HC fusion. | [ds4.c:20477](ds4.c#L20477) |
+| `DS4_METAL_DISABLE_ATTN_OUT_IDS_CACHE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables attn out ids cache. | [ds4_metal.m:27771](ds4_metal.m#L27771) |
+| `DS4_METAL_DISABLE_ATTN_OUT_LOW_DIRECT` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables attn out low direct. | [ds4_metal.m:27760](ds4_metal.m#L27760) |
+| `DS4_METAL_DISABLE_BATCH_HC_NORM_FUSION` | nonempty value other than exact 0 disables; unset/empty/0 leaves the default enabled path | Dominant rollback for batched HC norm fusion. | [ds4.c:20457](ds4.c#L20457) |
+| `DS4_METAL_DISABLE_COMPRESSOR_APE_ADD` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables compressor APE add. | [ds4_metal.m:25393](ds4_metal.m#L25393) |
+| `DS4_METAL_DISABLE_COMPRESSOR_EXACT_POOL_RATIO4` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables compressor exact pool ratio4. | [ds4_metal.m:26379](ds4_metal.m#L26379) |
+| `DS4_METAL_DISABLE_COMPRESSOR_PAIR_PROJ` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Disables compressor pair proj. | [ds4_metal.m:23248](ds4_metal.m#L23248) |
+| `DS4_METAL_DISABLE_COMPRESSOR_QUAD_STORE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables compressor quad store. | [ds4.c:23534](ds4.c#L23534) |
+| `DS4_METAL_DISABLE_COMPRESSOR_RATIO4_DIRECT_POOL` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables compressor ratio4 direct pool. | [ds4_metal.m:26245](ds4_metal.m#L26245) |
+| `DS4_METAL_DISABLE_COMPRESSOR_RATIO4_PACK_FUSION` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables compressor ratio4 pack fusion. | [ds4_metal.m:26199](ds4_metal.m#L26199) |
+| `DS4_METAL_DISABLE_COMPRESSOR_STORE_ONE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables compressor store one. | [ds4_metal.m:23249](ds4_metal.m#L23249) |
+| `DS4_METAL_DISABLE_CONTIG_F16_F16_COPY` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Disables contig F16 F16 copy. | [ds4_metal.m:29562](ds4_metal.m#L29562) |
+| `DS4_METAL_DISABLE_CONTIG_F32_F16_COPY` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Disables contig F32 F16 copy. | [ds4_metal.m:29338](ds4_metal.m#L29338) |
+| `DS4_METAL_DISABLE_DECODE_ARGMAX_TOP1` | presence rollback; unset uses the dedicated two-dispatch top-1 reduction for eligible large Metal rows; any defined value including empty or 0 restores the generic full argsort | Restore the generic indexer argsort for decode argmax A/B and emergency rollback. | [ds4_metal.m:21517](ds4_metal.m#L21517) |
+| `DS4_METAL_DISABLE_DECODE_NORM_EXACT_VIEWS` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables decode norm exact views. | [ds4_metal.m:36207](ds4_metal.m#L36207) |
+| `DS4_METAL_DISABLE_DECODE_RAW_GATHERED_ATTN` | presence rollback; unset: raw-only decode uses gathered attention; any value including 0 restores the legacy raw path | Restores the separate raw-only attention path instead of gathered staging and attention. | [ds4_metal.m:32968](ds4_metal.m#L32968) |
+| `DS4_METAL_DISABLE_DECODE_RAW_PACKED32` | presence rollback; unset: raw-only gathered attention may use packed32; any value including 0 disables it for raw-only layers | Disables the packed32 reduce kernel for raw-only gathered attention while leaving compressed layers unchanged. | [ds4_metal.m:31464](ds4_metal.m#L31464) |
+| `DS4_METAL_DISABLE_DECODE_ROUTER_BIAS_EXACT_VIEWS` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables decode router bias exact views. | [ds4_metal.m:39397](ds4_metal.m#L39397) |
+| `DS4_METAL_DISABLE_DSPARK_CAPTURE_FUSED_LAST` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Disables DSpark capture fused last. | [ds4.c:28361](ds4.c#L28361) |
+| `DS4_METAL_DISABLE_DSPARK_EXACTN_BATCH_HEAD` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Disables DSpark exactn batch head. | [ds4.c:37728](ds4.c#L37728) |
+| `DS4_METAL_DISABLE_EXACT_ROWS_PERSISTENT_CACHE` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Disables exact rows persistent cache. | [ds4_metal.m:13000](ds4_metal.m#L13000) |
+| `DS4_METAL_DISABLE_GATHERED_KV_PAD_FUSION` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables gathered KV pad fusion. | [ds4_metal.m:29706](ds4_metal.m#L29706) |
+| `DS4_METAL_DISABLE_GATHERED_KV_STAGE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables gathered KV stage. | [ds4_metal.m:29677](ds4_metal.m#L29677) |
+| `DS4_METAL_DISABLE_GLM_DECODE_KV_GROUP4` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Disables GLM decode KV group4. | [ds4_metal.m:36742](ds4_metal.m#L36742) |
+| `DS4_METAL_DISABLE_GLM_QKLOW_SG` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables GLM qklow sg. | [ds4_metal.m:37865](ds4_metal.m#L37865) |
+| `DS4_METAL_DISABLE_GLM_STREAMING_EXPERT_EARLY_LOAD` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables GLM streaming expert early load. | [ds4_metal.m:18058](ds4_metal.m#L18058) |
+| `DS4_METAL_DISABLE_GLM_STREAMING_EXPERT_SPLIT` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables GLM streaming expert split. | [ds4_metal.m:39796](ds4_metal.m#L39796) |
+| `DS4_METAL_DISABLE_GLM_STREAMING_PREFILL_FULL_LAYER` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables GLM streaming prefill full layer. | [ds4.c:42710](ds4.c#L42710) |
+| `DS4_METAL_DISABLE_GLM_STREAMING_PREFILL_FULL_LAYER_PREPARE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables GLM streaming prefill full layer prepare. | [ds4.c:42728](ds4.c#L42728) |
+| `DS4_METAL_DISABLE_GLM_STREAMING_PREFILL_SELECTED_ASYNC_LOAD` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables GLM streaming prefill selected async load. | [ds4.c:46421](ds4.c#L46421) |
+| `DS4_METAL_DISABLE_GLM_STREAMING_SELECTED_ASYNC_LOAD` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables GLM streaming selected async load. | [ds4.c:44329](ds4.c#L44329) |
+| `DS4_METAL_DISABLE_HC_FUSION` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Disables HC fusion. | [ds4.c:20406](ds4.c#L20406) |
+| `DS4_METAL_DISABLE_HC_NORM_FUSION` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Disables HC norm fusion. | [ds4.c:20450](ds4.c#L20450) |
+| `DS4_METAL_DISABLE_HC_PRODUCER_PRE_NORM_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables HC producer pre norm fuse. | [ds4_metal.m:46541](ds4_metal.m#L46541) |
+| `DS4_METAL_DISABLE_HC_RMS_SCALE_PROJ` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables HC RMS scale proj. | [ds4_metal.m:24148](ds4_metal.m#L24148) |
+| `DS4_METAL_DISABLE_HOT_PIPELINE_STATICS` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables hot pipeline statics. | [ds4_metal.m:2634](ds4_metal.m#L2634) |
+| `DS4_METAL_DISABLE_INPLACE_ROPE_PAIR` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables inplace RoPE pair. | [ds4_metal.m:25170](ds4_metal.m#L25170) |
+| `DS4_METAL_DISABLE_IQ2_SELECTED_EXPERT_VIEWS` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables IQ2 selected expert views. | [ds4.c:21080](ds4.c#L21080) |
+| `DS4_METAL_DISABLE_IQ2_SELECTED_SHARED_OVERLAP` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables IQ2 selected shared overlap. | [ds4.c:20996](ds4.c#L20996) |
+| `DS4_METAL_DISABLE_IQ2_STREAM_ADDR_TABLE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables IQ2 stream address table. | [ds4_metal.m:42402](ds4_metal.m#L42402) |
+| `DS4_METAL_DISABLE_IQ2_XXS_SSD_PREFILL_MM` | value-aware boolean; default off; true disables and dominates ENABLE; false leaves automatic policy | Rolls grouped IQ2_XXS/Q2_K SSD-prefill MM back to sparse matvec. | [ds4_metal.m:44784](ds4_metal.m#L44784) |
+| `DS4_METAL_DISABLE_KV_FUSION` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Disables KV fusion. | [ds4.c:20430](ds4.c#L20430) |
+| `DS4_METAL_DISABLE_M1_IQ2_MID_ONLY` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M1 IQ2 mid only. | [ds4_metal.m:14592](ds4_metal.m#L14592) |
+| `DS4_METAL_DISABLE_M3_COMPRESSOR_EXACT_POOL_RATIO4` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M3 compressor exact pool ratio4. | [ds4_metal.m:26381](ds4_metal.m#L26381) |
+| `DS4_METAL_DISABLE_M3_COMPRESSOR_PAIR_STATE_STORE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M3 compressor pair state store. | [ds4_metal.m:23247](ds4_metal.m#L23247) |
+| `DS4_METAL_DISABLE_M3_GATHERED_KV_STAGE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M3 gathered KV stage. | [ds4_metal.m:29678](ds4_metal.m#L29678) |
+| `DS4_METAL_DISABLE_M5_COMPRESSOR_EXACT_POOL_RATIO4` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M5 compressor exact pool ratio4. | [ds4_metal.m:26384](ds4_metal.m#L26384) |
+| `DS4_METAL_DISABLE_M5_COMP_FINALIZE_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M5 comp finalize fuse. | [ds4.c:23647](ds4.c#L23647) |
+| `DS4_METAL_DISABLE_M5_FLASH_ATTN_PACKED32_REDUCE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M5 flash attn packed32 reduce. | [ds4_metal.m:31519](ds4_metal.m#L31519) |
+| `DS4_METAL_DISABLE_M5_HC_NORM_MIX_CLUSTER2` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M5 HC norm mix cluster2. | [ds4_metal.m:46496](ds4_metal.m#L46496) |
+| `DS4_METAL_DISABLE_M5_HC_PRODUCER_PRE_NORM_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M5 HC producer pre norm fuse. | [ds4_metal.m:46548](ds4_metal.m#L46548) |
+| `DS4_METAL_DISABLE_M5_IQ2_PAIR_PACK2` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M5 IQ2 pair pack2. | [ds4_metal.m:42020](ds4_metal.m#L42020) |
+| `DS4_METAL_DISABLE_M5_PACKED_ZERO_MASK` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M5 packed zero mask. | [ds4_metal.m:31475](ds4_metal.m#L31475) |
+| `DS4_METAL_DISABLE_M5_PARALLEL_FULL_FFN` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M5 parallel full FFN. | [ds4.c:22707](ds4.c#L22707) |
+| `DS4_METAL_DISABLE_M5_PERSISTENT_ZERO_ATTN_MASK` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M5 persistent zero attn mask. | [ds4_metal.m:31473](ds4_metal.m#L31473) |
+| `DS4_METAL_DISABLE_M5_Q8_HC_VEC` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M5 Q8 HC vec. | [ds4_metal.m:47580](ds4_metal.m#L47580) |
+| `DS4_METAL_DISABLE_M5_QKV_PAIR_COMPRESSOR_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M5 QKV pair compressor fuse. | [ds4.c:23043](ds4.c#L23043) |
+| `DS4_METAL_DISABLE_M5_QKV_PAIR_QUAD_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M5 QKV pair quad fuse. | [ds4.c:23039](ds4.c#L23039) |
+| `DS4_METAL_DISABLE_M5_ROUTER_PROJECT_SELECT_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables M5 router project select fuse. | [ds4.c:24762](ds4.c#L24762) |
+| `DS4_METAL_DISABLE_METAL4` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Disables metal4. | [ds4_metal.m:2995](ds4_metal.m#L2995) |
+| `DS4_METAL_DISABLE_MOE_MM_ID_PAIR_SWIGLU` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables MoE MM ID pair SwiGLU. | [ds4_metal.m:44968](ds4_metal.m#L44968) |
+| `DS4_METAL_DISABLE_MOE_MM_ID_USE_RESOURCES` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables MoE MM ID use resources. | [ds4_metal.m:35169](ds4_metal.m#L35169) |
+| `DS4_METAL_DISABLE_MXFP4_SELECTED_EXPERT_VIEWS` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables MXFP4 selected expert views. | [ds4.c:21173](ds4.c#L21173) |
+| `DS4_METAL_DISABLE_PERSISTENT_ZERO_ATTN_MASK` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables persistent zero attn mask. | [ds4_metal.m:31480](ds4_metal.m#L31480) |
+| `DS4_METAL_DISABLE_PRE_M5_ATTN_INV_ROPE_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 attn inv RoPE fuse. | [ds4.c:22638](ds4.c#L22638) |
+| `DS4_METAL_DISABLE_PRE_M5_ATTN_OUT_LOW_Q8_STATIC` | presence rollback; unset: exact fixed-shape kernel is automatic on eligible pre-M5 Flash decode; any value including 0 disables | Restores the generic Q8 attention-output low projection kernel. | [ds4_metal.m:27987](ds4_metal.m#L27987) |
+| `DS4_METAL_DISABLE_PRE_M5_BATCH_INDEXER_QUERY_PRUNE` | presence rollback; unset: unused zero-prefix indexer queries are pruned before compressed rows exceed top-k; any value including 0 disables | Restores transient indexer query and weight dispatches during eligible pre-M5 prefill. | [ds4.c:30086](ds4.c#L30086) |
+| `DS4_METAL_DISABLE_PRE_M5_COMPRESSOR_EXACT_POOL_RATIO4` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 compressor exact pool ratio4. | [ds4_metal.m:26383](ds4_metal.m#L26383) |
+| `DS4_METAL_DISABLE_PRE_M5_COMPRESSOR_EXACT_REDUCTION_FUSION` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 compressor exact reduction fusion. | [ds4_metal.m:25907](ds4_metal.m#L25907) |
+| `DS4_METAL_DISABLE_PRE_M5_COMPRESSOR_QUAD_STORE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 compressor quad store. | [ds4.c:23535](ds4.c#L23535) |
+| `DS4_METAL_DISABLE_PRE_M5_COMPRESSOR_RATIO4_DECODE_PACK_FUSION` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 compressor ratio4 decode pack fusion. | [ds4_metal.m:26214](ds4_metal.m#L26214) |
+| `DS4_METAL_DISABLE_PRE_M5_COMP_FINALIZE_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 comp finalize fuse. | [ds4.c:23646](ds4.c#L23646) |
+| `DS4_METAL_DISABLE_PRE_M5_DECODE_EARLY_PIPELINE_FAST_LOOKUP` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 decode early pipeline fast lookup. | [ds4.c:28003](ds4.c#L28003) |
+| `DS4_METAL_DISABLE_PRE_M5_DECODE_EARLY_SECOND_SPLIT12` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 decode early second split12. | [ds4.c:28087](ds4.c#L28087) |
+| `DS4_METAL_DISABLE_PRE_M5_DECODE_EARLY_SPLIT3` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 decode early split3. | [ds4.c:27961](ds4.c#L27961) |
+| `DS4_METAL_DISABLE_PRE_M5_DECODE_EARLY_SPLIT5` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 decode early split5. | [ds4.c:27973](ds4.c#L27973) |
+| `DS4_METAL_DISABLE_PRE_M5_DECODE_PIPELINE_FAST_LOOKUP` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 decode pipeline fast lookup. | [ds4.c:28015](ds4.c#L28015) |
+| `DS4_METAL_DISABLE_PRE_M5_DECODE_PORTS` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 decode ports. | [ds4.c:22613](ds4.c#L22613) |
+| `DS4_METAL_DISABLE_PRE_M5_DECODE_RAW_ZERO_ATTN_MASK` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 decode raw zero attn mask. | [ds4_metal.m:29920](ds4_metal.m#L29920) |
+| `DS4_METAL_DISABLE_PRE_M5_DECODE_SECOND_SPLIT16` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 decode second split16. | [ds4.c:28095](ds4.c#L28095) |
+| `DS4_METAL_DISABLE_PRE_M5_FLASH_ATTN_BATCHED_MEMO` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 flash attn batched memo. | [ds4_metal.m:3752](ds4_metal.m#L3752) |
+| `DS4_METAL_DISABLE_PRE_M5_FLASH_ATTN_PACKED32_REDUCE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 flash attn packed32 reduce. | [ds4_metal.m:31460](ds4_metal.m#L31460) |
+| `DS4_METAL_DISABLE_PRE_M5_FLASH_ATTN_PAD_BLK_MEMO` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 flash attn pad blk memo. | [ds4_metal.m:3610](ds4_metal.m#L3610) |
+| `DS4_METAL_DISABLE_PRE_M5_HC_NORM_MIX_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 HC norm mix fuse. | [ds4.c:22867](ds4.c#L22867) |
+| `DS4_METAL_DISABLE_PRE_M5_HC_PRODUCER_PRE_NORM_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 HC producer pre norm fuse. | [ds4_metal.m:46546](ds4_metal.m#L46546) |
+| `DS4_METAL_DISABLE_PRE_M5_HEAD_RMS_ROPE_PIPELINE_STATIC` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 head RMS RoPE pipeline static. | [ds4_metal.m:10026](ds4_metal.m#L10026) |
+| `DS4_METAL_DISABLE_PRE_M5_KV_ROPE_FP8_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 KV RoPE fp8 fuse. | [ds4.c:23455](ds4.c#L23455) |
+| `DS4_METAL_DISABLE_PRE_M5_MXFP4_MM_ID_PAIR_HALF_SCALE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 MXFP4 MM ID pair half scale. | [ds4_metal.m:45129](ds4_metal.m#L45129) |
+| `DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_DECODE_FIXED_ROUTE_PAIR` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 MXFP4 MoE decode fixed route pair. | [ds4_metal.m:41948](ds4_metal.m#L41948) |
+| `DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_DECODE_FIXED_ROUTE_SUM6` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 MXFP4 MoE decode fixed route sum6. | [ds4_metal.m:41961](ds4_metal.m#L41961) |
+| `DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_DECODE_NSG1` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 MXFP4 MoE decode nsg1. | [ds4_metal.m:41719](ds4_metal.m#L41719) |
+| `DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_DECODE_STATIC_TRIP` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 MXFP4 MoE decode static trip. | [ds4_metal.m:41987](ds4_metal.m#L41987) |
+| `DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_DECODE_SUM6_FULL_ROWS` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 MXFP4 MoE decode sum6 full rows. | [ds4_metal.m:41974](ds4_metal.m#L41974) |
+| `DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_DECODE_TG_MULTIPLE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 MXFP4 MoE decode tg multiple. | [ds4_metal.m:41936](ds4_metal.m#L41936) |
+| `DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_MM_ID_DOWN_HALF_LUT` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 MXFP4 MoE MM ID down half lut. | [ds4_metal.m:45047](ds4_metal.m#L45047) |
+| `DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_MM_ID_DOWN_TAIL_SIMDGROUP_CULL` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 MXFP4 MoE MM ID down tail simdgroup cull. | [ds4_metal.m:45030](ds4_metal.m#L45030) |
+| `DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_MM_ID_MAP_SCATTER` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 MXFP4 MoE MM ID map scatter. | [ds4_metal.m:44997](ds4_metal.m#L44997) |
+| `DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_MM_ID_PAIR_SWIGLU_COMPACT_TILE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 MXFP4 MoE MM ID pair SwiGLU compact tile. | [ds4_metal.m:44981](ds4_metal.m#L44981) |
+| `DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_MM_ID_PAIR_TAIL_SIMDGROUP_CULL` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 MXFP4 MoE MM ID pair tail simdgroup cull. | [ds4_metal.m:45018](ds4_metal.m#L45018) |
+| `DS4_METAL_DISABLE_PRE_M5_PARALLEL_FULL_FFN` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 parallel full FFN. | [ds4.c:22706](ds4.c#L22706) |
+| `DS4_METAL_DISABLE_PRE_M5_Q2_DECODE_SPLIT2_32` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 q2 decode split2 32. | [ds4.c:27876](ds4.c#L27876) |
+| `DS4_METAL_DISABLE_PRE_M5_QKV_NORM_KV_STORE_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 QKV norm KV store fuse. | [ds4.c:23314](ds4.c#L23314) |
+| `DS4_METAL_DISABLE_PRE_M5_QKV_PAIR_COMPRESSOR_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 QKV pair compressor fuse. | [ds4.c:23042](ds4.c#L23042) |
+| `DS4_METAL_DISABLE_PRE_M5_QKV_PAIR_QUAD_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 QKV pair quad fuse. | [ds4.c:23038](ds4.c#L23038) |
+| `DS4_METAL_DISABLE_PRE_M5_ROUTER_SHARED_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 router shared fuse. | [ds4.c:24755](ds4.c#L24755) |
+| `DS4_METAL_DISABLE_PRE_M5_ROUTER_SIMD_FINALIZE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 router simd finalize. | [ds4_metal.m:35827](ds4_metal.m#L35827) |
+| `DS4_METAL_DISABLE_PRE_M5_ROUTER_SIMD_WEIGHTS_FUSION` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 router simd weights fusion. | [ds4_metal.m:35836](ds4_metal.m#L35836) |
+| `DS4_METAL_DISABLE_PRE_M5_ROUTER_TRANSFORM_FINALIZE_FUSION` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pre M5 router transform finalize fusion. | [ds4_metal.m:35840](ds4_metal.m#L35840) |
+| `DS4_METAL_DISABLE_PRO_Q4_EXPERT_ADDRESS_AUTO` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pro Q4 expert address auto. | [ds4_metal.m:19710](ds4_metal.m#L19710) |
+| `DS4_METAL_DISABLE_PRO_Q4_EXPERT_TABLE_AUTO` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pro Q4 expert table auto. | [ds4.c:21568](ds4.c#L21568) |
+| `DS4_METAL_DISABLE_PRO_Q4_EXPERT_TABLE_PRELOAD` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables pro Q4 expert table preload. | [ds4.c:63088](ds4.c#L63088) |
+| `DS4_METAL_DISABLE_Q4_ATTN_OUT_A_DIRECT` | presence rollback; unset enables the automatic fixed-route path for eligible Apple M1-M4 long prefills; any defined value including 0 disables | Restore the generic route-map/work-list Q4 attention output-A path instead of the bit-identical fixed-route direct kernel. | [ds4_metal.m:31562](ds4_metal.m#L31562) |
+| `DS4_METAL_DISABLE_Q4_ATTN_OUT_HC_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 attn out HC fuse. | [ds4_metal.m:47624](ds4_metal.m#L47624) |
+| `DS4_METAL_DISABLE_Q4_ATTN_OUT_TINY_BATCH` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Disables Q4 attn out tiny batch. | [ds4_metal.m:28396](ds4_metal.m#L28396) |
+| `DS4_METAL_DISABLE_Q4_BATCH_EXPERT_TABLE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 batch expert table. | [ds4_metal.m:44883](ds4_metal.m#L44883) |
+| `DS4_METAL_DISABLE_Q4_DENSE_PAIR` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 dense pair. | [ds4_metal.m:21990](ds4_metal.m#L21990) |
+| `DS4_METAL_DISABLE_Q4_EXACT_BOUNDARY` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 exact boundary. | [ds4_metal.m:42239](ds4_metal.m#L42239) |
+| `DS4_METAL_DISABLE_Q4_EXACT_TENSOR_ID` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 exact tensor ID. | [ds4_metal.m:42219](ds4_metal.m#L42219) |
+| `DS4_METAL_DISABLE_Q4_EXPERT_ADDRESS_TABLE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 expert address table. | [ds4_metal.m:19711](ds4_metal.m#L19711) |
+| `DS4_METAL_DISABLE_Q4_EXPERT_TABLE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 expert table. | [ds4.c:21041](ds4.c#L21041) |
+| `DS4_METAL_DISABLE_Q4_GATHER_SLOTS` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 gather slots. | [ds4_metal.m:42321](ds4_metal.m#L42321) |
+| `DS4_METAL_DISABLE_Q4_GROUP24_EXPERT_TABLE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 group24 expert table. | [ds4_metal.m:42201](ds4_metal.m#L42201) |
+| `DS4_METAL_DISABLE_Q4_GROUP6_EXPERT_TABLE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 group6 expert table. | [ds4_metal.m:42167](ds4_metal.m#L42167) |
+| `DS4_METAL_DISABLE_Q4_GROUP8_EXPERT_TABLE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 group8 expert table. | [ds4_metal.m:42184](ds4_metal.m#L42184) |
+| `DS4_METAL_DISABLE_Q4_GROUPED_BOUNDARY` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 grouped boundary. | [ds4_metal.m:42149](ds4_metal.m#L42149) |
+| `DS4_METAL_DISABLE_Q4_GROUPED_EXPERTS` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 grouped experts. | [ds4_metal.m:42130](ds4_metal.m#L42130) |
+| `DS4_METAL_DISABLE_Q4_MV_CLASSIC` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 MV classic. | [ds4_metal.m:21571](ds4_metal.m#L21571) |
+| `DS4_METAL_DISABLE_Q4_PREFILL_TAIL_SIMDGROUP_CULL` | presence rollback; unset: automatic on Apple M1-M4 for Q4_K single-tile prefill N=9..16 and production attn_q_b tails through N=65; any value including 0 disables | Restores the legacy four-SIMDgroup Q4_K prefill kernel on the measured short-prefill scopes. | [ds4_metal.m:22088](ds4_metal.m#L22088) |
+| `DS4_METAL_DISABLE_Q4_QKV_COMPRESSOR_FUSE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 QKV compressor fuse. | [ds4_metal.m:22083](ds4_metal.m#L22083) |
+| `DS4_METAL_DISABLE_Q4_SELECTED_EXPERT_VIEWS` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 selected expert views. | [ds4.c:21136](ds4.c#L21136) |
+| `DS4_METAL_DISABLE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Disables Q4 SSD prefill attn out exactn. | [ds4_metal.m:28095](ds4_metal.m#L28095) |
+| `DS4_METAL_DISABLE_Q4_SSD_PREFILL_ATTN_OUT_SCALE_META` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Disables shared scale/min metadata in the Q4 SSD prefill attention-output exact-N kernel. | [ds4_metal.m:28220](ds4_metal.m#L28220) |
+| `DS4_METAL_DISABLE_Q4_SSD_SESSION_UNION` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Disables Q4 SSD session union. | [ds4.c:65160](ds4.c#L65160) |
+| `DS4_METAL_DISABLE_Q4_STREAM_OVERLAP` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Disables Q4 stream overlap. | [ds4.c:65082](ds4.c#L65082) |
+| `DS4_METAL_DISABLE_Q4_TABLE_BOUNDARY` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q4 table boundary. | [ds4_metal.m:42318](ds4_metal.m#L42318) |
+| `DS4_METAL_DISABLE_Q8_DECODE_EXACT_VIEWS` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables Q8 decode exact views. | [ds4_metal.m:12851](ds4_metal.m#L12851) |
+| `DS4_METAL_DISABLE_QKV_NORM_FUSION` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Disables QKV norm fusion. | [ds4.c:20435](ds4.c#L20435) |
+| `DS4_METAL_DISABLE_QKV_PAIR_PROJ` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Disables QKV pair proj. | [ds4.c:20440](ds4.c#L20440) |
+| `DS4_METAL_DISABLE_QUEUE_RESIDENCY_SET` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables queue residency set. | [ds4_metal.m:2124](ds4_metal.m#L2124) |
+| `DS4_METAL_DISABLE_ROUTED_PAIR_SWIGLU_FUSION` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables routed pair SwiGLU fusion. | [ds4.c:18529](ds4.c#L18529) |
+| `DS4_METAL_DISABLE_ROUTER_SELECT_FUSION` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables router select fusion. | [ds4_metal.m:35816](ds4_metal.m#L35816) |
+| `DS4_METAL_DISABLE_ROUTER_WEIGHTS_BATCH_FUSION` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables router weights batch fusion. | [ds4_metal.m:36068](ds4_metal.m#L36068) |
+| `DS4_METAL_DISABLE_SHARED_DOWN_HC_FUSION` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Disables shared down HC fusion. | [ds4.c:20472](ds4.c#L20472) |
+| `DS4_METAL_DISABLE_SHARED_GATE_UP_SWIGLU_FUSION` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables shared gate up SwiGLU fusion. | [ds4.c:17567](ds4.c#L17567) |
+| `DS4_METAL_DISABLE_SHARED_KV_PAD` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables shared KV pad. | [ds4_metal.m:31510](ds4_metal.m#L31510) |
+| `DS4_METAL_DISABLE_SHARED_ROPE_COEFF` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables shared RoPE coeff. | [ds4_metal.m:6341](ds4_metal.m#L6341) |
+| `DS4_METAL_DISABLE_STREAMING_COLD_DECODE_PREFILL` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming cold decode prefill. | [ds4.c:32007](ds4.c#L32007) |
+| `DS4_METAL_DISABLE_STREAMING_COMPACT_ADDR` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming compact address. | [ds4_metal.m:14598](ds4_metal.m#L14598) |
+| `DS4_METAL_DISABLE_STREAMING_DECODE_PREFILL` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming decode prefill. | [ds4.c:31956](ds4.c#L31956) |
+| `DS4_METAL_DISABLE_STREAMING_EXPERT_ADDR_TABLE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming expert address table. | [ds4.c:18525](ds4.c#L18525) |
+| `DS4_METAL_DISABLE_STREAMING_EXPERT_COMBINED_BUFFER` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming expert combined buffer. | [ds4_metal.m:14023](ds4_metal.m#L14023) |
+| `DS4_METAL_DISABLE_STREAMING_EXPERT_EARLY_LOAD` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming expert early load. | [ds4_metal.m:17232](ds4_metal.m#L17232) |
+| `DS4_METAL_DISABLE_STREAMING_EXPERT_EVICT_DONTNEED` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming expert evict dontneed. | [ds4_metal.m:14395](ds4_metal.m#L14395) |
+| `DS4_METAL_DISABLE_STREAMING_EXPERT_HIT_VALIDATOR` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming expert hit validator. | [ds4_metal.m:14634](ds4_metal.m#L14634) |
+| `DS4_METAL_DISABLE_STREAMING_EXPERT_HOTLIST` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming expert hotlist. | [ds4.c:21290](ds4.c#L21290) |
+| `DS4_METAL_DISABLE_STREAMING_EXPERT_LIVE_INDEX` | value-aware boolean; default off; true disables and dominates ENABLE | Disables dense live-entry index and uses authoritative cache matrix. | [ds4_metal.m:15443](ds4_metal.m#L15443) |
+| `DS4_METAL_DISABLE_STREAMING_EXPERT_MASKED_ADDR` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming expert masked address. | [ds4_metal.m:14628](ds4_metal.m#L14628) |
+| `DS4_METAL_DISABLE_STREAMING_EXPERT_READAHEAD` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming expert readahead. | [ds4_metal.m:13196](ds4_metal.m#L13196) |
+| `DS4_METAL_DISABLE_STREAMING_EXPERT_SLABS` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming expert slabs. | [ds4_metal.m:14028](ds4_metal.m#L14028) |
+| `DS4_METAL_DISABLE_STREAMING_EXPERT_TIMING_SUMMARY` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming expert timing summary. | [ds4_metal.m:13062](ds4_metal.m#L13062) |
+| `DS4_METAL_DISABLE_STREAMING_FULL_EXPERT_ADDR_TABLE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming full expert address table. | [ds4_metal.m:14716](ds4_metal.m#L14716) |
+| `DS4_METAL_DISABLE_STREAMING_IQ2_CPU_ROUTER` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming IQ2 CPU router. | [ds4.c:20939](ds4.c#L20939) |
+| `DS4_METAL_DISABLE_STREAMING_LAYER_BATCH` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming layer batch. | [ds4.c:18239](ds4.c#L18239) |
+| `DS4_METAL_DISABLE_STREAMING_MADVISE_WILLNEED` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming madvise willneed. | [ds4.c:18212](ds4.c#L18212) |
+| `DS4_METAL_DISABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming prefill batch selected address. | [ds4.c:18523](ds4.c#L18523) |
+| `DS4_METAL_DISABLE_STREAMING_PREFILL_LAYER_MADVISE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming prefill layer madvise. | [ds4.c:18466](ds4.c#L18466) |
+| `DS4_METAL_DISABLE_STREAMING_PREFILL_LAYER_PAGEIN` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming prefill layer pagein. | [ds4.c:18434](ds4.c#L18434) |
+| `DS4_METAL_DISABLE_STREAMING_PREFILL_LAYER_PAGEIN_OVERLAP` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming prefill layer pagein overlap. | [ds4.c:19321](ds4.c#L19321) |
+| `DS4_METAL_DISABLE_STREAMING_PREFILL_LAYER_PREAD` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming prefill layer pread. | [ds4.c:18454](ds4.c#L18454) |
+| `DS4_METAL_DISABLE_STREAMING_PREFILL_LAYER_PREPARE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming prefill layer prepare. | [ds4.c:18446](ds4.c#L18446) |
+| `DS4_METAL_DISABLE_STREAMING_PREFILL_LAYER_PREPARE_OVERLAP` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming prefill layer prepare overlap. | [ds4.c:19319](ds4.c#L19319) |
+| `DS4_METAL_DISABLE_STREAMING_PREFILL_LAYER_READAHEAD` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming prefill layer readahead. | [ds4.c:18444](ds4.c#L18444) |
+| `DS4_METAL_DISABLE_STREAMING_PREFILL_SELECTED_ASYNC_LOAD` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming prefill selected async load. | [ds4.c:46424](ds4.c#L46424) |
+| `DS4_METAL_DISABLE_STREAMING_PREFILL_SELECTED_MADVISE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming prefill selected madvise. | [ds4.c:18424](ds4.c#L18424) |
+| `DS4_METAL_DISABLE_STREAMING_PREFILL_SELECTED_PAGEIN` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming prefill selected pagein. | [ds4.c:18414](ds4.c#L18414) |
+| `DS4_METAL_DISABLE_STREAMING_PREFILL_SELECTED_PROFILE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming prefill selected profile. | [ds4.c:18908](ds4.c#L18908) |
+| `DS4_METAL_DISABLE_STREAMING_PREFILL_SELECTED_READAHEAD` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming prefill selected readahead. | [ds4.c:19960](ds4.c#L19960) |
+| `DS4_METAL_DISABLE_STREAMING_PREFILL_SELECTED_READAHEAD_SHARED` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming prefill selected readahead shared. | [ds4.c:19970](ds4.c#L19970) |
+| `DS4_METAL_DISABLE_STREAMING_READAHEAD` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming readahead. | [ds4.c:18205](ds4.c#L18205) |
+| `DS4_METAL_DISABLE_STREAMING_SELECTED_ASYNC_EARLY_COMMIT` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming selected async early commit. | [ds4.c:21019](ds4.c#L21019) |
+| `DS4_METAL_DISABLE_STREAMING_SELECTED_ASYNC_LOAD` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming selected async load. | [ds4.c:21003](ds4.c#L21003) |
+| `DS4_METAL_DISABLE_STREAMING_SELECTED_READAHEAD_SHARED_DELAY` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming selected readahead shared delay. | [ds4.c:21720](ds4.c#L21720) |
+| `DS4_METAL_DISABLE_STREAMING_SELECTED_SHARED_OVERLAP` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming selected shared overlap. | [ds4.c:20995](ds4.c#L20995) |
+| `DS4_METAL_DISABLE_STREAMING_STATIC_DECODE_MAP` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming static decode map. | [ds4.c:18217](ds4.c#L18217) |
+| `DS4_METAL_DISABLE_STREAMING_STATIC_MAP_STATE_CACHE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming static map state cache. | [ds4.c:18230](ds4.c#L18230) |
+| `DS4_METAL_DISABLE_SUPPORT_Q8_DECODE_EXACT_VIEWS` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables support Q8 decode exact views. | [ds4_metal.m:12858](ds4_metal.m#L12858) |
+| `DS4_METAL_DISABLE_TINY_PAIR_SWIGLU_FUSION` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables tiny pair SwiGLU fusion. | [ds4_metal.m:44920](ds4_metal.m#L44920) |
+| `DS4_METAL_DISABLE_TOKEN_EMBED_EXACT_VIEW` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables token embed exact view. | [ds4_metal.m:11907](ds4_metal.m#L11907) |
+| `DS4_METAL_DISABLE_ZERO_PREFIX_PREFILL_MASK_CACHE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables zero prefix prefill mask cache. | [ds4_metal.m:2404](ds4_metal.m#L2404) |
+| `DS4_METAL_DSPARK_ACCEPTANCE_ONLY_VERIFY` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Verifies only the draft rows still needed for acceptance after the base target logit. | [ds4.c:52499](ds4.c#L52499) |
+| `DS4_METAL_DSPARK_DEVICE_PROPOSER` | boolean true values enable; false values/unset disable; NO_DEVICE_PROPOSER presence dominates | Keeps DSpark Q8 confidence/Markov proposal work on Metal and reads one compact result. | [ds4.c:34884](ds4.c#L34884) |
+| `DS4_METAL_DSPARK_EXACT2` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Enables the resident single-GPU Metal exact-2 verifier. | [ds4.c:52290](ds4.c#L52290) |
+| `DS4_METAL_DSPARK_EXACTN` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Enables the single-GPU Metal exact-N verifier. | [ds4.c:52311](ds4.c#L52311) |
+| `DS4_METAL_DSPARK_EXACTN_BATCH_HEAD` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Batches the output head across exact-N verifier rows. | [ds4.c:37726](ds4.c#L37726) |
+| `DS4_METAL_DSPARK_EXACTN_UNION` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Loads the union of experts for exact-N verifier rows once. | [ds4.c:52332](ds4.c#L52332) |
+| `DS4_METAL_DSPARK_EXACT_ROWS_ASYNC_TAILS` | presence control; unset: off/default; any value including 0 enables | Runs exact-row routed tails asynchronously after union routing. | [ds4.c:37932](ds4.c#L37932) |
+| `DS4_METAL_DSPARK_EXACT_ROWS_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for DSpark exact rows. | [ds4.c:37930](ds4.c#L37930) |
+| `DS4_METAL_DSPARK_HEADLESS_REPLAY` | unset/empty: enabled; exact 0 disables; every other nonempty value enables | Skips output heads for accepted intermediate DSpark replay tokens. | [ds4.c:52479](ds4.c#L52479) |
+| `DS4_METAL_DSPARK_NO_DEVICE_PROPOSER` | presence rollback; unset: automatic/default path; any value including 0 disables | Dominant presence-based rollback for the Metal DSpark device proposer. | [ds4.c:34886](ds4.c#L34886) |
+| `DS4_METAL_DSPARK_PIN_MAIN_PROJ` | nonempty value other than exact 0 enables; unset/empty/0 disables | mlock-pins only DSpark stage-0 main_norm/main_proj in Metal SSD streaming. | [ds4.c:39443](ds4.c#L39443) |
+| `DS4_METAL_DSPARK_PROPOSER_BLOCK_MAX` | uint32; unset: automatic cache/verifier cap; 0 or invalid: native width; positive: clamped to DSpark/native maximum | Caps rows proposed by single-device Metal DSpark. | [ds4.c:52430](ds4.c#L52430) |
+| `DS4_METAL_DSPARK_SAFE_EXPERT_COUNT` | exact 1 enables; unset or any other value disables | Caps an explicit expert-count cache request to the safe Metal working-set budget for DSpark SSD streaming. | [ds4.c:4880](ds4.c#L4880) |
+| `DS4_METAL_DSV4_HC_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the DeepSeek hidden-context Metal kernel source file loaded at runtime. | [ds4_metal.m:4938](ds4_metal.m#L4938) |
+| `DS4_METAL_DSV4_KV_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the DeepSeek KV Metal kernel source file loaded at runtime. | [ds4_metal.m:4940](ds4_metal.m#L4940) |
+| `DS4_METAL_DSV4_MISC_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the DeepSeek miscellaneous Metal kernel source file loaded at runtime. | [ds4_metal.m:4942](ds4_metal.m#L4942) |
+| `DS4_METAL_DSV4_ROPE_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the DeepSeek RoPE Metal kernel source file loaded at runtime. | [ds4_metal.m:4941](ds4_metal.m#L4941) |
+| `DS4_METAL_DUMP_PREFILL_LOGITS` | file path; unset/empty: no dump | Writes final GPU prefill logits as f32 binary. | [ds4.c:51347](ds4.c#L51347) |
+| `DS4_METAL_ENABLE_BATCH_HC_NORM_FUSION` | legacy value-aware alias; default enabled; exact 0 disables; unset/empty/every other value enables unless DISABLE is active | Legacy control for the now-default batched HC norm fusion. | [ds4.c:20462](ds4.c#L20462) |
+| `DS4_METAL_ENABLE_COMPRESSOR_EXACT_POOL_RATIO4` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables compressor exact pool ratio4. | [ds4_metal.m:26385](ds4_metal.m#L26385) |
+| `DS4_METAL_ENABLE_COMPRESSOR_PAIR_STATE_STORE` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables compressor pair state store. | [ds4_metal.m:23243](ds4_metal.m#L23243) |
+| `DS4_METAL_ENABLE_COMPRESSOR_QUAD_STORE` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables compressor quad store. | [ds4.c:23527](ds4.c#L23527) |
+| `DS4_METAL_ENABLE_DSPARK_CAPTURE_FUSED_LAST` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Enables DSpark capture fused last. | [ds4.c:28359](ds4.c#L28359) |
+| `DS4_METAL_ENABLE_GATHERED_KV_STAGE` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables gathered KV stage. | [ds4_metal.m:29675](ds4_metal.m#L29675) |
+| `DS4_METAL_ENABLE_GLM_STREAMING_SELECTED_ASYNC_LOAD` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables GLM streaming selected async load. | [ds4.c:44335](ds4.c#L44335) |
+| `DS4_METAL_ENABLE_HC_NORM_MIX_FUSE` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables HC norm mix fuse. | [ds4.c:22870](ds4.c#L22870) |
+| `DS4_METAL_ENABLE_HC_PRODUCER_PRE_NORM_FUSE` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables HC producer pre norm fuse. | [ds4_metal.m:46552](ds4_metal.m#L46552) |
+| `DS4_METAL_ENABLE_IQ2_SELECTED_ASYNC_EARLY_COMMIT` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables IQ2 selected async early commit. | [ds4.c:21018](ds4.c#L21018) |
+| `DS4_METAL_ENABLE_IQ2_XXS_SSD_PREFILL_MM` | value-aware boolean; default automatic/on for eligible shape; explicit 0 turns request off unless REQUIRE=1 | Overrides automatic IQ2_XXS/Q2_K grouped address-MM selection for SSD prefill. | [ds4_metal.m:44780](ds4_metal.m#L44780) |
+| `DS4_METAL_ENABLE_PRO_Q4_EXPERT_ADDRESS_AUTO` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables pro Q4 expert address auto. | [ds4.c:20982](ds4.c#L20982) |
+| `DS4_METAL_ENABLE_PRO_Q4_EXPERT_TABLE_AUTO` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables pro Q4 expert table auto. | [ds4.c:20981](ds4.c#L20981) |
+| `DS4_METAL_ENABLE_PRO_Q4_SELECTED_EXPERT_VIEWS` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables pro Q4 selected expert views. | [ds4.c:20978](ds4.c#L20978) |
+| `DS4_METAL_ENABLE_Q4_ATTN_OUT_TINY_BATCH` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Enables Q4 attn out tiny batch. | [ds4_metal.m:28405](ds4_metal.m#L28405) |
+| `DS4_METAL_ENABLE_Q4_BATCH_EXPERT_TABLE` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables Q4 batch expert table. | [ds4_metal.m:44869](ds4_metal.m#L44869) |
+| `DS4_METAL_ENABLE_Q4_EXACT_TENSOR_ID` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables Q4 exact tensor ID. | [ds4_metal.m:42218](ds4_metal.m#L42218) |
+| `DS4_METAL_ENABLE_Q4_EXPERT_ADDRESS_TABLE` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables Q4 expert address table. | [ds4.c:20980](ds4.c#L20980) |
+| `DS4_METAL_ENABLE_Q4_EXPERT_TABLE` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables Q4 expert table. | [ds4.c:20979](ds4.c#L20979) |
+| `DS4_METAL_ENABLE_Q4_GATHER_SLOTS` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables Q4 gather slots. | [ds4_metal.m:42320](ds4_metal.m#L42320) |
+| `DS4_METAL_ENABLE_Q4_GROUP24_EXPERT_TABLE` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables Q4 group24 expert table. | [ds4_metal.m:42200](ds4_metal.m#L42200) |
+| `DS4_METAL_ENABLE_Q4_GROUP6_EXPERT_TABLE` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables Q4 group6 expert table. | [ds4_metal.m:42166](ds4_metal.m#L42166) |
+| `DS4_METAL_ENABLE_Q4_GROUP8_EXPERT_TABLE` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables Q4 group8 expert table. | [ds4_metal.m:42183](ds4_metal.m#L42183) |
+| `DS4_METAL_ENABLE_Q4_GROUPED_EXPERTS` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables Q4 grouped experts. | [ds4_metal.m:42129](ds4_metal.m#L42129) |
+| `DS4_METAL_ENABLE_Q4_QKV_COMPRESSOR_FUSE` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables Q4 QKV compressor fuse. | [ds4.c:23140](ds4.c#L23140) |
+| `DS4_METAL_ENABLE_Q4_SELECTED_EXPERT_VIEWS` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables Q4 selected expert views. | [ds4.c:20977](ds4.c#L20977) |
+| `DS4_METAL_ENABLE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Enables Q4 SSD prefill attn out exactn. | [ds4_metal.m:28097](ds4_metal.m#L28097) |
+| `DS4_METAL_ENABLE_Q4_SSD_SESSION_UNION` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Enables Q4 SSD session union. | [ds4.c:65171](ds4.c#L65171) |
+| `DS4_METAL_ENABLE_Q4_STREAM_OVERLAP` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Enables Q4 stream overlap. | [ds4.c:65080](ds4.c#L65080) |
+| `DS4_METAL_ENABLE_Q8_DECODE_EXACT_VIEWS` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables Q8 decode exact views. | [ds4_metal.m:12865](ds4_metal.m#L12865) |
+| `DS4_METAL_ENABLE_Q8_QKV_COMPRESSOR_FUSE` | nonempty boolean; unset/empty/exact 0: no streamed/union opt-in; other values enable; eligible resident full-decode remains automatic | Extends the automatic resident Q8 QKV/compressor compound fusion to SSD streaming or exact-N union scope. | [ds4.c:23027](ds4.c#L23027) |
+| `DS4_METAL_ENABLE_STREAMING_COMPACT_ADDR` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming compact address. | [ds4_metal.m:14597](ds4_metal.m#L14597) |
+| `DS4_METAL_ENABLE_STREAMING_EXPERT_ADDR_TABLE` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming expert address table. | [ds4_metal.m:14604](ds4_metal.m#L14604) |
+| `DS4_METAL_ENABLE_STREAMING_EXPERT_EVICT_DONTNEED` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming expert evict dontneed. | [ds4_metal.m:14394](ds4_metal.m#L14394) |
+| `DS4_METAL_ENABLE_STREAMING_EXPERT_HIT_VALIDATOR` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming expert hit validator. | [ds4_metal.m:14605](ds4_metal.m#L14605) |
+| `DS4_METAL_ENABLE_STREAMING_EXPERT_LIVE_INDEX` | value-aware boolean; default automatic/on for validated IQ2 cache shape; explicit 0 disables | Overrides automatic dense live-entry index selection. | [ds4_metal.m:15442](ds4_metal.m#L15442) |
+| `DS4_METAL_ENABLE_STREAMING_EXPERT_MASKED_ADDR` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming expert masked address. | [ds4_metal.m:14606](ds4_metal.m#L14606) |
+| `DS4_METAL_ENABLE_STREAMING_FULL_EXPERT_ADDR_TABLE` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming full expert address table. | [ds4_metal.m:14715](ds4_metal.m#L14715) |
+| `DS4_METAL_ENABLE_STREAMING_IQ2_CPU_ROUTER` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming IQ2 CPU router. | [ds4.c:20938](ds4.c#L20938) |
+| `DS4_METAL_ENABLE_STREAMING_MADVISE_WILLNEED` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming madvise willneed. | [ds4.c:18210](ds4.c#L18210) |
+| `DS4_METAL_ENABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming prefill batch selected address. | [ds4_metal.m:14609](ds4_metal.m#L14609) |
+| `DS4_METAL_ENABLE_STREAMING_PREFILL_CACHE_SEED` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming prefill cache seed. | [ds4.c:21259](ds4.c#L21259) |
+| `DS4_METAL_ENABLE_STREAMING_PREFILL_EXPERT_READAHEAD` | value-aware boolean; for batches <32 readahead is automatic; for batches >=32 unset/false disables and true enables; global READHEAD rollback and F_NOCACHE still dominate | Restores F_RDADVISE immediately before parallel pread for large SSD-prefill batches. | [ds4_metal.m:13212](ds4_metal.m#L13212) |
+| `DS4_METAL_ENABLE_STREAMING_PREFILL_LAYER_PAGEIN` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming prefill layer pagein. | [ds4.c:18432](ds4.c#L18432) |
+| `DS4_METAL_ENABLE_STREAMING_PREFILL_LAYER_READAHEAD` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming prefill layer readahead. | [ds4.c:18442](ds4.c#L18442) |
+| `DS4_METAL_ENABLE_STREAMING_PREFILL_SELECTED_MADVISE` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming prefill selected madvise. | [ds4.c:18422](ds4.c#L18422) |
+| `DS4_METAL_ENABLE_STREAMING_PREFILL_SELECTED_PAGEIN` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming prefill selected pagein. | [ds4.c:18412](ds4.c#L18412) |
+| `DS4_METAL_ENABLE_STREAMING_PREFILL_SELECTED_READAHEAD` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming prefill selected readahead. | [ds4.c:19956](ds4.c#L19956) |
+| `DS4_METAL_ENABLE_STREAMING_PREFILL_SELECTED_READAHEAD_SHARED` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming prefill selected readahead shared. | [ds4.c:19958](ds4.c#L19958) |
+| `DS4_METAL_ENABLE_STREAMING_READAHEAD` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming readahead. | [ds4.c:18203](ds4.c#L18203) |
+| `DS4_METAL_ENABLE_STREAMING_SELECTED_READAHEAD_SHARED_DELAY` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming selected readahead shared delay. | [ds4.c:21719](ds4.c#L21719) |
+| `DS4_METAL_ENABLE_STREAMING_STATIC_DECODE_MAP` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables streaming static decode map. | [ds4.c:18222](ds4.c#L18222) |
+| `DS4_METAL_ENABLE_TOKEN_EMBED_EXACT_VIEW` | presence opt-in; unset: off/automatic; any value including 0 enables | Enables token embed exact view. | [ds4_metal.m:12219](ds4_metal.m#L12219) |
+| `DS4_METAL_EXACT_VIEW_CACHE_GIB` | unsigned GiB; default 64; 0 disables size-triggered eviction; MIB overrides it | Sets the cached exact-model-view eviction threshold. | [ds4_metal.m:1333](ds4_metal.m#L1333) |
+| `DS4_METAL_EXACT_VIEW_CACHE_MIB` | unsigned MiB; unset: inherit GIB/default; 0 disables size-triggered eviction; overrides GIB | Sets the cached exact-model-view eviction threshold with MiB precision. | [ds4_metal.m:1342](ds4_metal.m#L1342) |
+| `DS4_METAL_EXACT_VIEW_CACHE_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for exact view cache. | [ds4_metal.m:1377](ds4_metal.m#L1377) |
+| `DS4_METAL_FLASH_ATTN_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the FlashAttention Metal kernel source file loaded at runtime. | [ds4_metal.m:4935](ds4_metal.m#L4935) |
+| `DS4_METAL_FLASH_ATTN_STAGE_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for flash attn stage. | [ds4.c:65152](ds4.c#L65152) |
+| `DS4_METAL_FLASH_ATTN_STAGE_PROFILE_FILTER` | substring; unset/empty: all profiled modes/stages | Filters FlashAttention stage-profile output by mode or stage substring. | [ds4_metal.m:11178](ds4_metal.m#L11178) |
+| `DS4_METAL_GET_ROWS_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the get-rows Metal kernel source file loaded at runtime. | [ds4_metal.m:4946](ds4_metal.m#L4946) |
+| `DS4_METAL_GLM_DISABLE_STREAMING_EXPERT_CACHE` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming expert cache for GLM. | [ds4_metal.m:39667](ds4_metal.m#L39667) |
+| `DS4_METAL_GLM_DISABLE_STREAMING_GROUPED_ADDR_PREFILL` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming grouped address prefill for GLM. | [ds4_metal.m:41081](ds4_metal.m#L41081) |
+| `DS4_METAL_GLM_DISABLE_STREAMING_SEED_BEFORE_PREFILL` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming seed before prefill for GLM. | [ds4.c:51074](ds4.c#L51074) |
+| `DS4_METAL_GLM_DISABLE_STREAMING_TOKEN_PREFILL` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables streaming token prefill for GLM. | [ds4.c:49704](ds4.c#L49704) |
+| `DS4_METAL_GLM_MOE_ONE_STAGE_PROFILE` | unset: off; 1/true/yes/on/all enables all layers; accepts layer lists/ranges; 0/false/no/off disables | Prints timing/profile diagnostics for GLM MoE one stage. | [ds4_metal.m:39964](ds4_metal.m#L39964) |
+| `DS4_METAL_GLM_MOE_ONE_STAGE_PROFILE_LAYER` | layer index/list/ranges or all; unset: all layers selected by profiler | Restricts GLM one-stage MoE profiling to selected layers. | [ds4_metal.m:39965](ds4_metal.m#L39965) |
+| `DS4_METAL_GLM_MOE_STAGE_PROFILE_FILTER` | substring; unset/empty: all profiled stages | Filters GLM MoE stage-profile output. | [ds4_metal.m:39968](ds4_metal.m#L39968) |
+| `DS4_METAL_GLM_QKLOW_DEBUG` | presence diagnostic; unset: off; any value including 0 enables | Enables debug diagnostics for GLM qklow. | [ds4_metal.m:37868](ds4_metal.m#L37868) |
+| `DS4_METAL_GLM_STREAMING_ASYNC_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for GLM streaming async. | [ds4.c:44362](ds4.c#L44362) |
+| `DS4_METAL_GLM_STREAMING_DECODE_FULL_LAYER_MAP` | presence control; unset: off/default; any value including 0 enables | Maps complete GLM layers during SSD-streaming decode instead of decode-only spans. | [ds4.c:42616](ds4.c#L42616) |
+| `DS4_METAL_GLM_STREAMING_DECODE_SYNC_EACH_LAYER` | boolean text; Metal runtime always synchronizes and does not consult it; legacy fallback name read only in ROCm builds | Controls per-layer GLM streaming decode synchronization only as a legacy ROCm fallback alias. | [ds4.c:49780](ds4.c#L49780) |
+| `DS4_METAL_GLM_STREAMING_PREFILL_FULL_LAYER` | presence control; unset: off/default; any value including 0 enables | Forces full-layer GLM SSD prefill regardless of the token crossover. | [ds4_metal.m:14710](ds4_metal.m#L14710) |
+| `DS4_METAL_GLM_STREAMING_PREFILL_FULL_LAYER_MIN_TOKENS` | positive uint32; default 64 on Metal, 1024 when used as ROCm fallback; 0/invalid restores default | Sets the token crossover for GLM full-layer SSD prefill. | [ds4.c:42693](ds4.c#L42693) |
+| `DS4_METAL_GLM_STREAMING_PREFILL_SYNC_EACH_LAYER` | boolean text; Metal runtime always synchronizes and does not consult it; legacy fallback name read only in ROCm builds | Controls per-layer GLM streaming prefill synchronization only as a legacy ROCm fallback alias. | [ds4.c:42396](ds4.c#L42396) |
+| `DS4_METAL_GLM_STREAMING_TOKEN_PREFILL_MAX` | uint32; default 64 on Metal, 0 when used as ROCm fallback; 0 disables; invalid restores default | Sets largest GLM SSD prefill handled token-major by the decode graph. | [ds4.c:49683](ds4.c#L49683) |
+| `DS4_METAL_GLU_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the GLU Metal kernel source file loaded at runtime. | [ds4_metal.m:4950](ds4_metal.m#L4950) |
+| `DS4_METAL_GPU_BATCH_EMBED_MIN` | uint32 token threshold; default 512; invalid restores default | Sets the batch size at which prompt embedding moves from CPU upload to Metal kernels. | [ds4.c:28867](ds4.c#L28867) |
+| `DS4_METAL_GPU_BUSY_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for GPU busy. | [ds4_metal.m:1293](ds4_metal.m#L1293) |
+| `DS4_METAL_GRAPH_DUMP_LAYER` | unsigned layer index or all; unset: every layer | Restricts graph tensor dumps to one layer. | [ds4.c:16863](ds4.c#L16863) |
+| `DS4_METAL_GRAPH_DUMP_LOGITS` | file path; unset/empty: no graph-logit dump | Writes Metal graph-test logits as f32 binary. | [ds4.c:39080](ds4.c#L39080) |
+| `DS4_METAL_GRAPH_DUMP_NAME` | substring; unset/empty: every tensor name | Restricts graph tensor dumps by tensor-name substring. | [ds4.c:16859](ds4.c#L16859) |
+| `DS4_METAL_GRAPH_DUMP_POS` | unsigned token position; unset: every position | Restricts graph tensor dumps to one token position. | [ds4.c:16870](ds4.c#L16870) |
+| `DS4_METAL_GRAPH_DUMP_PREFIX` | path/prefix; unset/empty: tensor dumping disabled | Enables graph tensor dumps and supplies the filename prefix. | [ds4.c:65148](ds4.c#L65148) |
+| `DS4_METAL_GRAPH_DUMP_TRACE` | presence diagnostic; unset: off; any value including 0 enables | Emits trace diagnostics for graph dump. | [ds4.c:16900](ds4.c#L16900) |
+| `DS4_METAL_GRAPH_OUTPUT_ROW` | zero-based row smaller than current batch; default final row; invalid restores final row | Chooses which prefill output row is projected to logits. | [ds4.c:35846](ds4.c#L35846) |
+| `DS4_METAL_GRAPH_PREFILL_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for graph prefill. | [ds4.c:35523](ds4.c#L35523) |
+| `DS4_METAL_GRAPH_PREFILL_SPLIT_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for graph prefill split. | [ds4.c:66204](ds4.c#L66204) |
+| `DS4_METAL_GRAPH_PROMPT_TOKENS` | integer 1..prompt length; default full prompt | Limits prompt length used by the Metal graph parity test. | [ds4.c:39023](ds4.c#L39023) |
+| `DS4_METAL_GRAPH_RAW_CAP` | positive rows; default from SWA window+prefill; clamped to [raw_window,min(ctx,8192)] | Overrides raw sliding-window KV ring capacity. | [ds4.c:38390](ds4.c#L38390) |
+| `DS4_METAL_GRAPH_TEACHER_FORCE` | presence control; unset: off/default; any value including 0 enables | Feeds CPU reference state back into the first-token graph trace at each layer. | [ds4.c:27742](ds4.c#L27742) |
+| `DS4_METAL_GRAPH_TOKEN_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for graph token. | [ds4.c:32382](ds4.c#L32382) |
+| `DS4_METAL_GRAPH_TOKEN_SECOND_SPLIT_LAYERS` | integer 0..layer count; default 0 plus eligible automatic pre-M5 schedules; explicit value wins | Overrides second command-buffer split layer for token decode. | [ds4.c:28029](ds4.c#L28029) |
+| `DS4_METAL_GRAPH_TOKEN_SPLIT_LAYERS` | integer 0..layer count; default 4 on Apple and 0 elsewhere, with eligible pre-M5 adaptive override | Overrides first command-buffer split layer for token decode. | [ds4.c:27915](ds4.c#L27915) |
+| `DS4_METAL_GRAPH_TRACE_CACHE` | presence control; unset: off/default; any value including 0 enables | Prints raw KV cache parity diagnostics in the graph prompt test. | [ds4.c:39092](ds4.c#L39092) |
+| `DS4_METAL_GRAPH_TRACE_COMP` | presence control; unset: off/default; any value including 0 enables | Prints compressed-cache parity diagnostics in the graph prompt test. | [ds4.c:39093](ds4.c#L39093) |
+| `DS4_METAL_GRAPH_TRACE_LAYERS` | presence control; unset: off/default; any value including 0 enables | Enables per-layer first-token CPU/GPU graph tracing. | [ds4.c:27739](ds4.c#L27739) |
+| `DS4_METAL_GRAPH_TRACE_STAGE_LAYER` | signed layer index; unset gives -1/no stage-layer selection | Selects the layer used by first-token stage tracing. | [ds4.c:27743](ds4.c#L27743) |
+| `DS4_METAL_HC_NORM_FUSION_CHECK` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Compares fused HC normalization against the reference result. | [ds4.c:20521](ds4.c#L20521) |
+| `DS4_METAL_HC_NORM_FUSION_CHECK_TOL` | positive finite float; default 2e-4; invalid/nonpositive restores default | Sets the numerical tolerance for the HC norm-fusion oracle. | [ds4.c:20530](ds4.c#L20530) |
+| `DS4_METAL_HC_STABLE` | boolean empty/1/true/yes/on vs 0/false/no/off; default on | Compiles stable hidden-context drift arithmetic into the Metal library. | [ds4_metal.m:7047](ds4_metal.m#L7047) |
+| `DS4_METAL_INDEXER_STAGE_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for indexer stage. | [ds4.c:17569](ds4.c#L17569) |
+| `DS4_METAL_IQ2_XXS_SSD_PREFILL_MM_STATS` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Collects/prints statistics for IQ2 XXS SSD prefill MM. | [ds4_metal.m:6980](ds4_metal.m#L6980) |
+| `DS4_METAL_KV_RAW_F32` | boolean empty/1/true/yes/on vs 0/false/no/off; default off | Compiles raw KV storage as F32 for drift diagnosis. | [ds4_metal.m:7049](ds4_metal.m#L7049) |
+| `DS4_METAL_LAYER_STAGE_PROFILE` | unset: off; 1/true/yes/on/all enables all layers; a layer index selects one; 0/false/no/off disables | Prints timing/profile diagnostics for layer stage. | [ds4.c:65150](ds4.c#L65150) |
+| `DS4_METAL_LAYER_STAGE_PROFILE_LAYER` | single unsigned layer index; unset/empty: all layers enabled by the parent profile; invalid matches no layer | Restricts the corresponding shared graph stage profiler to one layer. | [ds4.c:29109](ds4.c#L29109) |
+| `DS4_METAL_MATH_SAFE` | boolean empty/1/true/yes/on vs 0/false/no/off; default off | Compiles Metal shaders with strict/safe IEEE math instead of fast math. | [ds4_metal.m:7051](ds4_metal.m#L7051) |
+| `DS4_METAL_MEMORY_REPORT` | presence control; unset: off/default; any value including 0 enables | Prints Metal allocation/cache/residency memory reports. | [ds4.c:39047](ds4.c#L39047) |
+| `DS4_METAL_MODEL_UNTRACKED` | presence control; unset: off/default; any value including 0 enables | Creates mapped model buffers with untracked Metal hazard tracking. | [ds4_metal.m:1547](ds4_metal.m#L1547) |
+| `DS4_METAL_MODEL_VIEW_MAX_GIB` | positive integer GiB; default device maximum (128-GiB cap for already-split span maps); cannot exceed device maximum | Caps each no-copy mapped Metal model view. | [ds4_metal.m:2216](ds4_metal.m#L2216) |
+| `DS4_METAL_MODEL_WARMUP_STRIDE_KB` | integer 1..1048576 KiB, at least one page; unset inherits MB/default; overrides STRIDE_MB | Sets the model-view warmup touch stride with KiB precision. | [ds4_metal.m:3057](ds4_metal.m#L3057) |
+| `DS4_METAL_MODEL_WARMUP_STRIDE_MB` | integer 1..1024 MiB; default 1 MiB; STRIDE_KB overrides | Sets the model-view warmup touch stride. | [ds4_metal.m:3049](ds4_metal.m#L3049) |
+| `DS4_METAL_MOE_MM_ID_USE_RESOURCES` | presence control; unset: off/default; any value including 0 enables | Declares MM-ID MoE resource usage explicitly on the command encoder. | [ds4_metal.m:35168](ds4_metal.m#L35168) |
+| `DS4_METAL_MOE_ONE_STAGE_PROFILE` | unset: off; 1/true/yes/on/all enables all layers; accepts layer lists/ranges; 0/false/no/off disables | Prints timing/profile diagnostics for MoE one stage. | [ds4.c:22714](ds4.c#L22714) |
+| `DS4_METAL_MOE_ONE_STAGE_PROFILE_LAYER` | layer index/list/ranges or all; unset: all profiler-selected layers | Restricts one-stage MoE profiling to selected layers. | [ds4_metal.m:43436](ds4_metal.m#L43436) |
+| `DS4_METAL_MOE_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the MoE Metal kernel source file loaded at runtime. | [ds4_metal.m:4937](ds4_metal.m#L4937) |
+| `DS4_METAL_MOE_STAGE_PROFILE` | unset: off; 1/true/yes/on/all enables all layers; accepts layer lists/ranges; 0/false/no/off disables | Prints timing/profile diagnostics for MoE stage. | [ds4.c:65154](ds4.c#L65154) |
+| `DS4_METAL_MOE_STAGE_PROFILE_FILTER` | substring; unset/empty: all profiled stages | Filters MoE stage-profile output. | [ds4_metal.m:43438](ds4_metal.m#L43438) |
+| `DS4_METAL_MOE_STAGE_PROFILE_LAYER` | layer index/list/ranges or all; unset: all profiler-selected layers | Restricts batched MoE stage profiling to selected layers. | [ds4_metal.m:45346](ds4_metal.m#L45346) |
+| `DS4_METAL_MOE_WRITE_CLAMPED_ACT` | presence control; unset: off/default; any value including 0 enables | Makes routed MoE write the clamped activation diagnostic. | [ds4.c:18527](ds4.c#L18527) |
+| `DS4_METAL_NORM_RSQRT_DISABLE` | boolean empty/1/true/yes/on vs 0/false/no/off; default on | Compiles unified normalization-rsqrt arithmetic into the Metal library. | [ds4_metal.m:7048](ds4_metal.m#L7048) |
+| `DS4_METAL_NORM_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the normalization Metal kernel source file loaded at runtime. | [ds4_metal.m:4951](ds4_metal.m#L4951) |
+| `DS4_METAL_NO_MODEL_WARMUP` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables model warmup. | [ds4_metal.m:2309](ds4_metal.m#L2309) |
+| `DS4_METAL_NO_PREFILL_KERNEL_WARMUP` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables prefill kernel warmup. | [ds4.c:28965](ds4.c#L28965) |
+| `DS4_METAL_NO_RESIDENCY` | presence rollback; unset: automatic/default path; any value including 0 disables | Disables residency. | [ds4_metal.m:2092](ds4_metal.m#L2092) |
+| `DS4_METAL_OUTPUT_STAGE_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for output stage. | [ds4.c:17570](ds4.c#L17570) |
+| `DS4_METAL_PREFILL_CHUNK` | positive token count used only when CLI chunk is absent; default full prompt, or 4096 for long non-PRO and 8192 for long PRO prompts; <=0 keeps automatic/full prompt | Provides the historical environment fallback for prefill chunk size. | [ds4.c:12802](ds4.c#L12802) |
+| `DS4_METAL_PRO_Q4_CPU_ROUTER` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Uses the CPU router for PRO Q4 selected-expert decode. | [ds4.c:20934](ds4.c#L20934) |
+| `DS4_METAL_PRO_Q4_CPU_ROUTER_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for pro Q4 CPU router. | [ds4.c:21644](ds4.c#L21644) |
+| `DS4_METAL_Q4_ADDR_USE_RESOURCES` | presence control; unset: off/default; any value including 0 enables | Declares Q4 address-table resources explicitly on encoders. | [ds4_metal.m:20259](ds4_metal.m#L20259) |
+| `DS4_METAL_Q4_EXPERT_GROUP_SIZE` | positive uint32; default 32; clamped to total expert count | Sets experts processed per grouped Q4 dispatch. | [ds4_metal.m:34088](ds4_metal.m#L34088) |
+| `DS4_METAL_Q4_EXPERT_TABLE_GROUP_SIZE` | integer 2..total experts; default/invalid 1 (ungrouped) | Sets grouped exact-view width while building Q4 expert tables. | [ds4_metal.m:19604](ds4_metal.m#L19604) |
+| `DS4_METAL_Q4_EXPERT_TABLE_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for Q4 expert table. | [ds4_metal.m:20103](ds4_metal.m#L20103) |
+| `DS4_METAL_Q4_GROUP24_BASE_VIEWS` | presence control; unset: off/default; any value including 0 enables | Uses broad base model views for Q4 group-24 instead of exact views. | [ds4_metal.m:42205](ds4_metal.m#L42205) |
+| `DS4_METAL_Q4_GROUP24_EXACT_VIEWS` | presence control; unset: off/default; any value including 0 enables | Uses exact mapped views for Q4 group-24 experts. | [ds4_metal.m:42204](ds4_metal.m#L42204) |
+| `DS4_METAL_Q4_GROUPED_CACHE_VIEWS` | presence control; unset: off/default; any value including 0 enables | Caches exact Q4 grouped expert views. | [ds4_metal.m:42151](ds4_metal.m#L42151) |
+| `DS4_METAL_Q4_PRO_MAP_GROUPS` | positive divisor of 384 in 1..384; default/invalid 1 | Splits each 384-expert PRO Q4 tensor into this many mapped views. | [ds4.c:6276](ds4.c#L6276) |
+| `DS4_METAL_Q4_SELECTED_EXACT_VIEWS` | presence control; unset: off/default; any value including 0 enables | Forces exact/cached views for selected Q4 experts instead of base views. | [ds4_metal.m:42528](ds4_metal.m#L42528) |
+| `DS4_METAL_Q4_SELECTED_OVERLAP_SHARED` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Overlaps selected Q4 expert preparation with the shared expert. | [ds4.c:20962](ds4.c#L20962) |
+| `DS4_METAL_Q4_SELECTED_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for Q4 selected. | [ds4.c:37944](ds4.c#L37944) |
+| `DS4_METAL_Q4_SELECTED_PROFILE_LAYER` | single nonnegative layer index; unset: every layer | Restricts legacy Q4 selected-expert profiling to one layer. | [ds4_metal.m:42509](ds4_metal.m#L42509) |
+| `DS4_METAL_Q4_SELECTED_SHARED_EVENT` | presence control; unset: off/default; any value including 0 enables | Coordinates selected Q4 work with a shared Metal event. | [ds4_metal.m:42524](ds4_metal.m#L42524) |
+| `DS4_METAL_Q4_SELECTED_TRANSIENT_VIEWS` | presence control; unset: off/default; any value including 0 enables | Uses transient exact views for selected Q4 experts. | [ds4_metal.m:42532](ds4_metal.m#L42532) |
+| `DS4_METAL_Q4_SELECTED_USE_BASE_VIEWS` | presence control; unset: off/default; any value including 0 enables | Uses broad base model views for selected Q4 experts. | [ds4_metal.m:42527](ds4_metal.m#L42527) |
+| `DS4_METAL_Q4_TABLE_BIND_ANCHORS` | presence control; unset: off/default; any value including 0 enables | Binds anchor buffers alongside the Q4 expert address table. | [ds4_metal.m:19716](ds4_metal.m#L19716) |
+| `DS4_METAL_Q4_TABLE_MODEL_RESIDENCY_SET` | presence control; unset: off/default; any value including 0 enables | Adds Q4 expert table allocations to the model residency set. | [ds4_metal.m:19657](ds4_metal.m#L19657) |
+| `DS4_METAL_Q4_TABLE_PER_TENSOR_RESIDENCY_SET` | presence control; unset: off/default; any value including 0 enables | Builds separate residency sets per Q4 expert tensor. | [ds4_metal.m:19760](ds4_metal.m#L19760) |
+| `DS4_METAL_Q4_TABLE_QUEUE_RESIDENCY_SET` | presence control; unset: off/default; any value including 0 enables | Attaches Q4 expert table residency sets to command queues. | [ds4_metal.m:19615](ds4_metal.m#L19615) |
+| `DS4_METAL_Q4_TABLE_RESIDENCY_SET` | presence control; unset: off/default; any value including 0 enables | Enables Q4 expert table residency-set handling. | [ds4_metal.m:19759](ds4_metal.m#L19759) |
+| `DS4_METAL_Q4_TABLE_USE_RESOURCES` | presence control; unset: off/default; any value including 0 enables | Declares Q4 table resources explicitly on encoders. | [ds4_metal.m:20258](ds4_metal.m#L20258) |
+| `DS4_METAL_Q8_DECODE_EXACT_VIEW_MAX_MIB` | integer 1..4096 MiB; default 1024; above max clamps, below min/invalid restores default | Caps weight ranges eligible for Q8 exact model views. | [ds4_metal.m:12876](ds4_metal.m#L12876) |
+| `DS4_METAL_Q8_MV_EXT_MAX_TOKENS` | integer 2..128; default 16; above max clamps, below min/invalid restores default | Sets largest batch handled by extended Q8 matvec. | [ds4_metal.m:21126](ds4_metal.m#L21126) |
+| `DS4_METAL_Q8_MV_NSG` | integer 1..8 simdgroups; default 4, or 2 with TP world=2; above max clamps, below min/invalid restores default | Overrides simdgroups per Q8 matvec threadgroup. | [ds4.c:22730](ds4.c#L22730) |
+| `DS4_METAL_Q8_PREFILL_PROFILE` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Prints timing/profile diagnostics for Q8 prefill. | [ds4_metal.m:21280](ds4_metal.m#L21280) |
+| `DS4_METAL_Q8_PREFILL_PROFILE_FILTER` | substring matched against generated operation label; unset/empty: all eligible calls | Filters Q8 prefill profiling. | [ds4_metal.m:21295](ds4_metal.m#L21295) |
+| `DS4_METAL_Q_STAGE_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for q stage. | [ds4.c:29270](ds4.c#L29270) |
+| `DS4_METAL_REPEAT_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the repeat Metal kernel source file loaded at runtime. | [ds4_metal.m:4949](ds4_metal.m#L4949) |
+| `DS4_METAL_REQUIRE_COMPRESSOR_EXACT_POOL_RATIO4` | presence strict check; unset: fallback allowed; any value including 0 requires the path | Requires compressor exact pool ratio4 and makes eligible fallback fail closed. | [ds4_metal.m:26387](ds4_metal.m#L26387) |
+| `DS4_METAL_REQUIRE_DECODE_ARGMAX_TOP1` | presence fail-closed assertion for rows of at least 4096 logits; unset permits generic fallback; any defined value including empty or 0 rejects disabled, ineligible, or failed dedicated top-1 preflight | Require the dedicated Metal top-1 reduction so correctness and performance oracles cannot silently exercise generic argsort. | [ds4_metal.m:21515](ds4_metal.m#L21515) |
+| `DS4_METAL_REQUIRE_EXACT_ROWS_PERSISTENT_CACHE` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Requires exact rows persistent cache and makes eligible fallback fail closed. | [ds4_metal.m:13002](ds4_metal.m#L13002) |
+| `DS4_METAL_REQUIRE_GATHERED_KV_STAGE` | presence strict check; unset: fallback allowed; any value including 0 requires the path | Requires gathered KV stage and makes eligible fallback fail closed. | [ds4_metal.m:29680](ds4_metal.m#L29680) |
+| `DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM` | value-aware boolean; default implicit fail-closed only with complete selected-address domain; explicit 1 is strict, 0 permits fallback | Makes eligible IQ2_XXS/Q2_K grouped SSD-prefill MM fail closed. | [ds4_metal.m:44782](ds4_metal.m#L44782) |
+| `DS4_METAL_REQUIRE_M1_IQ2_MID_ONLY` | presence strict check; unset: fallback allowed; any value including 0 requires the path | Requires M1 IQ2 mid only and makes eligible fallback fail closed. | [ds4_metal.m:42074](ds4_metal.m#L42074) |
+| `DS4_METAL_REQUIRE_OUTPUT_HC_WEIGHTS4` | presence strict check; unset: fallback allowed; any value including 0 requires the path | Requires output HC weights4 and makes eligible fallback fail closed. | [ds4_metal.m:46789](ds4_metal.m#L46789) |
+| `DS4_METAL_REQUIRE_Q4_ATTN_OUT_A_DIRECT` | presence strict check; unset permits the automatic path or fallback; any defined value including 0 requires the direct kernel and DISABLE wins | Require the bit-identical fixed-route Q4 attention output-A kernel and fail closed before dispatch when its production contract is unavailable. | [ds4_metal.m:31560](ds4_metal.m#L31560) |
+| `DS4_METAL_REQUIRE_Q4_ATTN_OUT_TINY_BATCH` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Requires Q4 attn out tiny batch and makes eligible fallback fail closed. | [ds4_metal.m:28373](ds4_metal.m#L28373) |
+| `DS4_METAL_REQUIRE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Requires Q4 SSD prefill attn out exactn and makes eligible fallback fail closed. | [ds4_metal.m:28089](ds4_metal.m#L28089) |
+| `DS4_METAL_REQUIRE_Q4_SSD_PREFILL_ATTN_OUT_SCALE_META` | value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables | Requires shared scale/min metadata in the Q4 SSD prefill attention-output exact-N kernel and makes fallback fail closed. | [ds4_metal.m:28209](ds4_metal.m#L28209) |
+| `DS4_METAL_REQUIRE_Q4_SSD_SESSION_UNION` | nonempty boolean; unset/empty or exact 0: off; every other value: on | Requires Q4 SSD session union and makes eligible fallback fail closed. | [ds4.c:65164](ds4.c#L65164) |
+| `DS4_METAL_REQUIRE_Q8_QKV_COMPRESSOR_FUSE` | nonempty boolean; unset/empty/exact 0: fallback allowed; other values require and imply the streamed/union enable | Requires eligible Q8 QKV/compressor compound fusion and fails closed. | [ds4.c:23023](ds4.c#L23023) |
+| `DS4_METAL_RESUME_PREFILL_MIN` | integer token threshold; default 4; <=0 disables resume-prefill | Sets the minimum shared-prefix suffix that uses batched resume-prefill. | [ds4.c:38419](ds4.c#L38419) |
+| `DS4_METAL_ROPE_EXP2_LOG2` | boolean empty/1/true/yes/on vs 0/false/no/off; default off | Compiles the exp2/log2 RoPE drift variant. | [ds4_metal.m:7050](ds4_metal.m#L7050) |
+| `DS4_METAL_SELECTED_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for selected. | [ds4.c:37943](ds4.c#L37943) |
+| `DS4_METAL_SELECTED_PROFILE_LAYER` | single nonnegative layer index; unset: every layer | Restricts selected-expert profiling to one layer. | [ds4_metal.m:42507](ds4_metal.m#L42507) |
+| `DS4_METAL_SESSION_BATCH_LOG` | presence diagnostic; unset: off; any value including 0 enables | Logs session batch decisions. | [ds4.c:66168](ds4.c#L66168) |
+| `DS4_METAL_SESSION_BATCH_QKV` | default enabled; exact 0 disables; every other value/unset leaves enabled | Controls native batched QKV work for multi-session decode. | [ds4.c:65482](ds4.c#L65482) |
+| `DS4_METAL_SESSION_BATCH_SHARED` | default enabled; exact 0 disables; every other value/unset leaves enabled | Controls native batched shared-expert work for multi-session decode. | [ds4.c:65432](ds4.c#L65432) |
+| `DS4_METAL_SET_ROWS_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the set-rows Metal kernel source file loaded at runtime. | [ds4_metal.m:4953](ds4_metal.m#L4953) |
+| `DS4_METAL_SOFTMAX_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the softmax Metal kernel source file loaded at runtime. | [ds4_metal.m:4948](ds4_metal.m#L4948) |
+| `DS4_METAL_STREAMING_DECODE_PREFILL_MAX` | integer token maximum; default 64 for wide Flash Q4/MXFP4, 18 for other PRO/Flash, 0 otherwise; <=0 disables | Sets maximum SSD-streaming micro-prefill width that reuses decode. | [ds4.c:31962](ds4.c#L31962) |
+| `DS4_METAL_STREAMING_EXPERT_AUTO_PRELOAD_CAP` | uint32 expert cap; default 4096; 0 means unlimited; invalid restores default | Caps automatic streaming-expert hotlist preload. | [ds4.c:21455](ds4.c#L21455) |
+| `DS4_METAL_STREAMING_EXPERT_BUFFER_MLOCK_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming expert buffer mlock. | [ds4_metal.m:13997](ds4_metal.m#L13997) |
+| `DS4_METAL_STREAMING_EXPERT_EARLY_LOAD_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming expert early load. | [ds4_metal.m:17059](ds4_metal.m#L17059) |
+| `DS4_METAL_STREAMING_EXPERT_EVICT_DONTNEED_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming expert evict dontneed. | [ds4_metal.m:14436](ds4_metal.m#L14436) |
+| `DS4_METAL_STREAMING_EXPERT_HOTLIST` | hotlist file path; unset/empty: built-in model hotlist | Loads the streaming-expert preload order from a file. | [ds4.c:21495](ds4.c#L21495) |
+| `DS4_METAL_STREAMING_EXPERT_HOTLIST_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming expert hotlist. | [ds4.c:32245](ds4.c#L32245) |
+| `DS4_METAL_STREAMING_EXPERT_LAYER_STATS` | presence diagnostic; unset: off; any value including 0 enables | Collects/prints statistics for streaming expert layer. | [ds4_metal.m:4638](ds4_metal.m#L4638) |
+| `DS4_METAL_STREAMING_EXPERT_LAYER_STATS_DELTA` | presence control; unset: off/default; any value including 0 enables | Prints delta statistics for streaming expert layer. | [ds4_metal.m:4675](ds4_metal.m#L4675) |
+| `DS4_METAL_STREAMING_EXPERT_NOCACHE` | nonempty value whose first character is not 0 enables; unset/empty/0 disables | Uses a reopened F_NOCACHE descriptor for SSD expert preads. | [ds4_metal.m:12602](ds4_metal.m#L12602) |
+| `DS4_METAL_STREAMING_EXPERT_PREAD_POOL` | default enabled; exact 0 disables; every other value/unset keeps enabled | Controls reuse of persistent expert-pread worker threads. | [ds4_metal.m:13433](ds4_metal.m#L13433) |
+| `DS4_METAL_STREAMING_EXPERT_PREAD_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming expert pread. | [ds4_metal.m:17005](ds4_metal.m#L17005) |
+| `DS4_METAL_STREAMING_EXPERT_PREAD_SPLIT` | integer clamped 1..8; unset: automatic 1 below 64 cache experts, 4 at 64+ | Sets aligned requests per expert pread. | [ds4_metal.m:13701](ds4_metal.m#L13701) |
+| `DS4_METAL_STREAMING_EXPERT_PREAD_THREADS` | unsigned integer clamped 1..18; default 9; invalid restores 9 | Sets expert-pread worker limit. | [ds4_metal.m:13337](ds4_metal.m#L13337) |
+| `DS4_METAL_STREAMING_EXPERT_PROFILE_SUMMARY` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming expert. | [ds4_metal.m:13061](ds4_metal.m#L13061) |
+| `DS4_METAL_STREAMING_EXPERT_SLAB_MB` | positive unsigned MiB; default 4096; 0/invalid restores default | Sets target allocation size for streaming-expert slabs. | [ds4_metal.m:14039](ds4_metal.m#L14039) |
+| `DS4_METAL_STREAMING_EXPERT_SPLIT_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming expert split. | [ds4_metal.m:43810](ds4_metal.m#L43810) |
+| `DS4_METAL_STREAMING_EXPERT_TIMING_SUMMARY` | presence control; unset: off/default; any value including 0 enables | Prints timing/profile diagnostics for streaming expert. | [ds4_metal.m:13060](ds4_metal.m#L13060) |
+| `DS4_METAL_STREAMING_IQ2_CPU_ROUTER_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming IQ2 CPU router. | [ds4.c:21645](ds4.c#L21645) |
+| `DS4_METAL_STREAMING_MAP_TRACE` | nonempty value other than exact 0 enables; unset/empty/0 disables | Emits SSD model-map decisions. | [ds4_metal.m:4849](ds4_metal.m#L4849) |
+| `DS4_METAL_STREAMING_PREFILL_BATCH_SELECTED_ADDR_MAX` | integer token maximum; default 800 for 384 experts, 760 for 256, 0 otherwise; <=0 disables automatic selection | Sets automatic maximum batch width for selected-address SSD prefill. | [ds4_metal.m:14639](ds4_metal.m#L14639) |
+| `DS4_METAL_STREAMING_PREFILL_BATCH_SELECTED_ADDR_MIN` | integer token minimum; default 2 for 256/384 experts, 0 otherwise; <=0 disables automatic selection | Sets automatic minimum batch width for selected-address SSD prefill. | [ds4_metal.m:14656](ds4_metal.m#L14656) |
+| `DS4_METAL_STREAMING_PREFILL_BATCH_SELECTED_ADDR_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming prefill batch selected address. | [ds4_metal.m:18493](ds4_metal.m#L18493) |
+| `DS4_METAL_STREAMING_PREFILL_CACHE_SEED_K` | uint32 seed rows; default 1; 0 disables; above 64 clamps to 64 | Sets how many prefill routing rows seed the decode expert cache. | [ds4.c:21268](ds4.c#L21268) |
+| `DS4_METAL_STREAMING_PREFILL_CACHE_SEED_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming prefill cache seed. | [ds4_metal.m:17899](ds4_metal.m#L17899) |
+| `DS4_METAL_STREAMING_PREFILL_LAYER_MADVISE_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming prefill layer madvise. | [ds4.c:19610](ds4.c#L19610) |
+| `DS4_METAL_STREAMING_PREFILL_LAYER_PAGEIN_NO_OVERLAP` | presence rollback; unset: automatic/default path; any value including 0 disables | Prevents full-layer page-in preparation from overlapping compute. | [ds4.c:19317](ds4.c#L19317) |
+| `DS4_METAL_STREAMING_PREFILL_LAYER_PAGEIN_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming prefill layer pagein. | [ds4.c:19606](ds4.c#L19606) |
+| `DS4_METAL_STREAMING_PREFILL_LAYER_PAGEIN_THREADS` | integer 1..16; default 8; invalid/0 becomes 1; PREPARE_THREADS takes precedence | Sets worker count for full-layer page-in preparation. | [ds4.c:19275](ds4.c#L19275) |
+| `DS4_METAL_STREAMING_PREFILL_LAYER_PREAD_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming prefill layer pread. | [ds4.c:19608](ds4.c#L19608) |
+| `DS4_METAL_STREAMING_PREFILL_LAYER_PREPARE_AHEAD` | integer 1..4 layers; default 1; invalid/0 becomes 1 | Sets number of future layers prepared concurrently. | [ds4.c:19329](ds4.c#L19329) |
+| `DS4_METAL_STREAMING_PREFILL_LAYER_PREPARE_NO_OVERLAP` | presence rollback; unset: automatic/default path; any value including 0 disables | Prevents generic full-layer preparation from overlapping compute. | [ds4.c:19315](ds4.c#L19315) |
+| `DS4_METAL_STREAMING_PREFILL_LAYER_PREPARE_THREADS` | integer 1..16; default 8; invalid/0 becomes 1; preferred over PAGEIN_THREADS | Sets worker count for full-layer preparation. | [ds4.c:19271](ds4.c#L19271) |
+| `DS4_METAL_STREAMING_PREFILL_LAYER_READAHEAD_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming prefill layer readahead. | [ds4.c:19612](ds4.c#L19612) |
+| `DS4_METAL_STREAMING_PREFILL_SELECTED_MADVISE_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming prefill selected madvise. | [ds4.c:19356](ds4.c#L19356) |
+| `DS4_METAL_STREAMING_PREFILL_SELECTED_MADVISE_THREADS` | integer 1..16; default inherits layer prepare threads; invalid/0 becomes 1; PREPARE_THREADS preferred | Sets worker count for selected-expert madvise preparation. | [ds4.c:19293](ds4.c#L19293) |
+| `DS4_METAL_STREAMING_PREFILL_SELECTED_PAGEIN_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming prefill selected pagein. | [ds4.c:19354](ds4.c#L19354) |
+| `DS4_METAL_STREAMING_PREFILL_SELECTED_PREPARE_GAP` | integer 0..8 layers; default 0; above 8 clamps; invalid restores 0 | Sets lookahead gap for selected-expert preparation. | [ds4.c:19305](ds4.c#L19305) |
+| `DS4_METAL_STREAMING_PREFILL_SELECTED_PREPARE_THREADS` | integer 1..16 for madvise preparation; default inherits layer prepare threads; invalid/0 becomes 1 | Sets worker count for selected-expert preparation. | [ds4.c:19289](ds4.c#L19289) |
+| `DS4_METAL_STREAMING_PREFILL_SELECTED_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming prefill selected. | [ds4.c:18906](ds4.c#L18906) |
+| `DS4_METAL_STREAMING_PREFILL_SELECTED_READAHEAD_GAP` | integer 0..8 layers; default 0; above 8 clamps; invalid restores 0 | Sets lookahead gap for selected-expert readahead. | [ds4.c:19978](ds4.c#L19978) |
+| `DS4_METAL_STREAMING_PREFILL_SELECTED_READAHEAD_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming prefill selected readahead. | [ds4.c:20063](ds4.c#L20063) |
+| `DS4_METAL_STREAMING_SELECTED_READAHEAD_PROFILE` | presence diagnostic; unset: off; any value including 0 enables | Prints timing/profile diagnostics for streaming selected readahead. | [ds4.c:21737](ds4.c#L21737) |
+| `DS4_METAL_SUM_ROWS_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the sum-rows Metal kernel source file loaded at runtime. | [ds4_metal.m:4947](ds4_metal.m#L4947) |
+| `DS4_METAL_TEST_POISON_COMPRESSOR_EXACT_REDUCTION_SCRATCH` | internal test presence flag; unset: off; any value including 0 poisons scratch before the exact reduction | Validates that compressor exact-reduction kernels overwrite all scratch state. | [ds4_metal.m:25925](ds4_metal.m#L25925) |
+| `DS4_METAL_TP_SESSION_BATCH` | default enabled; exact 0 disables; every other value/unset leaves enabled | Controls batched session evaluation with Metal TP. | [ds4.c:65310](ds4.c#L65310) |
+| `DS4_METAL_TRACE_ALLOCS` | presence diagnostic; unset: off; any value including 0 enables | Emits trace diagnostics for allocs. | [ds4_metal.m:4057](ds4_metal.m#L4057) |
+| `DS4_METAL_TRACE_M5_FLASH_ATTN_PACKED32_REDUCE` | presence diagnostic; unset: off; any value including 0 enables | Emits trace diagnostics for M5 flash attn packed32 reduce. | [ds4_metal.m:31535](ds4_metal.m#L31535) |
+| `DS4_METAL_UNARY_SOURCE` | file path; unset/empty: use the in-tree Metal source file | Overrides the unary operations Metal kernel source file loaded at runtime. | [ds4_metal.m:4939](ds4_metal.m#L4939) |
+| `DS4_METAL_UNRETAINED_COMMAND_BUFFERS` | presence control; unset: off/default; any value including 0 enables | Creates Metal command buffers with unretained references. | [ds4_metal.m:1315](ds4_metal.m#L1315) |
+| `DS4_METAL_USE_QUEUE_RESIDENCY_SET` | presence control; unset: off/default; any value including 0 enables | Allows queue-residency state to trigger Q4 expert address/table paths. | [ds4_metal.m:42280](ds4_metal.m#L42280) |
+
+</details>
+
+<details>
+<summary><strong>CUDA (336)</strong></summary>
+
+| Variable | Accepted value and default | Effect | Source |
+| --- | --- | --- | --- |
+| `DS4_CUDA_ATTENTION_OUTPUT_A_CUBLAS_MIN` | integer tokens; default 2; accepted range 2..4095, otherwise 2 | Set the token-count threshold for using cuBLAS on attention output-A. | [ds4_cuda.cu:23925](ds4_cuda.cu#L23925) |
+| `DS4_CUDA_ATTENTION_OUTPUT_PRELOAD` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Allow attention-output Q8 weights to be preloaded into the selective F16 cache. | [ds4_cuda.cu:2402](ds4_cuda.cu#L2402) |
+| `DS4_CUDA_ATTN_OUTPUT_PROFILE` | presence diagnostic flag; default off; any defined value including 0 enables | Measure and print CUDA attention-output stage timings. | [ds4_cuda.cu:23910](ds4_cuda.cu#L23910) |
+| `DS4_CUDA_ATTN_Q_B_F32_CACHE` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Enable an F32-derived-weight cache for attention Q-B weights. | [ds4_cuda.cu:2414](ds4_cuda.cu#L2414) |
+| `DS4_CUDA_BUILD_ARTIFACTS` | boolean-ish, default on for eligible derived artifacts; only exact 0 disables | Control construction of eligible CUDA derived/repacked weight artifacts. | [ds4_cuda.cu:8440](ds4_cuda.cu#L8440) |
+| `DS4_CUDA_COPY_MODEL` | nonempty-string opt-in (but mere presence also suppresses prefetch); default off; value 0 is nonempty and requests a full copy | Copy the complete mapped model image into device memory. | [ds4_cuda.cu:2740](ds4_cuda.cu#L2740) |
+| `DS4_CUDA_COPY_MODEL_CHUNKED` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Use range-by-range model prefetch/copy preparation instead of the normal bulk preparation. | [ds4_cuda.cu:37740](ds4_cuda.cu#L37740) |
+| `DS4_CUDA_DECODE_GRAPHS` | boolean, default on; any value starting with 0, or exact off/no/false in listed case variants, disables; oracle flags force off; effective only on one GPU | Control CUDA Graph capture and replay for decode. | [ds4_cuda.cu:1468](ds4_cuda.cu#L1468) |
+| `DS4_CUDA_DECODE_GRAPH_LOG` | presence diagnostic flag; default off; any defined value including 0 enables | Log CUDA decode-graph cache misses, capture failures, and lifecycle events. | [ds4_cuda.cu:1580](ds4_cuda.cu#L1580) |
+| `DS4_CUDA_DECODE_HEADS8_ONLINE` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Force the eight-head online CUDA decode-attention kernel when eligible. | [ds4_cuda.cu:371](ds4_cuda.cu#L371) |
+| `DS4_CUDA_DECODE_SCORE4` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Select four score lanes in the CUDA decode-attention fallback kernel. | [ds4_cuda.cu:372](ds4_cuda.cu#L372) |
+| `DS4_CUDA_DECODE_SCORE8` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Select eight score lanes in the CUDA decode-attention fallback kernel. | [ds4_cuda.cu:373](ds4_cuda.cu#L373) |
+| `DS4_CUDA_DIRECT_MODEL` | mixed presence/nonempty flag, default off; any defined value bypasses host caching, while backend direct lookup requires nonempty; value 0 therefore still changes behavior | Use the mapped model directly and bypass selective CUDA weight caching. | [ds4.c:3058](ds4.c#L3058); [ds4_cuda.cu:1250](ds4_cuda.cu#L1250) |
+| `DS4_CUDA_DISABLE_DSPARK_EXACTN` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Disable the CUDA DSpark exactn optimization. | [ds4.c:52270](ds4.c#L52270) |
+| `DS4_CUDA_DISABLE_DSPARK_EXACTN_BATCH_HEAD` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Disable the CUDA DSpark exactn batch head optimization. | [ds4.c:37285](ds4.c#L37285) |
+| `DS4_CUDA_DISABLE_DSPARK_EXACTN_GRAPHS` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Disable the CUDA DSpark exactn graphs optimization. | [ds4.c:37251](ds4.c#L37251) |
+| `DS4_CUDA_DISABLE_DSPARK_NONCAUSAL_ONLINE` | value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on | Disable the noncausal online-attention DSpark experiment. | [ds4_cuda.cu:21691](ds4_cuda.cu#L21691) |
+| `DS4_CUDA_DISABLE_HC_NORM_MIX_FUSE` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable fused HC RMSNorm-plus-mix. | [ds4_cuda.cu:20905](ds4_cuda.cu#L20905) |
+| `DS4_CUDA_DISABLE_HC_SPLIT_NORM_FUSED` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the fused HC split/weighted-sum/norm kernel. | [ds4_cuda.cu:31785](ds4_cuda.cu#L31785) |
+| `DS4_CUDA_DISABLE_IQ2_XXS_SSD_PREFILL_MMQ` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Disable the CUDA IQ2 XXS SSD prefill MMQ optimization/path. | [ds4_cuda.cu:4627](ds4_cuda.cu#L4627) |
+| `DS4_CUDA_DISABLE_Q4_ATTN_OUT_HC_FUSE` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable fused Q4 attention-output/HC expansion. | [ds4_cuda.cu:37357](ds4_cuda.cu#L37357) |
+| `DS4_CUDA_DISABLE_Q4_DENSE_PAIR` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q4 dense pair CUDA Q4 optimization. | [ds4_cuda.cu:20402](ds4_cuda.cu#L20402) |
+| `DS4_CUDA_DISABLE_Q8_HC_EXPAND_FUSED` | false-like-aware flag, default off; 0/false/no/off is off, other nonempty values request split; force-fused wins | Request the split Q8 shared-down/HC path when safe. | [ds4_cuda.cu:2086](ds4_cuda.cu#L2086) |
+| `DS4_CUDA_DISABLE_QKV_RMS_FUSED` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA QKV RMS fused optimization/path. | [ds4_cuda.cu:369](ds4_cuda.cu#L369); [ds4.c:17428](ds4.c#L17428) |
+| `DS4_CUDA_DISABLE_SHARED_GATE_UP_PAIR` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA shared gate up pair optimization/path. | [ds4_cuda.cu:24293](ds4_cuda.cu#L24293) |
+| `DS4_CUDA_DISABLE_STREAMING_EXPERT_PERSISTENT_CACHE` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Disable streaming expert persistent cache in CUDA SSD streaming. | [ds4_cuda.cu:4113](ds4_cuda.cu#L4113) |
+| `DS4_CUDA_DISABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable streaming prefill batch selected addr in CUDA SSD streaming. | [ds4.c:18592](ds4.c#L18592) |
+| `DS4_CUDA_DISABLE_STREAMING_PREFILL_BATCH_SELECTED_LOAD` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable streaming prefill batch selected load in CUDA SSD streaming. | [ds4.c:21917](ds4.c#L21917) |
+| `DS4_CUDA_DISABLE_STREAMING_SELECTED_BATCH_IO` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Disable streaming selected batch I/O in CUDA SSD streaming. | [ds4_cuda.cu:4734](ds4_cuda.cu#L4734) |
+| `DS4_CUDA_DISABLE_STREAMING_SELECTED_EVENT_PIPELINE` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Disable streaming selected event pipeline in CUDA SSD streaming. | [ds4_cuda.cu:4866](ds4_cuda.cu#L4866) |
+| `DS4_CUDA_DISABLE_STREAMING_SELECTED_SHARED_OVERLAP` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable streaming selected shared overlap in CUDA SSD streaming. | [ds4.c:20969](ds4.c#L20969) |
+| `DS4_CUDA_DSPARK_DEVICE_PROPOSER` | value-aware opt-in, default off; 0/off/no/false (lowercase only) disable; other nonempty enables unless rollback set | Enable the CUDA-resident DSpark proposer. | [ds4.c:34889](ds4.c#L34889); [ds4_cuda.cu:19035](ds4_cuda.cu#L19035) |
+| `DS4_CUDA_DSPARK_EXACT2` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Enable the exact two-draft CUDA DSpark support path. | [ds4.c:52247](ds4.c#L52247) |
+| `DS4_CUDA_DSPARK_EXACTN` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Enable the exact multi-draft CUDA DSpark support path. | [ds4.c:52268](ds4.c#L52268) |
+| `DS4_CUDA_DSPARK_EXACTN_BATCH_HEAD` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Enable the batched output-head stage for exact-N DSpark verification. | [ds4.c:37283](ds4.c#L37283) |
+| `DS4_CUDA_DSPARK_EXACTN_GRAPHS` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Enable CUDA Graph capture for stable exact-N DSpark islands. | [ds4.c:37249](ds4.c#L37249) |
+| `DS4_CUDA_DSPARK_NO_DEVICE_PROPOSER` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the CUDA-resident DSpark proposer. | [ds4.c:34899](ds4.c#L34899); [ds4_cuda.cu:19040](ds4_cuda.cu#L19040) |
+| `DS4_CUDA_DSPARK_NO_PADDED_HEAD` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the padded CUDA output-head optimization used by DSpark. | [ds4.c:34247](ds4.c#L34247) |
+| `DS4_CUDA_DSPARK_NO_Q_NORM_ROPE_FUSION` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable fused Q RMSNorm plus RoPE in DSpark support stages. | [ds4.c:33623](ds4.c#L33623) |
+| `DS4_CUDA_DSPARK_PROPOSER_BLOCK_MAX` | integer 0..UINT32_MAX; 0/invalid keeps native size; unset uses auto caps for exact-N/exact2; positive values cap the block and are limited by DS4_DSPARK_MAX_BLOCK_SIZE | Cap the CUDA DSpark proposal block length. | [ds4.c:52354](ds4.c#L52354) |
+| `DS4_CUDA_DSPARK_TINY_ALIGNED_VEC` | value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on | Use aligned routed-MoE vector kernels for tiny DSpark batches. | [ds4_cuda.cu:29523](ds4_cuda.cu#L29523) |
+| `DS4_CUDA_ENABLE_DSPARK_NONCAUSAL_ONLINE` | value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on | Enable the small-batch noncausal online-attention DSpark experiment. | [ds4_cuda.cu:21690](ds4_cuda.cu#L21690) |
+| `DS4_CUDA_ENABLE_HC_NORM_MIX_FUSE` | nonempty opt-in, default off; only exact 0 disables; the F32/F16 activation mode follows the selected standalone matmul path; disable/serial/alternate flags can veto | Enable and select the fused HC RMSNorm-plus-mix one-token implementation. | [ds4_cuda.cu:20902](ds4_cuda.cu#L20902) |
+| `DS4_CUDA_ENABLE_IQ2_XXS_SSD_PREFILL_MMQ` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Enable the CUDA IQ2 XXS SSD prefill MMQ experimental path. | [ds4_cuda.cu:4625](ds4_cuda.cu#L4625) |
+| `DS4_CUDA_ENABLE_Q4_ATTN_OUT_HC_FUSE` | value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on | Opt in to the fused Q4 attention-output/HC expansion path. | [ds4_cuda.cu:37375](ds4_cuda.cu#L37375) |
+| `DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_BATCH` | value-aware opt-in, default off; nonempty value other than exact 0 enables; rollback wins | Enable flattened grouped attention-A MMQ for two-to-eight-token GB10 batches. | [cuda/mmq/ds4_mmq.cu:4303](cuda/mmq/ds4_mmq.cu#L4303) |
+| `DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_PREFILL` | value-aware compatibility switch, default on; unset/empty uses the default, exact 0 opts out, any other nonempty value requests the path; REQUIRE also requests it; local/global rollback wins | Control direct-strided grouped Q4_K attention-A MMQ for GB10 prefill widths above eight tokens, removing per-group pack/unpack copies while preserving the per-group reduction tree. | [ds4_cuda.cu:41928](ds4_cuda.cu#L41928) |
+| `DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT` | presence flag, default off; any defined value including 0 requests the path; rollback wins | Enable the GB10 persistent-CTA kernel for M=32768, N=1, K=1024 Q4. | [cuda/mmq/ds4_mmq.cu:3905](cuda/mmq/ds4_mmq.cu#L3905) |
+| `DS4_CUDA_ENABLE_Q8_FOLD` | strict flag, default off; only exact value 1 enables; overridden by DS4_CUDA_NO_Q8_FOLD | Enable one-shot producer-to-consumer reuse of freshly quantized Q8_1 data. | [ds4_cuda.cu:785](ds4_cuda.cu#L785) |
+| `DS4_CUDA_ENABLE_STREAMING_EXPERT_PERSISTENT_CACHE` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Enable streaming expert persistent cache in CUDA SSD streaming. | [ds4_cuda.cu:4111](ds4_cuda.cu#L4111) |
+| `DS4_CUDA_ENABLE_STREAMING_SELECTED_BATCH_IO` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Enable streaming selected batch I/O in CUDA SSD streaming. | [ds4_cuda.cu:4732](ds4_cuda.cu#L4732) |
+| `DS4_CUDA_ENABLE_STREAMING_SELECTED_EVENT_PIPELINE` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Enable streaming selected event pipeline in CUDA SSD streaming. | [ds4_cuda.cu:4864](ds4_cuda.cu#L4864) |
+| `DS4_CUDA_END_STREAM_SYNC` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Synchronize only CUDA stream 0 at command-batch end instead of synchronizing the whole device. | [ds4_cuda.cu:376](ds4_cuda.cu#L376) |
+| `DS4_CUDA_EXACT_SCORE_SPLIT_CHUNK` | integer scores/chunk; default 512; clamped 1..8192 | Tune exact score split chunk for exact score-split CUDA decode attention. | [ds4_cuda.cu:13669](ds4_cuda.cu#L13669) |
+| `DS4_CUDA_EXACT_SCORE_SPLIT_DECODE` | value-aware boolean, default on; exact 0 disables; a nonzero explicit setting also takes precedence over split-KV selection | Control the exact score-split decode-attention implementation. | [ds4_cuda.cu:13631](ds4_cuda.cu#L13631) |
+| `DS4_CUDA_EXACT_SCORE_SPLIT_DIM2` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Tune exact score split dim2 for exact score-split CUDA decode attention. | [ds4_cuda.cu:387](ds4_cuda.cu#L387) |
+| `DS4_CUDA_EXACT_SCORE_SPLIT_FUSE_INV_ROPE` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Tune exact score split fuse inv rope for exact score-split CUDA decode attention. | [ds4_cuda.cu:390](ds4_cuda.cu#L390) |
+| `DS4_CUDA_EXACT_SCORE_SPLIT_GRAPH` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Tune exact score split graph for exact score-split CUDA decode attention. | [ds4_cuda.cu:379](ds4_cuda.cu#L379) |
+| `DS4_CUDA_EXACT_SCORE_SPLIT_LDG` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Tune exact score split ldg for exact score-split CUDA decode attention. | [ds4_cuda.cu:381](ds4_cuda.cu#L381) |
+| `DS4_CUDA_EXACT_SCORE_SPLIT_MIN_SCORE` | integer score count; default 1; clamped 0..8192 | Set the minimum visible-score count for exact score-split decode. | [ds4_cuda.cu:13665](ds4_cuda.cu#L13665) |
+| `DS4_CUDA_EXACT_SCORE_SPLIT_S` | integer exact split count; unset/invalid = automatic; valid value clamped 1..16 | Tune exact score split s for exact score-split CUDA decode attention. | [ds4_cuda.cu:13679](ds4_cuda.cu#L13679) |
+| `DS4_CUDA_EXACT_SCORE_SPLIT_S_FLOOR` | integer split count; default 6; clamped 1..16 | Tune exact score split s floor for exact score-split CUDA decode attention. | [ds4_cuda.cu:13672](ds4_cuda.cu#L13672) |
+| `DS4_CUDA_EXACT_SCORE_SPLIT_S_MAX` | integer split count; default 16; clamped 1..16 | Tune exact score split s max for exact score-split CUDA decode attention. | [ds4_cuda.cu:13675](ds4_cuda.cu#L13675) |
+| `DS4_CUDA_EXACT_SCORE_SPLIT_VEC4` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Tune exact score split vec4 for exact score-split CUDA decode attention. | [ds4_cuda.cu:383](ds4_cuda.cu#L383) |
+| `DS4_CUDA_EXACT_SCORE_SPLIT_VEC4_PLAIN` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Tune exact score split vec4 plain for exact score-split CUDA decode attention. | [ds4_cuda.cu:385](ds4_cuda.cu#L385) |
+| `DS4_CUDA_F16_CUBLAS_ONE` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Control the F16 cuBLAS one CUDA F16 matmul path. | [ds4_cuda.cu:20852](ds4_cuda.cu#L20852) |
+| `DS4_CUDA_F16_SMALL_BATCH` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Control the F16 small batch CUDA F16 matmul path. | [ds4_cuda.cu:20837](ds4_cuda.cu#L20837) |
+| `DS4_CUDA_F16_SMALL_OUT` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Control the F16 small out CUDA F16 matmul path. | [ds4_cuda.cu:20819](ds4_cuda.cu#L20819) |
+| `DS4_CUDA_GLM_VERIFY_NO_Q8_TOK2` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Control or tune the CUDA glm verify no Q8 tok2 path. | [ds4_cuda.cu:19824](ds4_cuda.cu#L19824) |
+| `DS4_CUDA_GREEDY_SPLITKV` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Enable greedy split-KV fast attention. | [ds4.c:17235](ds4.c#L17235) |
+| `DS4_CUDA_GREEDY_SPLITKV_FALLBACK_LOG` | presence diagnostic flag; default off; any defined value including 0 enables | Control greedy splitkv fallback log in CUDA greedy fast decode. | [ds4.c:55663](ds4.c#L55663) |
+| `DS4_CUDA_GREEDY_SPLITKV_MARGIN` | nonnegative finite float; default 0.25; invalid value warns and uses 0.25; 0 disables margin fallback | Control greedy splitkv margin in CUDA greedy fast decode. | [ds4.c:17313](ds4.c#L17313) |
+| `DS4_CUDA_GREEDY_SPLITKV_MAX_SEGMENT` | integer 0..INT32_MAX; default/invalid 0 (segment cap disabled) | Control greedy splitkv max segment in CUDA greedy fast decode. | [ds4.c:17389](ds4.c#L17389) |
+| `DS4_CUDA_GREEDY_SPLITKV_PAIR_REPLAY` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Replay greedy split-KV tokens in pairs. | [ds4.c:17363](ds4.c#L17363) |
+| `DS4_CUDA_GREEDY_SPLITKV_TOP2` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Use top-2 output margins with greedy split-KV. | [ds4.c:17345](ds4.c#L17345) |
+| `DS4_CUDA_GREEDY_SPLITKV_TRACE` | presence diagnostic flag; default off; any defined value including 0 enables | Control greedy splitkv trace in CUDA greedy fast decode. | [ds4.c:55698](ds4.c#L55698) |
+| `DS4_CUDA_GREEDY_SPLITKV_TRUST_REPLAY` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Trust replayed greedy split-KV results without the normal confirmation policy. | [ds4.c:17353](ds4.c#L17353) |
+| `DS4_CUDA_GREEDY_SPLIT_TOP1` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Enable split top-1 selection in greedy CUDA decode. | [ds4.c:17207](ds4.c#L17207) |
+| `DS4_CUDA_GREEDY_TOP1` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control greedy top1 in CUDA greedy fast decode. | [ds4.c:56126](ds4.c#L56126) |
+| `DS4_CUDA_GREEDY_VEC4` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Enable greedy vec4 fast attention. | [ds4.c:17245](ds4.c#L17245) |
+| `DS4_CUDA_GREEDY_VEC4_FALLBACK_LOG` | presence diagnostic flag; default off; any defined value including 0 enables | Control greedy vec4 fallback log in CUDA greedy fast decode. | [ds4.c:55664](ds4.c#L55664) |
+| `DS4_CUDA_GREEDY_VEC4_MARGIN` | nonnegative finite float; default 0.25; invalid value warns and uses 0.25; 0 disables margin fallback | Control greedy vec4 margin in CUDA greedy fast decode. | [ds4.c:17283](ds4.c#L17283) |
+| `DS4_CUDA_GREEDY_VEC4_MAX_SEGMENT` | integer 0..INT32_MAX; default/invalid 0 (segment cap disabled) | Control greedy vec4 max segment in CUDA greedy fast decode. | [ds4.c:17397](ds4.c#L17397) |
+| `DS4_CUDA_GREEDY_VEC4_TRACE` | presence diagnostic flag; default off; any defined value including 0 enables | Control greedy vec4 trace in CUDA greedy fast decode. | [ds4.c:55727](ds4.c#L55727) |
+| `DS4_CUDA_INDEXED_TWOPASS` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Force the two-pass indexed-attention path instead of the fused heads8 online kernel. | [ds4_cuda.cu:23587](ds4_cuda.cu#L23587) |
+| `DS4_CUDA_IQ2_XXS_SSD_PREFILL_MMQ_STATS` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Print CUDA IQ2 XXS SSD prefill MMQ counters. | [ds4_cuda.cu:4633](ds4_cuda.cu#L4633) |
+| `DS4_CUDA_KEEP_MODEL_PAGES` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Keep source model pages resident instead of advising the OS to discard copied pages. | [ds4_cuda.cu:2840](ds4_cuda.cu#L2840) |
+| `DS4_CUDA_MIXED_PREFILL_DECODE` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control native mixed prefill/decode scheduling. | [ds4.c:70701](ds4.c#L70701) |
+| `DS4_CUDA_MIXED_ROUTED_MAX_PREFILL` | integer rows 0..UINT32_MAX; default/invalid 512 | Set the maximum prefill rows admitted to the mixed routed-MoE path. | [ds4.c:70190](ds4.c#L70190) |
+| `DS4_CUDA_MIXED_ROUTED_SCATTER` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Use scattered row handling in mixed routed-MoE execution. | [ds4.c:69651](ds4.c#L69651) |
+| `DS4_CUDA_MMQ` | boolean-ish, default on; any value beginning with 0 disables; quality mode and multi-GPU disable normal MMQ tier (MXFP4 path differs) | Control the vendored CUDA MMQ prefill tier. | [ds4_cuda.cu:1722](ds4_cuda.cu#L1722) |
+| `DS4_CUDA_MMQ_Q81_PERSISTENT` | strict boolean, default off; accepts 1/on/true/yes and 0/off/false/no in listed lower/upper-case forms; unknown values are off | Reuse a persistent Q8_1 MMQ scratch arena on supported GB10 devices. | [cuda/mmq/ds4_mmq.cu:152](cuda/mmq/ds4_mmq.cu#L152) |
+| `DS4_CUDA_MMQ_X_MAX` | integer >=8; rounded down to multiple of 8 and only lowers the hardware base; invalid/unset = hardware base | Cap the MMQ X tile-width selector for architecture tuning. | [cuda/mmq/mmq.cuh:127](cuda/mmq/mmq.cuh#L127) |
+| `DS4_CUDA_MODEL_COPY_CHUNK_MB` | positive integer MiB; default 64; clamped 16..4096 | Set the chunk size used for CUDA model copying. | [ds4_cuda.cu:2827](ds4_cuda.cu#L2827) |
+| `DS4_CUDA_MODEL_COPY_VERBOSE` | presence diagnostic flag; default off; any defined value including 0 enables | Print periodic progress while copying the model to device memory. | [ds4_cuda.cu:6537](ds4_cuda.cu#L6537) |
+| `DS4_CUDA_MODEL_PREFETCH_SYNC` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Synchronize after each CUDA model prefetch range for diagnostics. | [ds4_cuda.cu:2808](ds4_cuda.cu#L2808) |
+| `DS4_CUDA_MOE_ATOMIC_DOWN` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Select or tune the atomic down variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30086](ds4_cuda.cu#L30086) |
+| `DS4_CUDA_MOE_DECODE_GRAPH` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Select or tune the decode graph variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:391](ds4_cuda.cu#L391) |
+| `DS4_CUDA_MOE_DIRECT_MIDQ` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Select or tune the direct midq variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30140](ds4_cuda.cu#L30140) |
+| `DS4_CUDA_MOE_DOWN_ROW1024` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Select or tune the down row1024 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30109](ds4_cuda.cu#L30109) |
+| `DS4_CUDA_MOE_DOWN_ROW128` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Select or tune the down row128 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30128](ds4_cuda.cu#L30128) |
+| `DS4_CUDA_MOE_DOWN_ROW2048` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Select or tune the down row2048 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30110](ds4_cuda.cu#L30110) |
+| `DS4_CUDA_MOE_DOWN_ROW256` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Select or tune the down row256 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30127](ds4_cuda.cu#L30127) |
+| `DS4_CUDA_MOE_DOWN_ROW512` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Select or tune the down row512 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30108](ds4_cuda.cu#L30108) |
+| `DS4_CUDA_MOE_DOWN_ROW64` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Select or tune the down row64 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30129](ds4_cuda.cu#L30129) |
+| `DS4_CUDA_MOE_GATE_ROW1024` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Select or tune the gate row1024 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30120](ds4_cuda.cu#L30120) |
+| `DS4_CUDA_MOE_GATE_ROW128` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Select or tune the gate row128 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30093](ds4_cuda.cu#L30093) |
+| `DS4_CUDA_MOE_GATE_ROW2048` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Select or tune the gate row2048 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30091](ds4_cuda.cu#L30091) |
+| `DS4_CUDA_MOE_GATE_ROW256` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Select or tune the gate row256 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30092](ds4_cuda.cu#L30092) |
+| `DS4_CUDA_MOE_MIDQ_SIDECAR` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Select or tune the midq sidecar variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30169](ds4_cuda.cu#L30169) |
+| `DS4_CUDA_MOE_NO_ATOMIC_DOWN` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the atomic down variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30087](ds4_cuda.cu#L30087) |
+| `DS4_CUDA_MOE_NO_DECODE_LUT_GATE` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the decode lut gate variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30117](ds4_cuda.cu#L30117) |
+| `DS4_CUDA_MOE_NO_DIRECT_DOWN_SUM6` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the direct down sum6 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30137](ds4_cuda.cu#L30137) |
+| `DS4_CUDA_MOE_NO_DIRECT_MIDQ` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the direct midq variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30141](ds4_cuda.cu#L30141) |
+| `DS4_CUDA_MOE_NO_DOWN_ROW128` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the down row128 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30133](ds4_cuda.cu#L30133) |
+| `DS4_CUDA_MOE_NO_DOWN_ROW2048` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the down row2048 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30131](ds4_cuda.cu#L30131) |
+| `DS4_CUDA_MOE_NO_DOWN_ROW256` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the down row256 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30132](ds4_cuda.cu#L30132) |
+| `DS4_CUDA_MOE_NO_DOWN_ROW64` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the down row64 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30134](ds4_cuda.cu#L30134) |
+| `DS4_CUDA_MOE_NO_DOWN_TILE16` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the down tile16 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30102](ds4_cuda.cu#L30102) |
+| `DS4_CUDA_MOE_NO_EXPERT_TILES` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the expert tiles variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30062](ds4_cuda.cu#L30062) |
+| `DS4_CUDA_MOE_NO_GATE_ROW128` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the gate row128 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30097](ds4_cuda.cu#L30097) |
+| `DS4_CUDA_MOE_NO_GATE_ROW2048` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the gate row2048 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30095](ds4_cuda.cu#L30095) |
+| `DS4_CUDA_MOE_NO_GATE_ROW256` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the gate row256 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30096](ds4_cuda.cu#L30096) |
+| `DS4_CUDA_MOE_NO_IQ2_ALIGNED` | value-aware kill switch, default off; nonempty value other than exact 0 disables aligned IQ2 path | Disable the IQ2 aligned variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:1011](ds4_cuda.cu#L1011) |
+| `DS4_CUDA_MOE_NO_MIDQ_SIDECAR` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the midq sidecar variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30170](ds4_cuda.cu#L30170) |
+| `DS4_CUDA_MOE_NO_OWNED_SPARSE_BUFFERS` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the owned sparse buffers variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30089](ds4_cuda.cu#L30089) |
+| `DS4_CUDA_MOE_NO_P2` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the p2 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30084](ds4_cuda.cu#L30084) |
+| `DS4_CUDA_MOE_NO_Q2K_ALIGNED` | value-aware kill switch, default off; nonempty value other than exact 0 disables aligned Q2_K path | Disable the q2k aligned variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:1016](ds4_cuda.cu#L1016) |
+| `DS4_CUDA_MOE_NO_Q4_DOWN_ROWSPAN` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the Q4 down rowspan variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30114](ds4_cuda.cu#L30114) |
+| `DS4_CUDA_MOE_NO_Q4_DOWN_SLOT3` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the Q4 down slot3 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30164](ds4_cuda.cu#L30164) |
+| `DS4_CUDA_MOE_NO_Q4_GATE_H16` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the Q4 gate H16 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30149](ds4_cuda.cu#L30149) |
+| `DS4_CUDA_MOE_NO_Q4_GATE_H16R8` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the Q4 gate H16R8 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30145](ds4_cuda.cu#L30145) |
+| `DS4_CUDA_MOE_NO_Q4_GATE_W32` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the Q4 gate W32 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30157](ds4_cuda.cu#L30157) |
+| `DS4_CUDA_MOE_NO_Q4_GATE_W32R16` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the Q4 gate W32R16 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30153](ds4_cuda.cu#L30153) |
+| `DS4_CUDA_MOE_NO_Q4_GATE_W32_NOAUX` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the Q4 gate W32 no-aux variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30160](ds4_cuda.cu#L30160) |
+| `DS4_CUDA_MOE_NO_Q4_MMA` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the Q4 MMA variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:312](ds4_cuda.cu#L312) |
+| `DS4_CUDA_MOE_NO_Q4_MMA_TILE16` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the Q4 MMA tile16 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30100](ds4_cuda.cu#L30100) |
+| `DS4_CUDA_MOE_NO_Q4_SORTED` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the Q4 sorted variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30061](ds4_cuda.cu#L30061) |
+| `DS4_CUDA_MOE_NO_SMALL_SORTED_PREP` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Disable the small sorted prep variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30106](ds4_cuda.cu#L30106) |
+| `DS4_CUDA_MOE_PROFILE` | presence diagnostic flag; default off; any defined value including 0 enables | Measure and print routed-MoE CUDA kernel-stage timings. | [ds4_cuda.cu:30045](ds4_cuda.cu#L30045) |
+| `DS4_CUDA_MOE_Q4_DOWN_SLOT3` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Select or tune the Q4 down slot3 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30163](ds4_cuda.cu#L30163) |
+| `DS4_CUDA_MOE_Q4_GATE_H16` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Select or tune the Q4 gate H16 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30148](ds4_cuda.cu#L30148) |
+| `DS4_CUDA_MOE_Q4_GATE_H16R8` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Select or tune the Q4 gate H16R8 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30144](ds4_cuda.cu#L30144) |
+| `DS4_CUDA_MOE_Q4_GATE_W32R16` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Select or tune the Q4 gate W32R16 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30152](ds4_cuda.cu#L30152) |
+| `DS4_CUDA_MOE_TILE4` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Select or tune the tile4 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30063](ds4_cuda.cu#L30063) |
+| `DS4_CUDA_MOE_TILE8` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Select or tune the tile8 variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30079](ds4_cuda.cu#L30079) |
+| `DS4_CUDA_MOE_WRITE_GATE_UP` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Select or tune the write gate up variant in CUDA routed-MoE dispatch. | [ds4_cuda.cu:30081](ds4_cuda.cu#L30081) |
+| `DS4_CUDA_NO_ATTENTION_OUTPUT_F16_CACHE` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the attention output F16 cache CUDA F16 path. | [ds4_cuda.cu:2364](ds4_cuda.cu#L2364) |
+| `DS4_CUDA_NO_ATTN_A_TOK2` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA attn a tok2 optimization/path. | [ds4_cuda.cu:24024](ds4_cuda.cu#L24024) |
+| `DS4_CUDA_NO_ATTN_Q_B_F16_CACHE` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the attn q b F16 cache CUDA F16 path. | [ds4_cuda.cu:2367](ds4_cuda.cu#L2367) |
+| `DS4_CUDA_NO_COMPRESSOR_PREFILL_BATCH` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA compressor prefill batch optimization/path. | [ds4.c:29890](ds4.c#L29890) |
+| `DS4_CUDA_NO_CUBLAS_ATTENTION` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA cuBLAS attention optimization/path. | [ds4_cuda.cu:23067](ds4_cuda.cu#L23067) |
+| `DS4_CUDA_NO_CUBLAS_ATTENTION_OUTPUT_A` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA cuBLAS attention output a optimization/path. | [ds4_cuda.cu:23934](ds4_cuda.cu#L23934) |
+| `DS4_CUDA_NO_DECODE_VALUE512` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the 512-thread CUDA decode value/finalize specialization. | [ds4_cuda.cu:374](ds4_cuda.cu#L374) |
+| `DS4_CUDA_NO_DERIVED_WEIGHTS` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA derived weights optimization/path. | [ds4_cuda.cu:1034](ds4_cuda.cu#L1034) |
+| `DS4_CUDA_NO_DIRECT_IO` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA direct I/O optimization/path. | [cuda/mmq/ds4_repack.cu:68](cuda/mmq/ds4_repack.cu#L68) |
+| `DS4_CUDA_NO_DIRECT_Q2_PREFILL` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA direct q2 prefill optimization/path. | [ds4_cuda.cu:395](ds4_cuda.cu#L395) |
+| `DS4_CUDA_NO_EXACT_SCORE_SPLIT_DECODE` | value-aware kill switch, default off; exact 0 is off, other nonempty values disable | Disable exact score split decode for exact score-split CUDA decode attention. | [ds4_cuda.cu:13629](ds4_cuda.cu#L13629); [ds4.c:68110](ds4.c#L68110) |
+| `DS4_CUDA_NO_EXACT_SCORE_SPLIT_DIM2` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable exact score split dim2 for exact score-split CUDA decode attention. | [ds4_cuda.cu:388](ds4_cuda.cu#L388) |
+| `DS4_CUDA_NO_F16_CUBLAS_BATCH` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the F16 cuBLAS batch CUDA F16 path. | [ds4_cuda.cu:20854](ds4_cuda.cu#L20854) |
+| `DS4_CUDA_NO_F16_CUBLAS_ONE` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the F16 cuBLAS one CUDA F16 path. | [ds4_cuda.cu:20851](ds4_cuda.cu#L20851) |
+| `DS4_CUDA_NO_F16_PAIR_COMPRESSOR_STORE` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the F16 pair compressor store CUDA F16 path. | [ds4_cuda.cu:399](ds4_cuda.cu#L399) |
+| `DS4_CUDA_NO_F16_PAIR_COMPRESSOR_TRANSPOSE` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the F16 pair compressor transpose CUDA F16 path. | [ds4_cuda.cu:21343](ds4_cuda.cu#L21343) |
+| `DS4_CUDA_NO_F16_PAIR_COMPRESSOR_TRANSPOSE_PREFETCH8` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the F16 pair compressor transpose prefetch8 CUDA F16 path. | [ds4_cuda.cu:21349](ds4_cuda.cu#L21349) |
+| `DS4_CUDA_NO_F16_PAIR_MATMUL` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the F16 pair matmul CUDA F16 path. | [ds4_cuda.cu:21130](ds4_cuda.cu#L21130) |
+| `DS4_CUDA_NO_F16_SMALL_BATCH` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the F16 small batch CUDA F16 path. | [ds4_cuda.cu:20838](ds4_cuda.cu#L20838) |
+| `DS4_CUDA_NO_F16_SMALL_OUT` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the F16 small out CUDA F16 path. | [ds4_cuda.cu:20821](ds4_cuda.cu#L20821) |
+| `DS4_CUDA_NO_FD_CACHE` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA fd cache optimization/path. | [ds4_cuda.cu:1274](ds4_cuda.cu#L1274) |
+| `DS4_CUDA_NO_GREEDY_SPLITKV` | value-aware kill switch; default off; nonempty value other than exact 0 disables | Disable greedy splitkv in CUDA greedy fast decode. | [ds4.c:17233](ds4.c#L17233) |
+| `DS4_CUDA_NO_GREEDY_SPLITKV_FALLBACK` | value-aware kill switch; default off; nonempty value other than exact 0 disables margin fallback | Disable greedy splitkv fallback in CUDA greedy fast decode. | [ds4.c:17333](ds4.c#L17333) |
+| `DS4_CUDA_NO_GREEDY_SPLITKV_PAIR_REPLAY` | value-aware kill switch; default off; nonempty value other than exact 0 disables | Disable greedy splitkv pair replay in CUDA greedy fast decode. | [ds4.c:17361](ds4.c#L17361) |
+| `DS4_CUDA_NO_GREEDY_SPLITKV_TOP2` | value-aware kill switch; default off; nonempty value other than exact 0 disables | Disable greedy splitkv top2 in CUDA greedy fast decode. | [ds4.c:17343](ds4.c#L17343) |
+| `DS4_CUDA_NO_GREEDY_SPLIT_TOP1` | value-aware kill switch; default off; nonempty value other than exact 0 disables | Disable greedy split top1 in CUDA greedy fast decode. | [ds4.c:17205](ds4.c#L17205) |
+| `DS4_CUDA_NO_GREEDY_VEC4` | value-aware kill switch; default off; nonempty value other than exact 0 disables | Disable greedy vec4 in CUDA greedy fast decode. | [ds4.c:17243](ds4.c#L17243) |
+| `DS4_CUDA_NO_GREEDY_VEC4_FALLBACK` | value-aware kill switch; default off; nonempty value other than exact 0 disables margin fallback | Disable greedy vec4 fallback in CUDA greedy fast decode. | [ds4.c:17303](ds4.c#L17303) |
+| `DS4_CUDA_NO_HC_SPLIT_NORM_SPLIT4096` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the split partial-reduction specialization for one-row, 4096-wide HC normalization. | [ds4_cuda.cu:31826](ds4_cuda.cu#L31826) |
+| `DS4_CUDA_NO_INDEXED_HEADS8` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA indexed heads8 optimization/path. | [ds4_cuda.cu:23586](ds4_cuda.cu#L23586) |
+| `DS4_CUDA_NO_INDEXED_TOPK_SORT` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA indexed topk sort optimization/path. | [ds4_cuda.cu:23577](ds4_cuda.cu#L23577) |
+| `DS4_CUDA_NO_INDEXER_DIRECT_ONE` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the indexer direct one CUDA indexer kernel/path. | [ds4_cuda.cu:18881](ds4_cuda.cu#L18881) |
+| `DS4_CUDA_NO_INDEXER_MXF4` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the indexer MXF4 CUDA indexer kernel/path. | [ds4_cuda.cu:17774](ds4_cuda.cu#L17774) |
+| `DS4_CUDA_NO_INDEXER_WMMA` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the indexer WMMA CUDA indexer kernel/path. | [ds4_cuda.cu:18891](ds4_cuda.cu#L18891) |
+| `DS4_CUDA_NO_INDEXER_WMMA128` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the indexer wmma128 CUDA indexer kernel/path. | [ds4_cuda.cu:18892](ds4_cuda.cu#L18892) |
+| `DS4_CUDA_NO_INDEXER_WMMA32` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the indexer wmma32 CUDA indexer kernel/path. | [ds4_cuda.cu:18910](ds4_cuda.cu#L18910) |
+| `DS4_CUDA_NO_INDEXER_WMMA64` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the indexer wmma64 CUDA indexer kernel/path. | [ds4_cuda.cu:18901](ds4_cuda.cu#L18901) |
+| `DS4_CUDA_NO_IQ2_XXS_SSD_PREFILL_MMQ` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Disable the CUDA IQ2 XXS SSD prefill MMQ optimization/path. | [ds4_cuda.cu:4629](ds4_cuda.cu#L4629) |
+| `DS4_CUDA_NO_MODEL_COPY` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA model copy optimization/path. | [ds4_cuda.cu:6472](ds4_cuda.cu#L6472) |
+| `DS4_CUDA_NO_MODEL_PREFETCH` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA model prefetch optimization/path. | [ds4_cuda.cu:2739](ds4_cuda.cu#L2739) |
+| `DS4_CUDA_NO_MOE_DEDUP` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA MoE dedup optimization/path. | [cuda/mmq/ds4_mmq.cu:6172](cuda/mmq/ds4_mmq.cu#L6172) |
+| `DS4_CUDA_NO_ORDERED_F16_MATMUL` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the ordered F16 matmul CUDA F16 path. | [ds4_cuda.cu:20811](ds4_cuda.cu#L20811) |
+| `DS4_CUDA_NO_PARALLEL_ROUTER_SELECT` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA parallel router select optimization/path. | [ds4_cuda.cu:24491](ds4_cuda.cu#L24491) |
+| `DS4_CUDA_NO_Q4_DENSE_SCRATCH` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q4 dense scratch CUDA Q4 optimization. | [cuda/mmq/ds4_mmq.cu:3986](cuda/mmq/ds4_mmq.cu#L3986) |
+| `DS4_CUDA_NO_Q4_GB10_FAST` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the GB10-specific Q4 fast-path family. | [cuda/mmq/ds4_mmq.cu:3908](cuda/mmq/ds4_mmq.cu#L3908) |
+| `DS4_CUDA_NO_Q4_GROUPED_ATTN_A` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q4 grouped attn a CUDA Q4 optimization. | [cuda/mmq/ds4_mmq.cu:4295](cuda/mmq/ds4_mmq.cu#L4295) |
+| `DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q4 grouped attn a batch CUDA Q4 optimization. | [cuda/mmq/ds4_mmq.cu:4305](cuda/mmq/ds4_mmq.cu#L4305) |
+| `DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL` | presence kill switch for the default-on GB10 path; default unset; any defined value including empty or 0 disables and dominates ENABLE/REQUIRE | Restore the eight pack/MMQ/unpack Q4 attention-A prefill projections. | [ds4_cuda.cu:41930](ds4_cuda.cu#L41930) |
+| `DS4_CUDA_NO_Q4_GROUPED_ATTN_A_Q81` | presence rollback for the default-on fixed-shape quantizer; default unset; any defined value including empty or 0 disables; REQUIRE then fails closed | Restore the canonical strided Q8_1 producer while retaining grouped Q4 attention-A prefill and its eight MMQ grids. | [cuda/mmq/ds4_mmq.cu:1762](cuda/mmq/ds4_mmq.cu#L1762); [ds4_cuda.cu:44143](ds4_cuda.cu#L44143) |
+| `DS4_CUDA_NO_Q4_K1024_PERSISTENT` | presence kill switch, default off; any defined value including 0 disables | Disable the Q4 K1024 persistent CUDA Q4 optimization. | [cuda/mmq/ds4_mmq.cu:3907](cuda/mmq/ds4_mmq.cu#L3907) |
+| `DS4_CUDA_NO_Q4_MMQ_16WARP` | value-aware rollback, default off; unset/empty/exact 0 permits the experiment, every other nonempty value disables it and overrides REQUEST/REQUIRE | Disable the experimental Stream-K-compatible CUDA Q4_K m128n128 16-warp prefill kernel. | [cuda/mmq/ds4_mmq.cu:1133](cuda/mmq/ds4_mmq.cu#L1133) |
+| `DS4_CUDA_NO_Q8_ALIGNED_DENSE_SCRATCH` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q8 aligned dense scratch CUDA Q8 optimization. | [cuda/mmq/ds4_mmq.cu:5578](cuda/mmq/ds4_mmq.cu#L5578) |
+| `DS4_CUDA_NO_Q8_ALIGNED_PERSISTENT` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q8 aligned persistent CUDA Q8 optimization. | [cuda/mmq/ds4_mmq.cu:5410](cuda/mmq/ds4_mmq.cu#L5410) |
+| `DS4_CUDA_NO_Q8_BATCH_EXACT_TOK2` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q8 batch exact tok2 CUDA Q8 optimization. | [ds4_cuda.cu:19852](ds4_cuda.cu#L19852) |
+| `DS4_CUDA_NO_Q8_BATCH_TOK4` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q8 batch tok4 CUDA Q8 optimization. | [ds4_cuda.cu:19805](ds4_cuda.cu#L19805) |
+| `DS4_CUDA_NO_Q8_BATCH_TOK8` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q8 batch tok8 CUDA Q8 optimization. | [ds4_cuda.cu:19788](ds4_cuda.cu#L19788) |
+| `DS4_CUDA_NO_Q8_BATCH_WARP` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q8 batch warp CUDA Q8 optimization. | [ds4_cuda.cu:19787](ds4_cuda.cu#L19787) |
+| `DS4_CUDA_NO_Q8_DP4A` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q8 DP4A CUDA Q8 optimization. | [ds4_cuda.cu:2391](ds4_cuda.cu#L2391) |
+| `DS4_CUDA_NO_Q8_F16_CACHE` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q8 F16 cache CUDA Q8 optimization. | [ds4_cuda.cu:2356](ds4_cuda.cu#L2356) |
+| `DS4_CUDA_NO_Q8_F32_CACHE` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q8 F32 cache CUDA Q8 optimization. | [ds4_cuda.cu:2411](ds4_cuda.cu#L2411) |
+| `DS4_CUDA_NO_Q8_FOLD` | value-aware kill switch, default off; nonempty value other than exact 0 disables and wins over enable | Disable the Q8_1 producer-to-consumer fold. | [ds4_cuda.cu:786](ds4_cuda.cu#L786) |
+| `DS4_CUDA_NO_Q8_FUSED_ALIGNED` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q8 fused aligned CUDA Q8 optimization. | [ds4_cuda.cu:20141](ds4_cuda.cu#L20141) |
+| `DS4_CUDA_NO_Q8_MMA` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q8 MMA CUDA Q8 optimization. | [ds4_cuda.cu:10781](ds4_cuda.cu#L10781) |
+| `DS4_CUDA_NO_Q8_PAIR_BATCH_EXACT` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q8 pair batch exact CUDA Q8 optimization. | [ds4_cuda.cu:20302](ds4_cuda.cu#L20302) |
+| `DS4_CUDA_NO_Q8_PAIR_BATCH_EXACT_TOK2` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the Q8 pair batch exact tok2 CUDA Q8 optimization. | [ds4_cuda.cu:20305](ds4_cuda.cu#L20305) |
+| `DS4_CUDA_NO_QKV_KV_ROPE_FUSE` | value-aware kill switch; default off; nonempty value other than exact 0 disables | Disable the CUDA QKV KV rope fuse optimization/path. | [ds4.c:17426](ds4.c#L17426) |
+| `DS4_CUDA_NO_QKV_PAIR` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA QKV pair optimization/path. | [ds4.c:17562](ds4.c#L17562) |
+| `DS4_CUDA_NO_SCORE_TILE` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA score tile optimization/path. | [ds4_cuda.cu:13734](ds4_cuda.cu#L13734); [ds4.c:68122](ds4.c#L68122) |
+| `DS4_CUDA_NO_SETDEVICE_CACHE` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the cached current-tier shortcut and call cudaSetDevice for every tier selection. | [ds4_cuda.cu:377](ds4_cuda.cu#L377) |
+| `DS4_CUDA_NO_SPLITKV_DECODE` | value-aware kill switch, default off; exact 0/empty is off, other nonempty values disable | Disable splitkv decode in CUDA split-KV attention/speculation. | [ds4_cuda.cu:2211](ds4_cuda.cu#L2211) |
+| `DS4_CUDA_NO_SPLITKV_SPEC` | value-aware kill switch; default off; nonempty value other than exact 0 disables | Disable splitkv spec in CUDA split-KV attention/speculation. | [ds4.c:17253](ds4.c#L17253) |
+| `DS4_CUDA_NO_SPLITKV_SPEC_BATCH_VERIFY` | value-aware kill switch; default off; nonempty value other than exact 0 disables | Disable splitkv spec batch verify in CUDA split-KV attention/speculation. | [ds4.c:17273](ds4.c#L17273) |
+| `DS4_CUDA_NO_SPLITKV_SPEC_TOPONLY_ROW0` | value-aware kill switch; default off; nonempty value other than exact 0 disables | Disable splitkv spec toponly row0 in CUDA split-KV attention/speculation. | [ds4.c:17263](ds4.c#L17263) |
+| `DS4_CUDA_NO_STREAMING_EXPERT_PERSISTENT_CACHE` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Disable streaming expert persistent cache in CUDA SSD streaming. | [ds4_cuda.cu:4115](ds4_cuda.cu#L4115) |
+| `DS4_CUDA_NO_STREAMING_SELECTED_BATCH_IO` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Disable streaming selected batch I/O in CUDA SSD streaming. | [ds4_cuda.cu:4736](ds4_cuda.cu#L4736) |
+| `DS4_CUDA_NO_STREAMING_SELECTED_EVENT_PIPELINE` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Disable streaming selected event pipeline in CUDA SSD streaming. | [ds4_cuda.cu:4868](ds4_cuda.cu#L4868) |
+| `DS4_CUDA_NO_TF32` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Use default cuBLAS math instead of TF32 tensor operations. | [ds4_cuda.cu:6657](ds4_cuda.cu#L6657) |
+| `DS4_CUDA_NO_TOP1` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the dedicated CUDA indexer top-1 kernel. | [ds4_cuda.cu:375](ds4_cuda.cu#L375) |
+| `DS4_CUDA_NO_TOPK1024` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the topk1024 CUDA indexer kernel/path. | [ds4_cuda.cu:19290](ds4_cuda.cu#L19290) |
+| `DS4_CUDA_NO_TOPK2048` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the topk2048 CUDA indexer kernel/path. | [ds4_cuda.cu:19297](ds4_cuda.cu#L19297) |
+| `DS4_CUDA_NO_TOPK2048_WIDE` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the topk2048 wide CUDA indexer kernel/path. | [ds4_cuda.cu:19212](ds4_cuda.cu#L19212) |
+| `DS4_CUDA_NO_TOPK8192` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the topk8192 CUDA indexer kernel/path. | [ds4_cuda.cu:19335](ds4_cuda.cu#L19335) |
+| `DS4_CUDA_NO_TOPK_CHUNKED` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the topk chunked CUDA indexer kernel/path. | [ds4_cuda.cu:19374](ds4_cuda.cu#L19374) |
+| `DS4_CUDA_NO_TOPK_STREAM` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the topk stream CUDA indexer kernel/path. | [ds4_cuda.cu:19366](ds4_cuda.cu#L19366) |
+| `DS4_CUDA_NO_TP_ATTN_OUT_HC_FUSE` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA TP attn out HC fuse optimization/path. | [ds4.c:17565](ds4.c#L17565) |
+| `DS4_CUDA_NO_VERIFY_DECODE2_SPLIT_TOP1` | value-aware kill switch; default off; nonempty value other than exact 0 disables | Disable the CUDA verify decode2 split top1 optimization/path. | [ds4.c:17223](ds4.c#L17223) |
+| `DS4_CUDA_NO_WARP_ROUTER_SELECT` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA warp router select optimization/path. | [ds4_cuda.cu:24490](ds4_cuda.cu#L24490) |
+| `DS4_CUDA_NO_WINDOW_ATTENTION` | presence kill switch; default unset (eligible path remains available); any defined value including 0 disables | Disable the CUDA window attention optimization/path. | [ds4_cuda.cu:23050](ds4_cuda.cu#L23050) |
+| `DS4_CUDA_NSYS_PREFILL_START_POS` | nonempty-string flag, default off; any nonempty value enables MMQ NVTX ranges (the value is not parsed as a position) | Enable MMQ NVTX annotations intended for Nsight Systems prefill capture. | [cuda/mmq/ds4_mmq.cu:49](cuda/mmq/ds4_mmq.cu#L49) |
+| `DS4_CUDA_NVTX` | strict flag, default off; only exact value 1 enables (a nonempty NSYS variable also enables ranges) | Enable NVTX ranges around MMQ work. | [cuda/mmq/ds4_mmq.cu:48](cuda/mmq/ds4_mmq.cu#L48) |
+| `DS4_CUDA_OUTPUT_FUSED_TOP1` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Fuse output projection with top-1 selection in greedy decode. | [ds4.c:17215](ds4.c#L17215) |
+| `DS4_CUDA_PREFILL_PIPELINE` | boolean, default follows CUDA TP decode; nonempty exact 0 disables, any other nonempty value enables | Control the CUDA multi-tier prefill pipeline. | [ds4.c:17454](ds4.c#L17454) |
+| `DS4_CUDA_PREFILL_PIPELINE_MB` | positive integer rows; default/invalid 512 | Set prefill-pipeline microbatch rows. | [ds4.c:17469](ds4.c#L17469) |
+| `DS4_CUDA_PREFILL_PIPELINE_Q8_CACHE` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Keep selective Q8 caches enabled while running the prefill pipeline. | [ds4.c:17464](ds4.c#L17464) |
+| `DS4_CUDA_PREFILL_PIPELINE_SEQUENTIAL` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Execute prefill pipeline stages sequentially for diagnosis. | [ds4.c:35525](ds4.c#L35525) |
+| `DS4_CUDA_PREFILL_PIPELINE_SYNC_BOUNDARY` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Synchronize CUDA at every prefill pipeline tier boundary. | [ds4.c:35572](ds4.c#L35572) |
+| `DS4_CUDA_Q4_ATTN_OUT_HC_ORACLE` | value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on | Compare fused Q4 attention-output/HC expansion with the canonical path and retain canonical output. | [ds4_cuda.cu:1475](ds4_cuda.cu#L1475) |
+| `DS4_CUDA_Q4_ATTN_OUT_HC_Q8K_EXPERIMENT` | value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on | Enable the experimental Q8_K-based Q4 attention-output/HC fusion. | [ds4_cuda.cu:37377](ds4_cuda.cu#L37377) |
+| `DS4_CUDA_Q4_GROUPED_ATTN_A_ORACLE` | value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on | Compare grouped attention-A against the canonical per-group result. | [ds4_cuda.cu:1477](ds4_cuda.cu#L1477) |
+| `DS4_CUDA_Q4_K1024_PERSISTENT_ORACLE` | value-aware flag, default off; nonempty value other than exact 0 enables and implies candidate admission | Bitwise-compare the exact-shape persistent Q4 K1024 kernel with canonical MMVQ and retain canonical output. | [cuda/mmq/ds4_mmq.cu:3738](cuda/mmq/ds4_mmq.cu#L3738) |
+| `DS4_CUDA_Q4_K1024_PERSISTENT_STATS` | value-aware flag, default off; nonempty value other than exact 0 enables | Print exact-shape persistent Q4 K1024 dispatch counters at exit. | [cuda/mmq/ds4_mmq.cu:3737](cuda/mmq/ds4_mmq.cu#L3737) |
+| `DS4_CUDA_Q4_MMQ_16WARP` | value-aware opt-in cached on the first Q4_K dense or dense-pair MMQ call; unset/empty/exact 0 is off, every other nonempty value requests the candidate; rollback wins; standalone dense requires M>=1024 and admits K<=8192 including attention output-B, while dense-pair admits legs down to M=512, remains bounded to K<=4096, and shares one Q8_1 activation; grids require at least 80% whole-tile SM-wave efficiency and use the canonical Stream-K partition/fixup below its 90% cutoff; ineligible optional shapes fall back | Enable the experimental exact-integer CUDA Q4_K m128n128 16-warp kernel for eligible dense and dense-pair prefills without changing the canonical FP32 reduction tree. | [cuda/mmq/ds4_mmq.cu:1126](cuda/mmq/ds4_mmq.cu#L1126) |
+| `DS4_CUDA_Q8_F16_ALL` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Control the Q8 F16 all CUDA quantized-matmul/cache optimization. | [ds4_cuda.cu:2358](ds4_cuda.cu#L2358) |
+| `DS4_CUDA_Q8_F16_CACHE_MB` | unsigned integer MiB, full-string parse; default unlimited; 0 disables this cache | Limit the selective Q8-to-F16 derived-weight cache. | [ds4_cuda.cu:2218](ds4_cuda.cu#L2218) |
+| `DS4_CUDA_Q8_F16_CACHE_RESERVE_MB` | unsigned integer MiB, full-string parse; default is VRAM-dependent (>=112 GiB: 512; >=40 GiB: max(768,1%); smaller: max(4096,5%)) | Reserve free VRAM when growing the selective Q8-to-F16 cache. | [ds4_cuda.cu:2224](ds4_cuda.cu#L2224) |
+| `DS4_CUDA_Q8_F32_ALL` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Control the Q8 F32 all CUDA quantized-matmul/cache optimization. | [ds4_cuda.cu:2412](ds4_cuda.cu#L2412) |
+| `DS4_CUDA_Q8_F32_LARGE` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Control the Q8 F32 large CUDA quantized-matmul/cache optimization. | [ds4_cuda.cu:2416](ds4_cuda.cu#L2416) |
+| `DS4_CUDA_Q8_F32_PRELOAD` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Control the Q8 F32 preload CUDA quantized-matmul/cache optimization. | [ds4_cuda.cu:8597](ds4_cuda.cu#L8597) |
+| `DS4_CUDA_Q8_FOLD_ORACLE` | strict flag, default off; only exact value 1 enables | Compare folded Q8_1 bytes and consumer outputs against canonical work while retaining canonical results. | [cuda/mmq/ds4_mmq.cu:381](cuda/mmq/ds4_mmq.cu#L381) |
+| `DS4_CUDA_Q8_HC_EXPAND_FUSED` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, other nonempty values force fused | Force the fused Q8 shared-down/HC expansion path. | [ds4_cuda.cu:2084](ds4_cuda.cu#L2084) |
+| `DS4_CUDA_Q8_HC_EXPAND_STATS` | false-like-aware flag, default off; 0/false/no/off is off, other nonempty values print report | Print Q8 shared-down/HC policy and dispatch counters at exit. | [ds4_cuda.cu:2090](ds4_cuda.cu#L2090) |
+| `DS4_CUDA_Q8_NO_ALIGNED` | value-aware kill switch, default off; nonempty value other than exact 0 disables aligned Q8 kernels | Disable aligned Q8 CUDA matmul kernels. | [ds4_cuda.cu:1021](ds4_cuda.cu#L1021) |
+| `DS4_CUDA_Q8_PAIR_BATCH` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Control the Q8 pair batch CUDA quantized-matmul/cache optimization. | [ds4_cuda.cu:20167](ds4_cuda.cu#L20167) |
+| `DS4_CUDA_QKV_KV_ROPE_FUSE` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control or tune the CUDA QKV KV rope fuse path. | [ds4.c:17429](ds4.c#L17429) |
+| `DS4_CUDA_Q_NORM_ROPE_FUSE` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control or tune the CUDA q norm rope fuse path. | [ds4.c:17418](ds4.c#L17418) |
+| `DS4_CUDA_REQUIRE_IQ2_XXS_SSD_PREFILL_MMQ` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Require the CUDA IQ2 XXS SSD prefill MMQ path; fail closed when unavailable. | [ds4_cuda.cu:4631](ds4_cuda.cu#L4631) |
+| `DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_BATCH` | value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on | Fail if grouped batched attention-A cannot be used. | [ds4_cuda.cu:40198](ds4_cuda.cu#L40198) |
+| `DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_PREFILL` | value-aware fail-closed assertion, default off; unset/empty/exact 0 is off, any other nonempty value requests the candidate and rejects ineligibility before enqueue | Require the GB10 grouped Q4_K attention-A prefill path instead of silently using pack/MMQ/unpack. | [ds4_cuda.cu:41889](ds4_cuda.cu#L41889) |
+| `DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_Q81` | value-aware fail-closed assertion, default off; unset/empty/exact 0 is off, any other nonempty value requests grouped prefill and the fixed K=4096, groups=8, rank=1024 Q8_1 producer; NO wins | Require the eight-warp K4096/G8x2 Q8_1 producer instead of silently using the generic strided quantizer. | [cuda/mmq/ds4_mmq.cu:1767](cuda/mmq/ds4_mmq.cu#L1767); [ds4_cuda.cu:44089](ds4_cuda.cu#L44089) |
+| `DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT` | presence flag, default off; any defined value including 0 makes ineligible candidate fail closed | Fail when the exact Q4 K1024 persistent candidate is unavailable instead of using MMVQ. | [cuda/mmq/ds4_mmq.cu:3929](cuda/mmq/ds4_mmq.cu#L3929) |
+| `DS4_CUDA_REQUIRE_Q4_MMQ_16WARP` | value-aware fail-closed prefill opt-in cached on the first Q4_K dense or dense-pair MMQ call; unset/empty/exact 0 is off, every other nonempty value requests and requires the candidate for N>8; a dense-pair is rejected before allocation unless both legs are eligible; rollback, disabled MMQ, ineligibility, or preflight failure prevents fallback; decode/speculative N<=8 remains on MMVQ | Require the experimental CUDA Q4_K 16-warp prefill kernel so benchmark runs cannot silently measure another path. | [cuda/mmq/ds4_mmq.cu:1130](cuda/mmq/ds4_mmq.cu#L1130); [ds4_cuda.cu:38208](ds4_cuda.cu#L38208); [ds4_cuda.cu:38358](ds4_cuda.cu#L38358) |
+| `DS4_CUDA_REQUIRE_STREAMING_EXPERT_PERSISTENT_CACHE` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Require streaming expert persistent cache in CUDA SSD streaming; fail closed when unavailable. | [ds4_cuda.cu:4117](ds4_cuda.cu#L4117) |
+| `DS4_CUDA_REQUIRE_STREAMING_SELECTED_BATCH_IO` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Require streaming selected batch I/O in CUDA SSD streaming; fail closed when unavailable. | [ds4_cuda.cu:4738](ds4_cuda.cu#L4738) |
+| `DS4_CUDA_REQUIRE_STREAMING_SELECTED_EVENT_PIPELINE` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Require streaming selected event pipeline in CUDA SSD streaming; fail closed when unavailable. | [ds4_cuda.cu:4870](ds4_cuda.cu#L4870) |
+| `DS4_CUDA_SERIAL_F16_MATMUL` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Control the serial F16 matmul CUDA F16 matmul path. | [ds4_cuda.cu:20801](ds4_cuda.cu#L20801) |
+| `DS4_CUDA_SERIAL_ROUTER` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Control or tune the CUDA serial router path. | [ds4_cuda.cu:20806](ds4_cuda.cu#L20806) |
+| `DS4_CUDA_SESSION_BATCH_ATTN_ALIAS` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control the grouped multi-session CUDA attn alias stage. | [ds4.c:69902](ds4.c#L69902) |
+| `DS4_CUDA_SESSION_BATCH_ATTN_CORE` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control the grouped multi-session CUDA attn core stage. | [ds4.c:69905](ds4.c#L69905) |
+| `DS4_CUDA_SESSION_BATCH_ATTN_POST` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control the grouped multi-session CUDA attn post stage. | [ds4.c:69917](ds4.c#L69917) |
+| `DS4_CUDA_SESSION_BATCH_ATTN_PRE` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control the grouped multi-session CUDA attn pre stage. | [ds4.c:69898](ds4.c#L69898) |
+| `DS4_CUDA_SESSION_BATCH_FFN_PRE` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control the grouped multi-session CUDA ffn pre stage. | [ds4.c:69894](ds4.c#L69894) |
+| `DS4_CUDA_SESSION_BATCH_INTERLEAVE` | boolean, default on; unset/empty/nonzero enables pipeline interleaving; exact 0 disables | Control the grouped multi-session CUDA interleave stage. | [ds4.c:70578](ds4.c#L70578) |
+| `DS4_CUDA_SESSION_BATCH_KV_STORE` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control the grouped multi-session CUDA KV store stage. | [ds4.c:69913](ds4.c#L69913) |
+| `DS4_CUDA_SESSION_BATCH_MOE` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control the grouped multi-session CUDA MoE stage. | [ds4.c:69649](ds4.c#L69649) |
+| `DS4_CUDA_SESSION_BATCH_MOE_COMBINE_ROWS` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control the grouped multi-session CUDA MoE combine rows stage. | [ds4.c:69431](ds4.c#L69431) |
+| `DS4_CUDA_SESSION_BATCH_QKV` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control the grouped multi-session CUDA QKV stage. | [ds4.c:69909](ds4.c#L69909) |
+| `DS4_CUDA_SESSION_BATCH_SHARED` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control the grouped multi-session CUDA shared stage. | [ds4.c:69890](ds4.c#L69890) |
+| `DS4_CUDA_SPLITKV_CHUNK` | integer scores/chunk; default 512; clamped 1..512 | Control splitkv chunk in CUDA split-KV attention/speculation. | [ds4_cuda.cu:22568](ds4_cuda.cu#L22568) |
+| `DS4_CUDA_SPLITKV_DECODE` | value-aware boolean, default off; exact 0/empty is off, other nonempty values enable; mere presence also excludes one session-batch path | Enable split-KV decode attention. | [ds4_cuda.cu:2213](ds4_cuda.cu#L2213); [ds4.c:68111](ds4.c#L68111) |
+| `DS4_CUDA_SPLITKV_GLOBAL_SOFTMAX` | value-aware opt-in, default off; exact 0/empty is off, other nonempty values enable | Use the global-softmax variant of split-KV attention. | [ds4_cuda.cu:22589](ds4_cuda.cu#L22589) |
+| `DS4_CUDA_SPLITKV_MIN_SCORE` | integer score count 0..UINT32_MAX; default 0 when explicitly enabled, otherwise 512; CUDA kernel clamps to 0..8192 | Set the minimum visible-score count for split-KV attention. | [ds4_cuda.cu:22557](ds4_cuda.cu#L22557); [ds4.c:17402](ds4.c#L17402) |
+| `DS4_CUDA_SPLITKV_S` | integer exact split count; unset/invalid = automatic; valid value clamped 1..16 | Control splitkv s in CUDA split-KV attention/speculation. | [ds4_cuda.cu:22578](ds4_cuda.cu#L22578) |
+| `DS4_CUDA_SPLITKV_SPEC` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Enable split-KV speculative decoding. | [ds4.c:17255](ds4.c#L17255) |
+| `DS4_CUDA_SPLITKV_SPEC_BATCH_VERIFY` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Use batched verification for split-KV speculation. | [ds4.c:17275](ds4.c#L17275) |
+| `DS4_CUDA_SPLITKV_SPEC_LOG` | presence diagnostic flag; default off; any defined value including 0 enables | Log split-KV speculative-decode admission and fallback decisions. | [ds4.c:55812](ds4.c#L55812) |
+| `DS4_CUDA_SPLITKV_SPEC_TIMING` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Print timing for split-KV speculative-decode stages. | [ds4.c:55851](ds4.c#L55851) |
+| `DS4_CUDA_SPLITKV_SPEC_TOPONLY_ROW0` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Compute only the top result for row zero in split-KV speculation. | [ds4.c:17265](ds4.c#L17265) |
+| `DS4_CUDA_SPLITKV_S_FLOOR` | integer split count; default 4; clamped 1..16 | Control splitkv s floor in CUDA split-KV attention/speculation. | [ds4_cuda.cu:22571](ds4_cuda.cu#L22571) |
+| `DS4_CUDA_SPLITKV_S_MAX` | integer split count; default 16; clamped 1..16 | Control splitkv s max in CUDA split-KV attention/speculation. | [ds4_cuda.cu:22574](ds4_cuda.cu#L22574) |
+| `DS4_CUDA_STREAMING_EXPERT_CACHE_PROFILE` | presence diagnostic flag; default off; any defined value including 0 enables | Profile CUDA SSD-streaming streaming expert cache. | [ds4.c:21849](ds4.c#L21849) |
+| `DS4_CUDA_STREAMING_EXPERT_PERSISTENT_CACHE_ORACLE` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Run the diagnostic oracle for CUDA SSD-streaming streaming expert persistent cache. | [ds4_cuda.cu:4121](ds4_cuda.cu#L4121) |
+| `DS4_CUDA_STREAMING_EXPERT_PERSISTENT_CACHE_STATS` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Print counters for CUDA SSD-streaming streaming expert persistent cache. | [ds4_cuda.cu:4119](ds4_cuda.cu#L4119) |
+| `DS4_CUDA_STREAMING_PREFILL_BATCH_SELECTED_PROFILE` | presence diagnostic flag; default off; any defined value including 0 enables | Profile CUDA SSD-streaming streaming prefill batch selected. | [ds4.c:21932](ds4.c#L21932) |
+| `DS4_CUDA_STREAMING_SELECTED_BATCH_IO_ORACLE` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Run the diagnostic oracle for CUDA SSD-streaming streaming selected batch I/O. | [ds4_cuda.cu:4740](ds4_cuda.cu#L4740) |
+| `DS4_CUDA_STREAMING_SELECTED_BATCH_IO_PROFILE` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Profile CUDA SSD-streaming streaming selected batch I/O. | [ds4_cuda.cu:5711](ds4_cuda.cu#L5711) |
+| `DS4_CUDA_STREAMING_SELECTED_EVENT_PIPELINE_ORACLE` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Run the diagnostic oracle for CUDA SSD-streaming streaming selected event pipeline. | [ds4_cuda.cu:4872](ds4_cuda.cu#L4872) |
+| `DS4_CUDA_STREAMING_SELECTED_EVENT_PIPELINE_STATS` | false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate | Print counters for CUDA SSD-streaming streaming selected event pipeline. | [ds4_cuda.cu:4874](ds4_cuda.cu#L4874) |
+| `DS4_CUDA_STRICT_WEIGHT_CACHE` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Fail a weight lookup when cache allocation fails instead of falling back to mapped model memory. | [ds4_cuda.cu:6388](ds4_cuda.cu#L6388) |
+| `DS4_CUDA_SYNC_XDEV` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Synchronize cross-device CUDA copies for debugging and error localization. | [ds4_cuda.cu:363](ds4_cuda.cu#L363) |
+| `DS4_CUDA_TP_ATTN` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel attn execution. | [ds4.c:17035](ds4.c#L17035) |
+| `DS4_CUDA_TP_ATTN_CACHE_DUP` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel attn cache dup execution. | [ds4.c:17059](ds4.c#L17059) |
+| `DS4_CUDA_TP_ATTN_HEADS` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel attn heads execution. | [ds4.c:17051](ds4.c#L17051) |
+| `DS4_CUDA_TP_ATTN_OUT_HC_FUSE` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Control CUDA tensor/expert-parallel attn out HC fuse execution. | [ds4.c:17564](ds4.c#L17564) |
+| `DS4_CUDA_TP_ATTN_PEER_READ` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel attn peer read execution. | [ds4.c:17043](ds4.c#L17043) |
+| `DS4_CUDA_TP_EP_BALANCED_SHARED_MID` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel EP balanced shared mid execution. | [ds4.c:17116](ds4.c#L17116) |
+| `DS4_CUDA_TP_EP_DELAY_REDUCE` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel EP delay reduce execution. | [ds4.c:17091](ds4.c#L17091) |
+| `DS4_CUDA_TP_EP_DIRECT_RETURN` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel EP direct return execution. | [ds4.c:17083](ds4.c#L17083) |
+| `DS4_CUDA_TP_EP_DUAL_PREQUANT` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel EP dual prequant execution. | [ds4.c:17125](ds4.c#L17125) |
+| `DS4_CUDA_TP_EP_FUSED_HC_REDUCE` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel EP fused HC reduce execution. | [ds4.c:17099](ds4.c#L17099) |
+| `DS4_CUDA_TP_EP_FUSED_SHARED_MID` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel EP fused shared mid execution. | [ds4.c:17107](ds4.c#L17107) |
+| `DS4_CUDA_TP_EP_PACK_EXACT` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel EP pack exact execution. | [ds4.c:17075](ds4.c#L17075) |
+| `DS4_CUDA_TP_MOE` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel MoE execution. | [ds4.c:17067](ds4.c#L17067) |
+| `DS4_CUDA_TP_MOE_COPY3` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel MoE copy3 execution. | [ds4.c:17149](ds4.c#L17149) |
+| `DS4_CUDA_TP_MOE_DELAY_REDUCE` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel MoE delay reduce execution. | [ds4.c:17133](ds4.c#L17133) |
+| `DS4_CUDA_TP_MOE_PACK` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel MoE pack execution. | [ds4.c:17141](ds4.c#L17141) |
+| `DS4_CUDA_TP_MOE_PEER_READ` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel MoE peer read execution. | [ds4.c:17157](ds4.c#L17157) |
+| `DS4_CUDA_TP_MOE_PEER_ROUTER` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel MoE peer router execution. | [ds4.c:17165](ds4.c#L17165) |
+| `DS4_CUDA_TP_OUTPUT` | boolean, default on; empty/unset/nonzero enables, exact 0 disables | Control CUDA tensor/expert-parallel output execution. | [ds4.c:51713](ds4.c#L51713) |
+| `DS4_CUDA_TP_OUTPUT_WAYS` | integer 2..DS4_MAX_GPUS (16); default 8; invalid value falls back to 2; capped by available GPUs | Set the number of GPU ways used to shard CUDA tensor-parallel output projection. | [ds4.c:58](ds4.c#L58); [ds4.c:51720](ds4.c#L51720) |
+| `DS4_CUDA_TP_PREFILL_ATTN_OUTPUT` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel prefill attn output execution. | [ds4.c:17445](ds4.c#L17445) |
+| `DS4_CUDA_TP_PREFILL_FFN` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel prefill ffn execution. | [ds4.c:17437](ds4.c#L17437) |
+| `DS4_CUDA_TP_Q` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel q execution. | [ds4.c:17189](ds4.c#L17189) |
+| `DS4_CUDA_TP_SHARED` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel shared execution. | [ds4.c:17173](ds4.c#L17173) |
+| `DS4_CUDA_TP_SHARED_FOLD` | value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables | Control CUDA tensor/expert-parallel shared fold execution. | [ds4.c:17181](ds4.c#L17181) |
+| `DS4_CUDA_VERIFY_DECODE2_SPLIT_TOP1` | value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables | Enable the split top-1 path for two-row verification decode. | [ds4.c:17225](ds4.c#L17225) |
+| `DS4_CUDA_WEIGHT_ARENA_CHUNK_MB` | positive integer MiB; default 1792; clamped 256..8192 and raised/aligned when one allocation needs more | Set the CUDA selective-weight arena allocation chunk. | [ds4_cuda.cu:6311](ds4_cuda.cu#L6311) |
+| `DS4_CUDA_WEIGHT_CACHE` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Force selective CUDA weight caching instead of direct mapped access. | [ds4_cuda.cu:1246](ds4_cuda.cu#L1246) |
+| `DS4_CUDA_WEIGHT_CACHE_LIMIT_GB` | unsigned integer GiB; default/0 = unlimited; parser accepts a numeric prefix even with trailing text | Limit total CUDA selective-weight cache allocation. | [ds4_cuda.cu:6299](ds4_cuda.cu#L6299) |
+| `DS4_CUDA_WEIGHT_CACHE_VERBOSE` | presence diagnostic flag; default off; any defined value including 0 enables | Print CUDA weight mapping, caching, and preload diagnostics. | [ds4_cuda.cu:1297](ds4_cuda.cu#L1297) |
+| `DS4_CUDA_WEIGHT_PRELOAD` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Request proactive CUDA weight caching/preloading. | [ds4_cuda.cu:1247](ds4_cuda.cu#L1247) |
+| `DS4_CUDA_WEIGHT_PRELOAD_SPAN_MB` | positive integer MiB; default 1024; clamped 64..4096 | Set the maximum span size used by CUDA weight preload. | [ds4.c:2880](ds4.c#L2880) |
+| `DS4_CUDA_WINDOW_ATTENTION` | presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies | Control or tune the CUDA window attention path. | [ds4_cuda.cu:23051](ds4_cuda.cu#L23051) |
+
+</details>
+
+<details>
+<summary><strong>ROCm (147)</strong></summary>
+
+| Variable | Accepted value and default | Effect | Source |
+| --- | --- | --- | --- |
+| `DS4_ROCM_DECODE_STAGE_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm decode stage profile. | [ds4.c:18244](ds4.c#L18244) |
+| `DS4_ROCM_DECODE_STAGE_PROFILE_LAYER` | layer filter subordinate to DS4_ROCM_DECODE_STAGE_PROFILE; unset or whitespace-only: all layers allowed by the parent flag; otherwise the whitespace-trimmed value must be a complete base-10 strtoul result <= UINT32_MAX equal to the current layer; invalid values match none | Restricts the ROCm decode stage profiler to one layer; it does not enable profiling by itself. | [ds4.c:29116](ds4.c#L29116) |
+| `DS4_ROCM_DISABLE_GLM_STREAMING_PREFILL_FULL_LAYER` | integer selector/tuning value; unset or invalid uses internal automatic/default value | Disable/roll back rocm disable glm streaming prefill full layer. | [ds4.c:42709](ds4.c#L42709) |
+| `DS4_ROCM_DISABLE_GLM_STREAMING_PREFILL_FULL_LAYER_PREPARE` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable glm streaming prefill full layer prepare. | [ds4.c:42727](ds4.c#L42727) |
+| `DS4_ROCM_DISABLE_GLM_STREAMING_PREFILL_SELECTED_ASYNC_LOAD` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable glm streaming prefill selected async load. | [ds4.c:46420](ds4.c#L46420) |
+| `DS4_ROCM_DISABLE_GLM_STREAMING_SELECTED_ASYNC_LOAD` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable glm streaming selected async load. | [ds4.c:44328](ds4.c#L44328) |
+| `DS4_ROCM_DISABLE_IQ2_SELECTED_EXPERT_VIEWS` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable iq2 selected expert views. | [ds4.c:21079](ds4.c#L21079) |
+| `DS4_ROCM_DISABLE_IQ2_STREAM_ADDR_TABLE` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable iq2 stream addr table. | [ds4.c:6548](ds4.c#L6548) |
+| `DS4_ROCM_DISABLE_Q4_DENSE_PAIR` | presence rollback; unset leaves opt-in policy unchanged | Disable/roll back rocm disable q4 dense pair. | [rocm/ds4_rocm_q4.cuh:435](rocm/ds4_rocm_q4.cuh#L435) |
+| `DS4_ROCM_DISABLE_Q4_GROUPED_ATTN_A` | presence rollback; unset permits the caller-marked resident decode production-shape default and explicit ENABLE/REQUIRE; any defined value including empty or 0 disables all grouped attention-A paths and wins over ENABLE/REQUIRE | Restore eight standalone Q4 attention-A projections instead of the two-dispatch grouped path. | [rocm/ds4_rocm_q4.cuh:868](rocm/ds4_rocm_q4.cuh#L868) |
+| `DS4_ROCM_DISABLE_Q4_PREFILL_TILE8` | presence rollback; TILE8 is default for 9..4096 tokens | Disable/roll back rocm disable q4 prefill tile8. | [rocm/ds4_rocm_q4.cuh:448](rocm/ds4_rocm_q4.cuh#L448) |
+| `DS4_ROCM_DISABLE_Q4_PREFILL_WMMA` | value-aware authoritative opt-out for the automatic resident path and explicit SSD/REQUIRE requests; unset/0/false/no/off leaves policy unchanged, while empty or any other value disables; REQUIRE then fails closed | Prevent the gfx1151 direct-Q4 WMMA prefill path from dispatching and retain the Q8_K-plus-TILE8/TILE4 path. | [rocm/ds4_rocm_q4.cuh:1626](rocm/ds4_rocm_q4.cuh#L1626) |
+| `DS4_ROCM_DISABLE_Q4_PREFILL_WMMA_K128` | value-aware rollback for the default K128/P144 stage; unset/0/false/no/off keeps K128 after the normal direct-Q4 WMMA gates, K64 control, 256-row geometry, and 16-byte activation alignment pass; empty or any other value restores K64; incompatible launches also retain K64 and K64=0 retains the K32 rollback | Roll aligned resident q_b-shaped 256-row direct-WMMA launches back from four-qgroup K128/P144 staging and float4 activation loads to K64/P80. | [rocm/ds4_rocm_q4.cuh:1874](rocm/ds4_rocm_q4.cuh#L1874) |
+| `DS4_ROCM_DISABLE_Q4_SELECTED_EXPERT_VIEWS` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable q4 selected expert views. | [ds4.c:21150](ds4.c#L21150) |
+| `DS4_ROCM_DISABLE_RESIDENT_IQ2_SORTED` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable resident iq2 sorted. | [rocm/ds4_rocm_moe_launch.cuh:751](rocm/ds4_rocm_moe_launch.cuh#L751) |
+| `DS4_ROCM_DISABLE_ROUTED_PAIR_SWIGLU_FUSION` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable routed pair swiglu fusion. | [ds4.c:18543](ds4.c#L18543) |
+| `DS4_ROCM_DISABLE_STREAMING_COLD_DECODE_PREFILL` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming cold decode prefill. | [ds4.c:32021](ds4.c#L32021) |
+| `DS4_ROCM_DISABLE_STREAMING_DECODE_PREFILL` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming decode prefill. | [ds4.c:31970](ds4.c#L31970) |
+| `DS4_ROCM_DISABLE_STREAMING_EXPERT_ADDR_TABLE` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming expert addr table. | [ds4.c:18539](ds4.c#L18539) |
+| `DS4_ROCM_DISABLE_STREAMING_EXPERT_HOTLIST` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming expert hotlist. | [ds4.c:21304](ds4.c#L21304) |
+| `DS4_ROCM_DISABLE_STREAMING_FULL_EXPERT_ADDR_TABLE` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming full expert addr table. | [ds4.c:18257](ds4.c#L18257) |
+| `DS4_ROCM_DISABLE_STREAMING_LAYER_BATCH` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming layer batch. | [ds4.c:18253](ds4.c#L18253) |
+| `DS4_ROCM_DISABLE_STREAMING_MADVISE_WILLNEED` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming madvise willneed. | [ds4.c:18226](ds4.c#L18226) |
+| `DS4_ROCM_DISABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill batch selected addr. | [ds4.c:18537](ds4.c#L18537) |
+| `DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_MADVISE` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill layer madvise. | [ds4.c:18480](ds4.c#L18480) |
+| `DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_PAGEIN` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill layer pagein. | [ds4.c:18448](ds4.c#L18448) |
+| `DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_PAGEIN_OVERLAP` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill layer pagein overlap. | [ds4.c:19335](ds4.c#L19335) |
+| `DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_PREAD` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill layer pread. | [ds4.c:18468](ds4.c#L18468) |
+| `DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_PREPARE` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill layer prepare. | [ds4.c:18460](ds4.c#L18460) |
+| `DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_PREPARE_OVERLAP` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill layer prepare overlap. | [ds4.c:19333](ds4.c#L19333) |
+| `DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_READAHEAD` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill layer readahead. | [ds4.c:18458](ds4.c#L18458) |
+| `DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_ASYNC_LOAD` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill selected async load. | [ds4.c:46684](ds4.c#L46684) |
+| `DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_MADVISE` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill selected madvise. | [ds4.c:18438](ds4.c#L18438) |
+| `DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_PAGEIN` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill selected pagein. | [ds4.c:18428](ds4.c#L18428) |
+| `DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_PROFILE` | presence rollback flag; unset keeps automatic/default path | Collect timing/profile diagnostics for rocm disable streaming prefill selected profile. | [ds4.c:18922](ds4.c#L18922) |
+| `DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_READAHEAD` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill selected readahead. | [ds4.c:19974](ds4.c#L19974) |
+| `DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_READAHEAD_SHARED` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming prefill selected readahead shared. | [ds4.c:19984](ds4.c#L19984) |
+| `DS4_ROCM_DISABLE_STREAMING_READAHEAD` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming readahead. | [ds4.c:18219](ds4.c#L18219) |
+| `DS4_ROCM_DISABLE_STREAMING_SELECTED_ASYNC_LOAD` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming selected async load. | [ds4.c:44587](ds4.c#L44587) |
+| `DS4_ROCM_DISABLE_STREAMING_SPLIT_SELECTED` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming split selected. | [rocm/ds4_rocm_moe_launch.cuh:665](rocm/ds4_rocm_moe_launch.cuh#L665) |
+| `DS4_ROCM_DISABLE_STREAMING_STATIC_DECODE_MAP` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming static decode map. | [ds4.c:18231](ds4.c#L18231) |
+| `DS4_ROCM_DISABLE_STREAMING_STATIC_MAP_STATE_CACHE` | presence rollback flag; unset keeps automatic/default path | Disable/roll back rocm disable streaming static map state cache. | [ds4.c:18244](ds4.c#L18244) |
+| `DS4_ROCM_DSV4_PREQUANT_DECODE` | sampled once; unset: enabled; present empty or exact 0: disabled; every other present value: enabled; quality mode and GLM models force it off regardless | ROCm DeepSeek-V4 decode: quantizes one-token F32 activations to Q8 once and selects the prequantized Q8_0/DP4A projection kernels instead of the full-F32 activation paths. | [rocm/ds4_rocm_runtime.cuh:4775](rocm/ds4_rocm_runtime.cuh#L4775) |
+| `DS4_ROCM_ENABLE_MXFP4_LDSB` | presence opt-in; unset=off; any defined value including empty or 0 enables the candidate when the MXFP4 path, sorted expert tiles, token count >= 128, LDS-size limit, and dimension-alignment gates all pass | Select the ROCm MXFP4 prefill gate/up kernel that stages eight gate and eight up weight rows in LDS and reuses them across expert tiles of up to 128 tokens. | [rocm/ds4_rocm_moe_launch.cuh:782](rocm/ds4_rocm_moe_launch.cuh#L782) |
+| `DS4_ROCM_ENABLE_MXFP4_ROW64` | presence opt-in; unset=off; any defined value including empty or 0 enables the candidate when the MXFP4 sorted-tile path has at least 8 tokens and the TILE32, LDSB, and TILE4 candidates are not selected | Select the ROCm MXFP4 gate/up tile8 occupancy variant with 64 row slots and 512 threads per block. | [rocm/ds4_rocm_moe_launch.cuh:798](rocm/ds4_rocm_moe_launch.cuh#L798) |
+| `DS4_ROCM_ENABLE_MXFP4_TILE32` | presence opt-in; unset=off; any defined value including empty or 0 enables the candidate when the MXFP4 sorted-tile path has at least 32 tokens and the expert intermediate dimension is divisible by 32 | Select the ROCm MXFP4 gate/up tile32 kernel, reusing each loaded expert-weight chunk across as many as 32 tokens. | [rocm/ds4_rocm_moe_launch.cuh:786](rocm/ds4_rocm_moe_launch.cuh#L786) |
+| `DS4_ROCM_ENABLE_MXFP4_TILE4` | presence opt-in; unset=off; any defined value including empty or 0 enables the candidate when the MXFP4 sorted-tile path has at least 5 tokens and neither TILE32 nor LDSB is selected | Select the ROCm MXFP4 gate/up tile4 occupancy variant, reducing staged-activation LDS per block. | [rocm/ds4_rocm_moe_launch.cuh:794](rocm/ds4_rocm_moe_launch.cuh#L794) |
+| `DS4_ROCM_ENABLE_Q4_DENSE_PAIR` | presence opt-in; unset=off; DISABLE takes precedence | Enable rocm enable q4 dense pair. | [rocm/ds4_rocm_q4.cuh:434](rocm/ds4_rocm_q4.cuh#L434) |
+| `DS4_ROCM_ENABLE_Q4_GROUPED_ATTN_A` | presence opt-in outside the default scope; the exact caller-marked resident decode shape groups=8, N=1, K=4096, M=1024 is automatic, while row-at-a-time batch fallbacks are not; DISABLE wins | Enable grouped Q4 attention-A for eligible slices, non-production shapes, or explicit experiments in addition to the resident decode default. | [rocm/ds4_rocm_q4.cuh:872](rocm/ds4_rocm_q4.cuh#L872) |
+| `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA` | value-aware compatibility control; unset keeps automatic resident direct-Q4 WMMA for standalone dense and attention-output A while attention-output B remains on Q8_K+TILE8; empty or any value other than 0/false/no/off explicitly retains the same eligible A paths but no longer opts B into direct WMMA; explicit 0/false/no/off opts out unless REQUIRE is set, while DISABLE is the authoritative rollback; N=256..4096, K a positive multiple of 256, resident non-quality gfx1151 wave32 only; SSD has a separate gate | Use compressed Q4_K-to-F16 register dequantization plus shape-selected 64-token by 64/128/256-row WMMA tiles, K64/P80 staging on 64/128 rows, and default K128/P144 float4 staging on aligned 256 rows, without Q8_K activation scratch or an F16 weight sidecar. | [rocm/ds4_rocm_q4.cuh:1622](rocm/ds4_rocm_q4.cuh#L1622) |
+| `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_K64` | value-aware base staging control; unset, empty, or any value other than 0/false/no/off uses K64/P80 on 64/128-row or K128-incompatible launches and permits default K128/P144 on aligned 256-row launches; 0/false/no/off suppresses both wider stages and rolls back to K32; DS4_ROCM_DISABLE_Q4_PREFILL_WMMA wins | Stage two adjacent 32-value Q4_K groups and a 64-value activation slice in one padded P80 LDS tile as the narrower geometry and K128 fallback, halving K32 workgroup barriers while preserving its activation traffic and accumulation order. | [rocm/ds4_rocm_q4.cuh:1870](rocm/ds4_rocm_q4.cuh#L1870) |
+| `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_SSD` | value-aware SSD-only opt-in, default off; unset/0/false/no/off retains TILE8/TILE4, while empty or any other value requests direct-Q4 WMMA for eligible standalone projections and attention-output A but leaves attention-output B on Q8_K+TILE8; eligibility additionally requires each complete projection weight range in physical device storage rather than mapped/registered host memory; DISABLE wins | Allow the compressed direct-Q4 WMMA kernel to consume an already device-resident/cache-backed Q4_K projection during SSD streaming without changing model I/O. | [rocm/ds4_rocm_q4.cuh:1356](rocm/ds4_rocm_q4.cuh#L1356) |
+| `DS4_ROCM_ENABLE_STREAMING_FULL_EXPERT_ADDR_TABLE` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming full expert addr table. | [ds4.c:18255](ds4.c#L18255) |
+| `DS4_ROCM_ENABLE_STREAMING_MADVISE_WILLNEED` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming madvise willneed. | [ds4.c:18224](ds4.c#L18224) |
+| `DS4_ROCM_ENABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming prefill batch selected addr. | [ds4.c:18584](ds4.c#L18584) |
+| `DS4_ROCM_ENABLE_STREAMING_PREFILL_CACHE_SEED` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming prefill cache seed. | [ds4.c:21273](ds4.c#L21273) |
+| `DS4_ROCM_ENABLE_STREAMING_PREFILL_LAYER_PAGEIN` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming prefill layer pagein. | [ds4.c:18446](ds4.c#L18446) |
+| `DS4_ROCM_ENABLE_STREAMING_PREFILL_LAYER_READAHEAD` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming prefill layer readahead. | [ds4.c:18456](ds4.c#L18456) |
+| `DS4_ROCM_ENABLE_STREAMING_PREFILL_SELECTED_MADVISE` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming prefill selected madvise. | [ds4.c:18436](ds4.c#L18436) |
+| `DS4_ROCM_ENABLE_STREAMING_PREFILL_SELECTED_PAGEIN` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming prefill selected pagein. | [ds4.c:18426](ds4.c#L18426) |
+| `DS4_ROCM_ENABLE_STREAMING_PREFILL_SELECTED_READAHEAD` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming prefill selected readahead. | [ds4.c:19970](ds4.c#L19970) |
+| `DS4_ROCM_ENABLE_STREAMING_PREFILL_SELECTED_READAHEAD_SHARED` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming prefill selected readahead shared. | [ds4.c:19972](ds4.c#L19972) |
+| `DS4_ROCM_ENABLE_STREAMING_READAHEAD` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming readahead. | [ds4.c:18217](ds4.c#L18217) |
+| `DS4_ROCM_ENABLE_STREAMING_STATIC_DECODE_MAP` | presence opt-in flag; unset=off unless paired policy is automatic | Enable rocm enable streaming static decode map. | [ds4.c:18236](ds4.c#L18236) |
+| `DS4_ROCM_GLM_CAUSAL_ATTN_GEMM` | Enabled by default when unset. Exact "0" or an empty value disables; every other nonempty value enables (including false/off/no), because cuda_env_present only tests nonempty and != "0". Eligibility still requires causal_range && !has_selected; a failed GEMM helper falls through to the scalar attention kernel. | Use FP16 BLAS GEMMs for dense causal GLM indexed prefill; =0 is the correctness/performance rollback to the scalar attention kernel. | [rocm/ds4_rocm_glm.cuh:3283](rocm/ds4_rocm_glm.cuh#L3283) |
+| `DS4_ROCM_GLM_DISABLE_STREAMING_EXPERT_CACHE` | Pure presence flag: any defined value, including empty or "0", disables. Unset leaves automatic GLM streaming expert-cache eligibility enabled for supported model/quant/quality/SSD configurations. On ROCm builds DS4_METAL_GLM_DISABLE_STREAMING_EXPERT_CACHE is an accepted fallback alias. | Disable selected/resident streamed-expert cache paths and force generic/full-layer expert handling for GLM SSD streaming. | [ds4.c:21197](ds4.c#L21197) |
+| `DS4_ROCM_GLM_DISABLE_STREAMING_SEED_BEFORE_PREFILL` | Pure presence flag: any defined value, including empty or "0", disables. Unset seeds before prefill whenever SSD streaming is active. On ROCm builds DS4_METAL_GLM_DISABLE_STREAMING_SEED_BEFORE_PREFILL is an accepted fallback alias. | Skip the pre-prefill hotlist seed of the streaming expert cache in both one-shot GLM generation and session setup. | [ds4.c:51073](ds4.c#L51073) |
+| `DS4_ROCM_GLM_DISABLE_STREAMING_TOKEN_PREFILL` | Pure presence flag: any defined value, including empty or "0", disables. Unset leaves the token-major path eligible only for SSD streaming, non-quality mode, a nonempty batch fitting full attention, and n_tokens <= the configured nonzero maximum. DS4_METAL_GLM_DISABLE_STREAMING_TOKEN_PREFILL and generic DS4_GLM_DISABLE_STREAMING_TOKEN_PREFILL are also accepted presence aliases. | Roll back GLM SSD-streaming token-major prefill to the normal prefill implementation. | [ds4.c:49703](ds4.c#L49703) |
+| `DS4_ROCM_GLM_GROUPED_QK_LOW` | sampled once; unset: enabled; present empty or exact 0: disabled; every other present value: enabled | Selects the grouped shared-input ROCm kernel for eligible multi-token GLM qk-lowrank projection; disabling uses the per-head/per-token projection kernel. | [rocm/ds4_rocm_runtime.cuh:4800](rocm/ds4_rocm_runtime.cuh#L4800) |
+| `DS4_ROCM_GLM_GROUPED_VALUE_PROJECT` | sampled once; unset: enabled; present empty or exact 0: disabled; every other present value: enabled | Selects the grouped shared-input ROCm kernel for eligible multi-token GLM value projection; disabling uses the non-grouped batch projection path. | [rocm/ds4_rocm_runtime.cuh:4791](rocm/ds4_rocm_runtime.cuh#L4791) |
+| `DS4_ROCM_GLM_LAYER_SLICE_TOKEN_DECODE` | Opt-in truthy parser; unset, empty, "0", false, off, or no (case-insensitive words) are false, every other nonempty value is true. Default off and compiled only for ROCm. | Allow a one-token, pos>0 GLM layer-slice with inter-node input/output hidden buffers to use the optimized resident token graph; without it only the no-hidden-buffer case takes that shortcut. | [ds4.c:43573](ds4.c#L43573) |
+| `DS4_ROCM_GLM_SELECTED_ATTN_GEMM` | Enabled by default when unset. Exact "0" or an empty value disables; every other nonempty value enables (including false/off/no). Eligibility still requires !causal_range && has_selected; failure/ineligibility falls through to the scalar attention kernel. | Gather per-token selected cache rows into FP16 matrices and use strided-batched BLAS GEMMs for GLM selected indexed prefill; =0 forces the scalar path. | [rocm/ds4_rocm_glm.cuh:3239](rocm/ds4_rocm_glm.cuh#L3239) |
+| `DS4_ROCM_GLM_SELECTED_ATTN_HEAD_TILE` | Unsigned integer read and cached once; valid values are exactly 1,2,4,8,16,32,64. Unset/empty defaults to 16. A nonnumeric, partially parsed, overflowed, or unsupported value prints a warning and uses 16; the effective tile is min(requested,n_head). | Set how many attention heads each selected-attention GEMM workspace tile processes. | [rocm/ds4_rocm_glm.cuh:2509](rocm/ds4_rocm_glm.cuh#L2509) |
+| `DS4_ROCM_GLM_SELECTED_ATTN_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm glm selected attn profile. | [rocm/ds4_rocm_glm.cuh:2534](rocm/ds4_rocm_glm.cuh#L2534) |
+| `DS4_ROCM_GLM_STREAMING_ASYNC_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm glm streaming async profile. | [ds4.c:44361](ds4.c#L44361) |
+| `DS4_ROCM_GLM_STREAMING_DECODE_FULL_LAYER_MAP` | Pure presence flag: any defined value, including empty or "0", forces full mapping. Unset uses automatic mapping: resident layers map fully, eligible expert-cache layers use decode-only/expert mapping, otherwise full mapping. DS4_METAL_GLM_STREAMING_DECODE_FULL_LAYER_MAP and generic DS4_GLM_STREAMING_DECODE_FULL_LAYER_MAP are also accepted presence aliases. | Force every GLM SSD-streaming decode layer through full-layer mapping, bypassing the selected-expert/decode mapping optimization. | [ds4.c:42615](ds4.c#L42615) |
+| `DS4_ROCM_GLM_STREAMING_DECODE_SYNC_EACH_LAYER` | ROCm-only primary value; nonempty takes priority over DS4_METAL_GLM_STREAMING_DECODE_SYNC_EACH_LAYER and the generic DS4_GLM_STREAMING_DECODE_SYNC_EACH_LAYER fallback; empty acts as unset; truthy unless exact 0 or case-insensitive false/off/no; with all aliases unset: false | For non-static GLM SSD decode on ROCm, opts into a full command/device synchronization after token mapping and every layer; default keeps ordered work queued across layer mappings, and static-map decode bypasses it. | [ds4.c:49779](ds4.c#L49779) |
+| `DS4_ROCM_GLM_STREAMING_GROW_CACHE_AFTER_PREFILL` | Enabled by default when absent. If defined, only a truthy nonempty value enables; empty, "0", false, off, or no disable. Growth also requires SSD streaming plus nonzero base cache and prefill-headroom budgets, and occurs only if the recomputed expert count exceeds the current count. | After successful ROCm GLM prefill, add the released prefill headroom to the dynamic streaming expert-cache byte budget. | [ds4.c:51111](ds4.c#L51111) |
+| `DS4_ROCM_GLM_STREAMING_PREFILL_FULL_LAYER` | presence force-on; any presence including empty or 0 enables; unset falls back to the Metal alias and then the automatic token threshold (1024 by default on ROCm); DS4_ROCM_DISABLE_GLM_STREAMING_PREFILL_FULL_LAYER dominates | Forces GLM SSD prefill into full-layer mapping/cache mode even below the automatic large-batch threshold. | [ds4.c:42692](ds4.c#L42692) |
+| `DS4_ROCM_GLM_STREAMING_PREFILL_FULL_LAYER_MIN_TOKENS` | Positive uint32 threshold parsed with strtoul; ROCm default is 1024. Missing/empty, no leading number, errno/overflow, zero, or >UINT32_MAX returns 1024. The parser does not require end-of-string, so trailing junk after a valid leading number is accepted. A nonempty ROCm value takes precedence; otherwise DS4_METAL_GLM_STREAMING_PREFILL_FULL_LAYER_MIN_TOKENS is a fallback alias. | Set the automatic token-count crossover for ROCm GLM SSD prefill to load/use full resident expert layers when the layer supports that mode. | [ds4.c:42692](ds4.c#L42692) |
+| `DS4_ROCM_GLM_STREAMING_PREFILL_SYNC_EACH_LAYER` | ROCm-only primary value; nonempty takes priority over DS4_METAL_GLM_STREAMING_PREFILL_SYNC_EACH_LAYER and the generic DS4_GLM_STREAMING_PREFILL_SYNC_EACH_LAYER fallback; empty acts as unset; truthy unless exact 0 or case-insensitive false/off/no; with all aliases unset: false for compact prefill; full-layer prefill always returns true | For compact GLM SSD prefill on ROCm, opts into a full command/device synchronization at every layer boundary; default preserves queued work across mappings, while full-layer cache mode always synchronizes. | [ds4.c:42395](ds4.c#L42395) |
+| `DS4_ROCM_GLM_STREAMING_TOKEN_PREFILL_MAX` | primary nonempty value, then the Metal alias, then generic DS4_GLM_STREAMING_TOKEN_PREFILL_MAX; parsed by strtoul without requiring full-string consumption; 0 is valid and disables; no digits, ERANGE, or > UINT32_MAX uses the ROCm default 0 | Sets the largest non-quality GLM SSD-prefill chunk eligible for token-major/decode-style execution; ROCm defaults to canonical indexed batch prefill (0 disables token-major mode). | [ds4.c:49682](ds4.c#L49682) |
+| `DS4_ROCM_GLM_VALUE_PROJECT_WAVE_DECODE` | Enabled by default when unset. Exact "0" or empty disables; every other nonempty value enables (including false/off/no). It applies only when n_tokens==1; =0 or multi-token input uses the generic per-head batch kernel. | Select the validated wave-per-output-row ROCm Q8 GLM value-projection kernel for one-token decode; =0 is the generic-kernel rollback. | [rocm/ds4_rocm_glm.cuh:2019](rocm/ds4_rocm_glm.cuh#L2019) |
+| `DS4_ROCM_GRAPH_DUMP_LAYER` | unsigned layer or all; unset=all layers | Filter ROCm graph dumps by layer. | [ds4.c:16862](ds4.c#L16862) |
+| `DS4_ROCM_GRAPH_DUMP_NAME` | nonempty substring filter; unset=all tensor names | Filter ROCm graph dumps by tensor/stage name. | [ds4.c:16858](ds4.c#L16858) |
+| `DS4_ROCM_GRAPH_DUMP_NONINVASIVE` | truthy value under the shared parser; unset lets dumping select conservative kernels | Keep production ROCm kernel selection while graph dumping. | [rocm/ds4_rocm_runtime.cuh:4822](rocm/ds4_rocm_runtime.cuh#L4822) |
+| `DS4_ROCM_GRAPH_DUMP_POS` | unsigned token position; unset=all positions | Filter ROCm graph dumps by position. | [ds4.c:16869](ds4.c#L16869) |
+| `DS4_ROCM_GRAPH_DUMP_PREFIX` | nonempty output path prefix; unset=off | Enable ROCm intermediate graph/tensor dumps. | [ds4_cuda.cu:397](ds4_cuda.cu#L397) |
+| `DS4_ROCM_GRAPH_DUMP_TRACE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Emit trace diagnostics for rocm graph dump trace. | [ds4.c:16899](ds4.c#L16899) |
+| `DS4_ROCM_GRAPH_OUTPUT_ROW` | Nonempty string with a leading strtoul-parsable unsigned value < n_tokens selects that zero-based row. Default, empty, unparsable, or out-of-range selects n_tokens-1. Trailing characters are accepted because full consumption/errno are not checked. A nonempty ROCm value takes precedence; otherwise DS4_METAL_GRAPH_OUTPUT_ROW is a fallback alias. | Choose which prefill hidden-state row is sent through the output head to produce logits, primarily for graph/correctness diagnostics. | [ds4.c:35845](ds4.c#L35845) |
+| `DS4_ROCM_GRAPH_PREFILL_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm graph prefill profile. | [ds4.c:32038](ds4.c#L32038) |
+| `DS4_ROCM_GRAPH_PREFILL_SPLIT_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm graph prefill split profile. | [ds4.c:35768](ds4.c#L35768) |
+| `DS4_ROCM_GRAPH_TOKEN_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm graph token profile. | [ds4.c:31722](ds4.c#L31722) |
+| `DS4_ROCM_INDEXER_STAGE_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm indexer stage profile. | [ds4.c:29265](ds4.c#L29265) |
+| `DS4_ROCM_LAYER_STAGE_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm layer stage profile. | [ds4.c:29104](ds4.c#L29104) |
+| `DS4_ROCM_LAYER_STAGE_PROFILE_LAYER` | layer filter subordinate to DS4_ROCM_LAYER_STAGE_PROFILE; unset or whitespace-only: all layers allowed by the parent flag; otherwise the whitespace-trimmed value must be a complete base-10 strtoul result <= UINT32_MAX equal to the current layer; invalid values match none | Restricts the ROCm layer/prefill stage profiler to one layer; it does not enable profiling by itself. | [ds4.c:29105](ds4.c#L29105) |
+| `DS4_ROCM_MOE_DECODE_DOWN_RPB` | sampled once; nonempty value is parsed by strtoul (a numeric prefix is sufficient), cast to uint32_t, and accepted only if 1/2/4/8/16/32; unset/empty/invalid inherits DS4_ROCM_MOE_DECODE_RPB, with defaults quality=8, non-quality SSD=2, resident=1 | Sets output rows (warps) per block for ROCm Q2_K routed-MoE decode down-projection kernels; threads per block are value * 32. | [rocm/ds4_rocm_runtime.cuh:4842](rocm/ds4_rocm_runtime.cuh#L4842) |
+| `DS4_ROCM_MOE_DECODE_GATE_RPB` | sampled once; nonempty value is parsed by strtoul (a numeric prefix is sufficient), cast to uint32_t, and accepted only if 1/2/4/8/16/32; unset/empty/invalid defaults to 1 in non-quality SSD mode when DS4_ROCM_MOE_DECODE_RPB is unset/empty, otherwise inherits the resolved base RPB | Sets output rows (warps) per block for ROCm Q2_K routed-MoE decode gate/up kernels; threads per block are value * 32. | [rocm/ds4_rocm_runtime.cuh:4836](rocm/ds4_rocm_runtime.cuh#L4836) |
+| `DS4_ROCM_MOE_DECODE_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm moe decode profile. | [rocm/ds4_rocm_moe_launch.cuh:82](rocm/ds4_rocm_moe_launch.cuh#L82) |
+| `DS4_ROCM_MOE_DECODE_RPB` | sampled once; nonempty value is parsed by strtoul (a numeric prefix is sufficient), cast to uint32_t, and accepted only if 1/2/4/8/16/32; unset/empty/invalid default: quality=8, non-quality SSD=2, resident=1 | Sets the base ROCm Q2_K decode-MoE rows-per-block value inherited by gate/up and down controls, except the automatic SSD gate specialization defaults to 1 when this variable is unset/empty. | [rocm/ds4_rocm_runtime.cuh:4832](rocm/ds4_rocm_runtime.cuh#L4832) |
+| `DS4_ROCM_MOE_PATH_DEBUG` | presence diagnostic; unset=off; any defined value including empty or 0 enables it | Print ROCm routed-MoE path selection, sorted-tile scratch state, and MXFP4 gate/up launch diagnostics to stderr. | [rocm/ds4_rocm_moe_launch.cuh:831](rocm/ds4_rocm_moe_launch.cuh#L831) |
+| `DS4_ROCM_MOE_WRITE_CLAMPED_ACT` | Pure presence sentinel: any defined value, including empty or "0", is active; DS4_METAL_MOE_WRITE_CLAMPED_ACT is an accepted fallback alias. On ROCm the variable is only consumed as a path-admission veto: it disables selected-expert cache/address-table, selected-slot and CPU-router/fused optimized paths. No ROCm call site parses a clamp amount or directly enables a write-clamped kernel. | Force shared graph selection away from optimizations incompatible with the clamped-intermediate MoE diagnostic; on ROCm this is a compatibility/rollback gate, not itself a clamped-write implementation. | [ds4.c:18526](ds4.c#L18526) |
+| `DS4_ROCM_MXFP4_DOWN_RGROUP` | nonempty value is parsed by strtol and a numeric prefix is sufficient; integers 1..8 are accepted; unset, empty, invalid, or out-of-range values use 1 | Set how many 32-row output blocks each ROCm MXFP4 tiled down-projection block computes, reducing the first launch-grid dimension as the value increases. | [rocm/ds4_rocm_moe_launch.cuh:801](rocm/ds4_rocm_moe_launch.cuh#L801) |
+| `DS4_ROCM_Q4_GROUPED_ATTN_A_STATS` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Print counters for rocm q4 grouped attn a stats. | [rocm/ds4_rocm_q4.cuh:614](rocm/ds4_rocm_q4.cuh#L614) |
+| `DS4_ROCM_Q4_PREFILL_TILE8_STATS` | presence diagnostic; unset=off; any defined value including empty or 0 prints at exit | Print tiled-prefill dense/pair/attention counters, total and SSD-specific K=1024 TILE4 dispatches, and direct-WMMA total/K32/K64/K128 launch counts. | [rocm/ds4_rocm_q4.cuh:1454](rocm/ds4_rocm_q4.cuh#L1454) |
+| `DS4_ROCM_Q4_PREFILL_WMMA_ROW_TILE` | unsigned integer; unset, empty, malformed, negative, or values other than 64/128/256 use shape selection (64 rows when M<1024, 128 when M<8192, otherwise 256); 64 retains the previous geometry | Override the number of output rows sharing each direct-Q4 64x32 activation tile for controlled 64/128/256-row ROCm WMMA A/B measurements. | [rocm/ds4_rocm_q4.cuh:1517](rocm/ds4_rocm_q4.cuh#L1517) |
+| `DS4_ROCM_Q8_DECODE_SHAREDX_64K` | sampled once; unset: enabled; present empty or exact 0: disabled; every other present value: enabled; effective only for one-token non-prequant Q8_0 matmul with 8192 < in_dim <= 16384 | Allows the ROCm shared-input Q8 decode kernel to use up to 64 KiB dynamic LDS for wide inputs; an unsupported/failed LDS launch automatically falls back to the regular kernel. | [rocm/ds4_rocm_runtime.cuh:4805](rocm/ds4_rocm_runtime.cuh#L4805) |
+| `DS4_ROCM_Q_STAGE_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm q stage profile. | [ds4.c:29284](ds4.c#L29284) |
+| `DS4_ROCM_REQUIRE_Q4_GROUPED_ATTN_A` | presence fail-closed assertion; also requests the candidate outside the caller-marked resident decode default; DISABLE remains authoritative and causes failure | Require grouped Q4 attention-A and fail instead of silently falling back. | [rocm/ds4_rocm_q4.cuh:870](rocm/ds4_rocm_q4.cuh#L870) |
+| `DS4_ROCM_REQUIRE_Q4_PREFILL_TILE8` | presence fail-closed assertion for eligible TILE8 calls | Require rocm require q4 prefill tile8 and fail instead of silently falling back. | [rocm/ds4_rocm_q4.cuh:452](rocm/ds4_rocm_q4.cuh#L452) |
+| `DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA` | value-aware strict diagnostic assertion, not required for the automatic standalone/attention-output-A resident default; unset/0/false/no/off is off; empty or any other value requires every selected Q4 dense or attention-output projection to use direct-Q4 WMMA and is the only control that selects the numerically compounded attention-output B stage; unsupported shape/device, quality mode, DISABLE, or an SSD weight range without physical device residency fails before dispatch | Prevent a strict ROCm Q4 prefill WMMA kernel oracle from silently timing TILE8/TILE4; normal runtime controls keep attention-output B on Q8_K+TILE8. | [rocm/ds4_rocm_q4.cuh:1360](rocm/ds4_rocm_q4.cuh#L1360) |
+| `DS4_ROCM_STREAMING_DECODE_PREFILL_MAX` | primary nonempty value over the Metal alias; parsed by strtol when it has a numeric prefix (trailing text is accepted); <= 0 disables, values > UINT32_MAX clamp, no numeric prefix uses automatic default: 64 for Flash with uniform Q4_K/MXFP4 experts, 18 for other Pro/Flash, otherwise 0; the disable flag dominates | Sets the largest short, non-quality SSD-streaming prefill batch routed through the decode-style path instead of canonical layer-major prefill. | [ds4.c:31976](ds4.c#L31976) |
+| `DS4_ROCM_STREAMING_EXPERT_AUTO_PRELOAD_CAP` | primary nonempty value over the Metal alias; strict full-string strtoul; valid values > UINT32_MAX clamp, invalid uses 4096, and 0 means no cap (not disabled); when CLI preload is auto/0, unset defaults to cap 4096 except ROCm GLM52, where absent/empty disables automatic preload entirely | Caps the number of hot experts synchronously seeded into the SSD-streaming expert cache in automatic preload mode; an explicit CLI preload count bypasses this cap, and setting this variable opts ROCm GLM52 back into auto preload. | [ds4.c:21469](ds4.c#L21469) |
+| `DS4_ROCM_STREAMING_EXPERT_CACHE_VERBOSE` | presence flag; unset=off | Print verbose ROCm streaming expert-cache seed/load diagnostics. | [rocm/ds4_rocm_runtime.cuh:2904](rocm/ds4_rocm_runtime.cuh#L2904) |
+| `DS4_ROCM_STREAMING_EXPERT_HOTLIST` | Nonempty filesystem path; a nonempty ROCm value takes precedence, otherwise DS4_METAL_STREAMING_EXPERT_HOTLIST is a fallback. The file contains whitespace-separated layer expert hits rows; blank/comment lines are ignored, zero-hit rows skipped, malformed/open/read errors fail seeding. Unset/empty uses the built-in Pro/Flash/GLM52 hotlist. Effective only when non-cold SSD hotlist seeding is enabled and cache/preload budget is nonzero. | Select a custom ranked expert hotlist used to preseed the streaming resident expert cache before decode. | [ds4.c:21494](ds4.c#L21494) |
+| `DS4_ROCM_STREAMING_EXPERT_HOTLIST_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm streaming expert hotlist profile. | [ds4.c:32244](ds4.c#L32244) |
+| `DS4_ROCM_STREAMING_MAP_TRACE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Emit trace diagnostics for rocm streaming map trace. | [ds4.c:42643](ds4.c#L42643) |
+| `DS4_ROCM_STREAMING_PREFILL_BATCH_SELECTED_ADDR_MAX` | primary nonempty value over the Metal alias; strtol accepts a numeric prefix; <= 0 returns 0, > UINT32_MAX clamps, invalid uses ROCm default UINT32_MAX for Pro/Flash/GLM52 and 0 otherwise | Sets the inclusive upper token-count bound for automatically using selected-expert address-table kernels during eligible non-quality SSD batch prefill; 0 disables automatic selection. | [ds4.c:18471](ds4.c#L18471) |
+| `DS4_ROCM_STREAMING_PREFILL_BATCH_SELECTED_ADDR_MIN` | primary nonempty value over the Metal alias; strtol accepts a numeric prefix; <= 0 returns 0, > UINT32_MAX clamps, invalid uses ROCm default 2 for Pro/Flash/GLM52 and 0 otherwise | Sets the inclusive lower token-count bound for automatically using selected-expert address-table kernels during eligible non-quality SSD batch prefill (the path independently requires more than one token). | [ds4.c:18494](ds4.c#L18494) |
+| `DS4_ROCM_STREAMING_PREFILL_CACHE_SEED_K` | primary nonempty value over the Metal alias; strict full-string strtoul; unset/empty/invalid: 1; 0 disables; positive values clamp to 64; ignored unless SSD streaming and DS4_ROCM_ENABLE_STREAMING_PREFILL_CACHE_SEED (or Metal alias) is present | Chooses how many trailing token router selections per layer are captured from prefill and used to seed the streaming expert cache afterward. | [ds4.c:21267](ds4.c#L21267) |
+| `DS4_ROCM_STREAMING_PREFILL_CACHE_SEED_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm streaming prefill cache seed profile. | [ds4.c:32153](ds4.c#L32153) |
+| `DS4_ROCM_STREAMING_PREFILL_LAYER_MADVISE_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm streaming prefill layer madvise profile. | [ds4.c:19609](ds4.c#L19609) |
+| `DS4_ROCM_STREAMING_PREFILL_LAYER_PAGEIN_NO_OVERLAP` | Pure presence flag: any defined value, including empty or "0", disables overlap. Default overlap is enabled only if this, PREPARE_NO_OVERLAP, DISABLE_*_PREPARE_OVERLAP, and DISABLE_*_PAGEIN_OVERLAP are all absent. The corresponding DS4_METAL name is an accepted fallback alias. In current code PAGEIN_NO_OVERLAP and PREPARE_NO_OVERLAP are exact synonyms. | Serialize SSD-streaming prefill layer page-in/preparation instead of overlapping preparation of upcoming layers. | [ds4.c:19316](ds4.c#L19316) |
+| `DS4_ROCM_STREAMING_PREFILL_LAYER_PAGEIN_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm streaming prefill layer pagein profile. | [ds4.c:19605](ds4.c#L19605) |
+| `DS4_ROCM_STREAMING_PREFILL_LAYER_PAGEIN_THREADS` | legacy fallback read only when DS4_ROCM_STREAMING_PREFILL_LAYER_PREPARE_THREADS and its Metal alias are absent/empty; strict full-string strtoul; unset/empty across both names: 8; invalid or 0: 1; values > 16 clamp to 16 | Sets worker count for full-layer SSD-prefill preparation (page touch, pread, readahead, or madvise) when the canonical PREPARE_THREADS control is not set. | [ds4.c:19274](ds4.c#L19274) |
+| `DS4_ROCM_STREAMING_PREFILL_LAYER_PREAD_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm streaming prefill layer pread profile. | [ds4.c:19607](ds4.c#L19607) |
+| `DS4_ROCM_STREAMING_PREFILL_LAYER_PREPARE_AHEAD` | primary nonempty value over the Metal alias; strict full-string strtoul; unset/empty: 1; invalid or 0: 1; values > 4 clamp to 4 | Sets how many future layer-preparation jobs may be queued concurrently while SSD-prefill preparation overlap is enabled. | [ds4.c:19328](ds4.c#L19328) |
+| `DS4_ROCM_STREAMING_PREFILL_LAYER_PREPARE_NO_OVERLAP` | Pure presence flag: any defined value, including empty or "0", disables overlap. Default overlap is enabled only if this, PAGEIN_NO_OVERLAP, DISABLE_*_PREPARE_OVERLAP, and DISABLE_*_PAGEIN_OVERLAP are all absent. The corresponding DS4_METAL name is an accepted fallback alias. In current code PREPARE_NO_OVERLAP and PAGEIN_NO_OVERLAP are exact synonyms. | Serialize SSD-streaming prefill layer preparation/page-in instead of overlapping preparation of upcoming layers. | [ds4.c:19314](ds4.c#L19314) |
+| `DS4_ROCM_STREAMING_PREFILL_LAYER_PREPARE_THREADS` | primary nonempty value over the Metal alias; strict full-string strtoul; unset/empty falls back to LAYER_PAGEIN_THREADS, then default 8; invalid or 0: 1; values > 16 clamp to 16 | Sets worker count used to split full-layer SSD-prefill page-touch, pread, readahead, or madvise preparation ranges. | [ds4.c:19270](ds4.c#L19270) |
+| `DS4_ROCM_STREAMING_PREFILL_LAYER_READAHEAD_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm streaming prefill layer readahead profile. | [ds4.c:19611](ds4.c#L19611) |
+| `DS4_ROCM_STREAMING_PREFILL_SELECTED_MADVISE_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm streaming prefill selected madvise profile. | [ds4.c:19355](ds4.c#L19355) |
+| `DS4_ROCM_STREAMING_PREFILL_SELECTED_MADVISE_THREADS` | legacy fallback read only for selected-expert madvise preparation when DS4_ROCM_STREAMING_PREFILL_SELECTED_PREPARE_THREADS and its Metal alias are absent/empty; strict full-string strtoul; if all selected controls are unset it inherits layer preparation threads (default 8); invalid or 0: 1; values > 16 clamp to 16 | Sets worker count for selected-expert madvise preparation under its legacy name; non-madvise selected page-in always uses one worker. | [ds4.c:19292](ds4.c#L19292) |
+| `DS4_ROCM_STREAMING_PREFILL_SELECTED_PAGEIN_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm streaming prefill selected pagein profile. | [ds4.c:19353](ds4.c#L19353) |
+| `DS4_ROCM_STREAMING_PREFILL_SELECTED_PREPARE_GAP` | primary nonempty value over the Metal alias; strict full-string strtoul; unset/empty/invalid: 0; values > 8 clamp to 8 | For selected-expert madvise preparation, merges selected expert runs separated by at most this many unselected expert IDs, trading broader hints for fewer ranges. | [ds4.c:19304](ds4.c#L19304) |
+| `DS4_ROCM_STREAMING_PREFILL_SELECTED_PREPARE_THREADS` | primary nonempty value over the Metal alias; strict full-string strtoul; for selected-expert madvise, unset/empty falls back to SELECTED_MADVISE_THREADS then layer preparation threads (default 8); invalid or 0: 1; values > 16 clamp to 16; non-madvise selected page-in ignores it and uses 1 | Sets worker count for selected-expert madvise preparation using the canonical control name. | [ds4.c:19288](ds4.c#L19288) |
+| `DS4_ROCM_STREAMING_PREFILL_SELECTED_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm streaming prefill selected profile. | [ds4.c:18905](ds4.c#L18905) |
+| `DS4_ROCM_STREAMING_PREFILL_SELECTED_READAHEAD_GAP` | primary nonempty value over the Metal alias; strict full-string strtoul; unset/empty/invalid: 0; values > 8 clamp to 8 | For selected-expert file readahead, merges selected expert runs separated by at most this many unselected expert IDs, reducing readahead calls at the cost of hinting extra weights. | [ds4.c:19977](ds4.c#L19977) |
+| `DS4_ROCM_STREAMING_PREFILL_SELECTED_READAHEAD_PROFILE` | presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose) | Collect timing/profile diagnostics for rocm streaming prefill selected readahead profile. | [ds4.c:20062](ds4.c#L20062) |
+| `DS4_ROCM_STREAM_CACHE_LAYER_STATS` | presence flag; unset=off | Collect per-layer ROCm streaming cache statistics; also enables aggregate stats. | [rocm/ds4_rocm_runtime.cuh:390](rocm/ds4_rocm_runtime.cuh#L390) |
+| `DS4_ROCM_STREAM_CACHE_STATS` | presence flag; unset=off unless layer stats are enabled | Collect aggregate ROCm streaming cache statistics. | [rocm/ds4_rocm_runtime.cuh:398](rocm/ds4_rocm_runtime.cuh#L398) |
+| `DS4_ROCM_STREAM_EVICT_PAST_LAYERS_FIRST` | nonempty and not 0 enables; unset/empty/0=off | Prefer evicting cached experts from already-processed layers. | [rocm/ds4_rocm_runtime.cuh:406](rocm/ds4_rocm_runtime.cuh#L406) |
+| `DS4_ROCM_STREAM_FREE_RESERVE_GB` | integer 2..64 GiB; default 16 | Reserve unified-memory headroom while growing the ROCm expert cache. | [rocm/ds4_rocm_runtime.cuh:1526](rocm/ds4_rocm_runtime.cuh#L1526) |
+| `DS4_ROCM_STREAM_MODEL_CACHE_GB` | positive GiB integer; unset/invalid uses automatic streaming model cache limit | Cap cached streaming model spans. | [rocm/ds4_rocm_runtime.cuh:5464](rocm/ds4_rocm_runtime.cuh#L5464) |
+| `DS4_ROCM_STREAM_NO_DIRECT` | nonempty and not 0 disables direct reads; unset/empty/0 keeps direct I/O eligible | Force the buffered ROCm SSD-streaming read path. | [rocm/ds4_rocm_runtime.cuh:1932](rocm/ds4_rocm_runtime.cuh#L1932) |
+| `DS4_ROCM_STREAM_Q8_F16_CACHE_GB` | non-negative GiB integer; unset/invalid uses automatic Q8-F16 cache limit | Cap converted Q8-to-F16 weights in SSD mode. | [rocm/ds4_rocm_runtime.cuh:4859](rocm/ds4_rocm_runtime.cuh#L4859) |
+| `DS4_ROCM_STREAM_READ_PROFILE` | nonempty and not 0 enables; unset/empty/0=off | Print ROCm SSD-streaming read/locality statistics at exit. | [rocm/ds4_rocm_runtime.cuh:1918](rocm/ds4_rocm_runtime.cuh#L1918) |
+| `DS4_ROCM_STREAM_READ_WORKERS` | integer; default DS4_ROCM_STREAM_READ_DEFAULT_WORKERS; 0 coerces to 1; capped at compile-time max | Set parallel ROCm SSD read/upload workers. | [rocm/ds4_rocm_runtime.cuh:2077](rocm/ds4_rocm_runtime.cuh#L2077) |
+
+</details>
+
+<details>
+<summary><strong>GLM shared (41)</strong></summary>
+
+| Variable | Accepted value and default | Effect | Source |
+| --- | --- | --- | --- |
+| `DS4_GLM_ABLATE_COMBINE` | presence ablation; unset: exchange the local TP partial with the peer and add both halves; any presence including empty or 0 skips the exchange | Metal two-rank TP timing probe: doubles the local routed-MoE or split-attention partial instead of combining with the peer, deliberately producing invalid output; both ranks must set it or their exchange gates desynchronize. | [ds4.c:44181](ds4.c#L44181) |
+| `DS4_GLM_ATTN_NO_LORA_VEC2` | presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it | Disable vectorized two-row LoRA accumulation in CUDA GLM indexed attention. | [ds4_cuda.cu:34176](ds4_cuda.cu#L34176) |
+| `DS4_GLM_ATTN_NO_SCORE_VEC2` | presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it | Disable vectorized two-row score computation in CUDA GLM indexed attention. | [ds4_cuda.cu:34165](ds4_cuda.cu#L34165) |
+| `DS4_GLM_ATTN_NO_STAGED_DECODE` | presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it | Disable staged CUDA GLM indexed-decode attention for large selected sets. | [ds4_cuda.cu:34214](ds4_cuda.cu#L34214) |
+| `DS4_GLM_DECODE_ABLATE` | cached substring list; default empty mask; recognized tokens are attn_out, attn_core, qpath, indexer, routed, shared, qklow; unknown text has no effect; matching stages are skipped and output is invalid | Skip selected GLM decode stages for timing attribution; generated output is invalid. | [ds4.c:44411](ds4.c#L44411) |
+| `DS4_GLM_DECODE_FLUSH_INTERVAL` | integer layers via atoi; default 4 for indexed decode and 32 otherwise; <=0/nonnumeric disables periodic flush; capped to layer count and forced to 0 for deferred completion | Set how often non-streaming GLM decode command work is flushed between layers. | [ds4.c:49818](ds4.c#L49818) |
+| `DS4_GLM_DISABLE_FLASH_PREFILL` | presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it | Disable GLM Flash Attention prefill. | [ds4.c:45087](ds4.c#L45087) |
+| `DS4_GLM_DISABLE_STREAMING_TOKEN_PREFILL` | presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it; either backend-specific ROCm/Metal alias also disables | Disable token-major GLM SSD-streaming prefill. | [ds4.c:49702](ds4.c#L49702) |
+| `DS4_GLM_FENCE_TRACE` | presence flag; unset is off; any defined value, including empty or 0, is on; normal eligibility still applies | Log fenced CUDA tier switches used by GLM multi-GPU execution. | [ds4_cuda.cu:7984](ds4_cuda.cu#L7984) |
+| `DS4_GLM_GEMM_TRACE` | presence flag; unset is off; any defined value, including empty or 0, is on; normal eligibility still applies | Measure and print CUDA GLM dequantization and cuBLAS GEMM timings. | [ds4_cuda.cu:19706](ds4_cuda.cu#L19706) |
+| `DS4_GLM_HIDDEN_DUMP` | nonempty filesystem path/prefix; unset/empty disables; writes final hidden row or per-layer files selected by DS4_GLM_HIDDEN_DUMP_LAYER | Dump GLM hidden-state rows for correctness isolation. | [ds4.c:38523](ds4.c#L38523) |
+| `DS4_GLM_HIDDEN_DUMP_LAYER` | selector; unset/empty = -1 (no per-layer dump, final hidden still dumped when path set); all = every layer; otherwise atoi result selects a layer, so invalid text selects layer 0 | Choose which GLM layer hidden states are dumped. | [ds4.c:38543](ds4.c#L38543) |
+| `DS4_GLM_KV_DUMP` | nonempty filesystem prefix; unset/empty disables; writes layer-0 lora and rope compact-cache files after sync | Dump layer-0 compact GLM KV cache data after prompt synchronization. | [ds4.c:63131](ds4.c#L63131) |
+| `DS4_GLM_LOGIT_DUMP` | nonempty filesystem path; unset/empty disables; dumps the first post-prefill logits vector once per process | Dump the first post-prefill GLM logits vector. | [ds4.c:38601](ds4.c#L38601) |
+| `DS4_GLM_MEMORY_GUARD` | guard is on by default; exact 0 or case-insensitive false/off/no disables it; any other value and unset keep it enabled | Control the pre-allocation GLM host/GPU memory safety guard. | [ds4.c:41989](ds4.c#L41989) |
+| `DS4_GLM_MEMORY_GUARD_FRACTION` | floating-point fraction; default 0.99; parsed numeric prefix is accepted, invalid/nonfinite falls back, values clamp to 0.50..1.00 | Set the fraction of detected memory usable by the GLM memory guard. | [ds4.c:42028](ds4.c#L42028) |
+| `DS4_GLM_MEMORY_GUARD_REPORT` | nonempty diagnostic flag; unset/empty is off; any nonempty value including 0 prints successful-admission accounting (refusals always report) | Print successful GLM memory-guard budget accounting. | [ds4.c:42068](ds4.c#L42068) |
+| `DS4_GLM_MEMORY_GUARD_RESERVE_GB` | floating-point GiB; dynamic default (normally 32, 24 on near-full 480..640 GiB hosts, possibly lower for resident ROCm slices); numeric prefixes accepted; invalid falls back; clamp 0..1024 | Set fixed headroom subtracted by the GLM memory guard. | [ds4.c:42046](ds4.c#L42046) |
+| `DS4_GLM_MOE_EXPERT_MAJOR` | presence selector; unset: off; any presence including empty or 0 requests the path only for n_tokens >= 16; the automatic tile-8 path takes precedence when enabled (normally n_tokens >= 128) | CUDA only: groups selected token/expert pairs by expert and uses expert-major Q2_K routed-MoE gate/up/down kernels to reuse expert weights; otherwise the normal token-major path is used. | [ds4_cuda.cu:35973](ds4_cuda.cu#L35973) |
+| `DS4_GLM_MOE_NO_DOWN_TILE8_EXACT` | presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it | Disable the exact tile-8 CUDA GLM routed-MoE down projection. | [ds4_cuda.cu:36028](ds4_cuda.cu#L36028) |
+| `DS4_GLM_MOE_NO_EXPERT_TILE8` | presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it | Disable automatic expert tile-8 CUDA GLM routed-MoE batching. | [ds4_cuda.cu:35971](ds4_cuda.cu#L35971) |
+| `DS4_GLM_MOE_NO_LOCAL_BATCH_IO` | presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it | Disable device-local batch scratch I/O for large CUDA GLM MoE batches. | [ds4_cuda.cu:35942](ds4_cuda.cu#L35942) |
+| `DS4_GLM_MOE_SCALAR` | presence rollback; unset: use optimized warp kernels; any presence including empty or 0 selects scalar baseline kernels where an earlier expert-tile/expert-major path does not return; the special two-token MTP gate/up kernel still takes precedence | CUDA only: forces the baseline scalar Q2_K routed-MoE gate/up and down implementations for A/B or correctness testing (for two-token MTP, only the down half is forced). | [ds4_cuda.cu:36154](ds4_cuda.cu#L36154) |
+| `DS4_GLM_MOE_SCRATCH_TIER0` | presence placement override; unset: allocate xq/midq quantization scratch on the current logical tier; any presence including empty or 0 allocates it on logical tier 0 | CUDA only: pins the routed-MoE xq_scratch and midq_scratch allocations to GPU tier 0 for multi-tier placement experiments; other MoE scratch remains on the current tier. | [ds4_cuda.cu:35918](ds4_cuda.cu#L35918) |
+| `DS4_GLM_MTP_NO_ATTN_TOK2` | presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it | Disable the exact two-token CUDA GLM MTP attention kernel. | [ds4_cuda.cu:34168](ds4_cuda.cu#L34168) |
+| `DS4_GLM_MTP_NO_MOE_TOK2` | presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it | Disable the exact two-token CUDA GLM MTP routed-MoE kernel. | [ds4_cuda.cu:36140](ds4_cuda.cu#L36140) |
+| `DS4_GLM_MTP_NO_SHARED_TOK2` | presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it | Disable the exact two-token CUDA GLM MTP shared-FFN kernel. | [ds4_cuda.cu:37940](ds4_cuda.cu#L37940) |
+| `DS4_GLM_MTP_PROBE` | presence flag; unset is off; any defined value, including empty or 0, is on; its second call site also rejects the normal batching path | Run the GLM next-N/MTP acceptance quality-and-timing probe without changing output and force probe-compatible scheduling. | [ds4.c:64959](ds4.c#L64959) |
+| `DS4_GLM_PREFILL_TRUNC` | nonempty value parsed by atoi (leading whitespace/sign accepted and trailing junk ignored); effective only when the resulting int is > 0 and < the current prompt length; unset/empty or a result <= 0 or >= prompt length leaves the prompt unchanged | GPU GLM debug hook: truncates the prompt before prefill/checkpoint handling so dumped prefill logits can be aligned with a CPU first-token reference. | [ds4.c:63270](ds4.c#L63270) |
+| `DS4_GLM_RESUME_PREFILL_MIN` | integer suffix tokens, non-ROCm builds only; default 4; parsed numeric prefixes accepted; <=0 maps to UINT32_MAX and effectively disables batched resume; ROCm build ignores it and stays at 4 | Set the suffix-length crossover from token decode to batched resumed prefill. | [ds4.c:38434](ds4.c#L38434) |
+| `DS4_GLM_ROUTER_SCALAR` | presence rollback; unset: use the 256-thread parallel router when n_expert <= 256 (the scalar path is already automatic above 256); any presence including empty or 0 forces the scalar path | CUDA only: selects the one-active-thread-per-token sigmoid/top-k router kernel instead of the parallel shared-memory reduction, for A/B or correctness testing. | [ds4_cuda.cu:36390](ds4_cuda.cu#L36390) |
+| `DS4_GLM_SHARED_SPLIT` | presence rollback; unset: use the fused one-token shared-expert Q8_0 gate+up+SwiGLU kernel when its shape/buffers are eligible; any presence including empty or 0 skips that fused one-token path; the earlier two-token MTP-specialized path is unaffected | CUDA only: forces shared-expert gate and up through two separate Q8_0 matmuls followed by a separate SwiGLU operation for one-token decode. | [ds4_cuda.cu:37966](ds4_cuda.cu#L37966) |
+| `DS4_GLM_STREAMING_DECODE_FULL_LAYER_MAP` | presence compatibility alias; unset: automatic mapping; any presence including empty or 0 independently forces full-layer mapping, equivalent to the backend-specific DS4_ROCM_GLM_STREAMING_DECODE_FULL_LAYER_MAP or DS4_METAL_GLM_STREAMING_DECODE_FULL_LAYER_MAP control | Backend-neutral alias for supported GLM SSD streaming (Metal/ROCm): maps every tensor in each decode layer instead of using the decode-only map that can omit routed experts served by the expert cache; layers that already require a full map are unchanged. | [ds4.c:42617](ds4.c#L42617) |
+| `DS4_GLM_STREAMING_DECODE_SYNC_EACH_LAYER` | ROCm-only third-priority legacy value: a nonempty DS4_ROCM_GLM_STREAMING_DECODE_SYNC_EACH_LAYER wins, otherwise a nonempty DS4_METAL_GLM_STREAMING_DECODE_SYNC_EACH_LAYER wins, otherwise this name is read; nonempty values are true except exact 0 or case-insensitive false/off/no; unset/empty: false; non-ROCm builds always return true and ignore this name | On ROCm non-static GLM SSD decode, opts into ending/synchronizing commands after token mapping and after every layer; the default keeps ordered work alive across layer mappings. Static-map decode bypasses this control. | [ds4.c:49781](ds4.c#L49781) |
+| `DS4_GLM_STREAMING_PREFILL_SYNC_EACH_LAYER` | ROCm-only third-priority legacy value: a nonempty DS4_ROCM_GLM_STREAMING_PREFILL_SYNC_EACH_LAYER wins, otherwise a nonempty DS4_METAL_GLM_STREAMING_PREFILL_SYNC_EACH_LAYER wins, otherwise this name is read; nonempty values are true except exact 0 or case-insensitive false/off/no; unset/empty: false for compact prefill; full-layer prefill and non-ROCm builds always synchronize and ignore this name | On ROCm compact GLM SSD prefill, opts into ending/synchronizing commands at every layer boundary; the default carries ordered work across mappings, while full-layer expert-cache prefill always retains the boundary. | [ds4.c:42397](ds4.c#L42397) |
+| `DS4_GLM_STREAMING_TOKEN_PREFILL_MAX` | unsigned token limit; backend-specific ROCm/Metal variable takes precedence, generic is fallback; default 0 on ROCm and 64 otherwise; invalid/overflow falls back, numeric prefixes accepted; 0 disables token-major streaming prefill | Set the largest SSD-streaming prefill handled by the token-major decode-like path. | [ds4.c:49684](ds4.c#L49684) |
+| `DS4_GLM_SYNC_TRACE` | presence flag; unset is off; any defined value, including empty or 0, is on; normal eligibility still applies | Log GLM checkpoint/resume and dense-versus-indexed prefill decisions. | [ds4.c:63368](ds4.c#L63368) |
+| `DS4_GLM_TP_DEBUG` | presence flag; unset is off; any defined value, including empty or 0, is on; normal eligibility still applies | Print CUDA/GLM tensor-parallel dispatch, gate, selected-ID, and failure diagnostics. | [ds4.c:44083](ds4.c#L44083) |
+| `DS4_GLM_TP_EXACT_PREFILL_MAX` | integer suffix limit via atoi cast to uint32; default 64; nonnumeric becomes 0; negative values wrap to a very large unsigned limit | Set the maximum two-way TP suffix that uses exact token-by-token prefill. | [ds4.c:63310](ds4.c#L63310) |
+| `DS4_GLM_TP_HEAD_SPLIT_MIN` | cached integer token threshold via atoi; default 64; negative values clamp to 0, nonnumeric becomes 0; 0 admits all otherwise-eligible batches | Set the minimum batch size for GLM tensor-parallel output-head splitting. | [ds4.c:38513](ds4.c#L38513) |
+| `DS4_GLM_VALUE_NO_TILE16` | presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it | Disable the CUDA GLM 16-token tiled value-projection kernel. | [ds4_cuda.cu:36806](ds4_cuda.cu#L36806) |
+
+</details>
+
+<details>
+<summary><strong>Distributed (15)</strong></summary>
+
+| Variable | Accepted value and default | Effect | Source |
+| --- | --- | --- | --- |
+| `DS4_DIST_CONNECT_BIND_HOST` | non-empty string; unset/empty means no local bind constraint | Bind outgoing distributed connections to a local host/address. | [ds4_distributed.c:1332](ds4_distributed.c#L1332) |
+| `DS4_DIST_CONNECT_BIND_IF` | non-empty string; unset/empty means no local bind constraint | Bind outgoing distributed connections to a network interface. | [ds4_distributed.c:1334](ds4_distributed.c#L1334) |
+| `DS4_DIST_CONNECT_TRACE` | presence flag; unset=off, any set value=on | Emit trace diagnostics for dist connect trace. | [ds4_distributed.c:1112](ds4_distributed.c#L1112) |
+| `DS4_DIST_DECODE_PROFILE` | presence flag; unset=off, any set value=on | Collect timing/profile diagnostics for dist decode profile. | [ds4_distributed.c:753](ds4_distributed.c#L753) |
+| `DS4_DIST_DISABLE_PREFILL_ACK_ONLY` | presence flag; unset keeps optimized/default behavior, any set value disables it | Disable/roll back dist disable prefill ack only. | [ds4_distributed.c:3692](ds4_distributed.c#L3692) |
+| `DS4_DIST_DISABLE_PREFILL_PIPELINE` | presence flag; unset keeps optimized/default behavior, any set value disables it | Disable/roll back dist disable prefill pipeline. | [ds4_distributed.c:3427](ds4_distributed.c#L3427) |
+| `DS4_DIST_DISABLE_WORKER_PREFETCH` | presence flag; unset keeps optimized/default behavior, any set value disables it | Disable/roll back dist disable worker prefetch. | [ds4_distributed.c:7878](ds4_distributed.c#L7878) |
+| `DS4_DIST_PREFILL_CHUNK` | positive integer; unset/0 uses session prefill capacity; explicit value may not exceed capacity | Set distributed prefill chunk size. | [ds4_distributed.c:3455](ds4_distributed.c#L3455) |
+| `DS4_DIST_PREFILL_SEND_DEPTH` | integer 1..8; default 2; capped to chunk count | Set coordinator prefill sender queue depth. | [ds4_distributed.c:470](ds4_distributed.c#L470) |
+| `DS4_DIST_PREFILL_WINDOW` | positive integer <=64; auto default remote stages+2 clamped 2..8 and chunk count | Set maximum distributed prefill chunks in flight. | [ds4_distributed.c:3485](ds4_distributed.c#L3485) |
+| `DS4_DIST_SOCKET_BUFFER_MB` | integer 0..512 MiB; default 128; 0 disables socket buffer override | Set TCP send/receive buffer sizes. | [ds4_distributed.c:712](ds4_distributed.c#L712) |
+| `DS4_DIST_SOCKET_RECV_TIMEOUT_SEC` | Nonempty base-10 integer parsed completely; valid range 1..3600 seconds. Unset, empty, partially parsed, or out-of-range values install no SO_RCVTIMEO at all. | Optionally bound blocking receives on distributed TCP sockets; the default deliberately permits indefinitely idle control connections during separate KV transfers. | [ds4_distributed.c:1049](ds4_distributed.c#L1049) |
+| `DS4_DIST_SOCKET_TIMEOUT_SEC` | Nonempty base-10 integer parsed completely; valid range 1..3600 seconds. Default 60 seconds for unset, empty, partially parsed, or out-of-range values. | Set SO_SNDTIMEO on distributed TCP sockets so blocked coordinator/worker sends eventually fail. | [ds4_distributed.c:1032](ds4_distributed.c#L1032) |
+| `DS4_DIST_WORKER_FORWARD_WINDOW` | integer 1..64; default 4 | Set worker forward-results window. | [ds4_distributed.c:740](ds4_distributed.c#L740) |
+| `DS4_DIST_WORKER_PREFETCH_DEPTH` | integer 1..8; default 2 | Set worker input-prefetch queue depth. | [ds4_distributed.c:726](ds4_distributed.c#L726) |
+
+</details>
+
+<details>
+<summary><strong>DSpark shared (24)</strong></summary>
+
+| Variable | Accepted value and default | Effect | Source |
+| --- | --- | --- | --- |
+| `DS4_DSPARK_CACHE_RESERVE_GB` | integer GiB via atoi; default 4.5 GiB; values 1..32 replace it, all other values fall back; decimal/trailing text is truncated/accepted by atoi | Reserve VRAM on DSpark support-cache tiers before packing support-model tensors. | [ds4.c:59770](ds4.c#L59770) |
+| `DS4_DSPARK_DISABLE_FINAL_OUTPUT_ALIAS` | presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it | Disable aliasing the final DSpark stage output to the next-stage buffer and use an explicit copy. | [ds4.c:33784](ds4.c#L33784) |
+| `DS4_DSPARK_DISABLE_FUSED_CPU_MARKOV_ARGMAX` | cached value-aware kill switch; default off; nonempty value other than exact 0 disables; false/off also disable because only 0 is recognized as false | Disable the fused CPU Markov-bias plus argmax implementation. | [ds4.c:34487](ds4.c#L34487) |
+| `DS4_DSPARK_DISABLE_REUSE_CONFIDENCE0_MARKOV` | cached value-aware kill switch; default off; nonempty value other than exact 0 disables; false/off also disable because only 0 is recognized as false | Disable reuse of the first confidence score during Markov proposal. | [ds4.c:34496](ds4.c#L34496) |
+| `DS4_DSPARK_DISABLE_VERIFY_SELECTED_PROFILE` | presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it | Override and disable the selected-expert verifier profiler. | [ds4.c:36660](ds4.c#L36660) |
+| `DS4_DSPARK_EXEC_TIER` | integer tier via atoi; default is placement/TP free-VRAM heuristic; valid 0..n_gpus-1 overrides; invalid numeric range falls back, but nonnumeric text becomes tier 0 | Choose the GPU tier that executes and primarily caches the DSpark support model. | [ds4.c:59743](ds4.c#L59743) |
+| `DS4_DSPARK_FAKE_ARGMAX_PROPOSAL` | nonempty boolean; unset/empty or exact 0: off; every other nonempty value enables, but only while DSpark itself is enabled | If the real DSpark proposer produced no draft, installs a one-token fallback proposal equal to the argmax of the current target logits; debug/test mode also selects the non-fused stage-0 setup path. | [ds4.c:64278](ds4.c#L64278) |
+| `DS4_DSPARK_LOW_MEMORY_PREFILL_CHUNK` | unsigned integer rows; default 128; 0 disables the low-memory policy; invalid/overflow falls back, numeric prefixes are accepted; only consulted for Metal SSD+DSpark on <=24 GiB hosts without an explicit chunk | Set the automatic low-memory Metal prefill chunk for SSD-streamed DSpark. | [ds4.c:60714](ds4.c#L60714) |
+| `DS4_DSPARK_NO_GPU_MARKOV` | presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it | Disable GPU Markov bias/argmax and the fully device-resident proposal path. | [ds4.c:34711](ds4.c#L34711) |
+| `DS4_DSPARK_NO_MARKOV` | cached value-aware kill switch; default off; nonempty value other than exact 0 disables Markov bias; false/off also disable | Disable Markov bias in DSpark proposal generation. | [ds4.c:34478](ds4.c#L34478) |
+| `DS4_DSPARK_PROBE` | nonempty-string diagnostic; unset/empty is off, any nonempty value including 0 is on | Log DSpark proposal/probe diagnostics. | [ds4.c:64275](ds4.c#L64275) |
+| `DS4_DSPARK_PROP_PROFILE` | presence flag; unset is off; any defined value, including empty or 0, is on; normal eligibility still applies | Print fine-grained timings for DSpark proposal setup. | [ds4.c:32959](ds4.c#L32959) |
+| `DS4_DSPARK_SPEC_LOG` | presence flag; unset is off; any defined value, including empty or 0, is on; normal eligibility still applies | Log speculative proposal, verification, acceptance, and fallback decisions. | [ds4.c:66596](ds4.c#L66596) |
+| `DS4_DSPARK_SSD_VERIFY_BLOCK_MAX` | unsigned integer rows; default/fallback 0 means automatic policy; numeric prefixes accepted; used both as verifier cap and as an exact-2 proposer-policy discriminator | Cap speculative rows verified from SSD and influence exact-2 proposal sizing. | [ds4.c:52380](ds4.c#L52380) |
+| `DS4_DSPARK_STAGE_PROFILE` | presence flag; unset is off; any defined value, including empty or 0, is on; DS4_DSPARK_STAGE_PROFILE_STAGE must also match | Profile DSpark support stages with command-boundary timings. | [ds4.c:33416](ds4.c#L33416) |
+| `DS4_DSPARK_STAGE_PROFILE_STAGE` | selector subordinate to DS4_DSPARK_STAGE_PROFILE; unset/empty: match every stage; otherwise strtoul base 10 must consume the whole value, fit uint32_t, and equal the current stage; invalid/out-of-range values match no stage | Restricts DSpark stage-boundary timing output to one stage; it does not enable profiling by itself. | [ds4.c:33417](ds4.c#L33417) |
+| `DS4_DSPARK_STATS` | value-aware flag; default off; nonempty value other than exact 0 enables; false/off are treated as enabled | Collect and print aggregate DSpark runtime statistics. | [ds4.c:61607](ds4.c#L61607) |
+| `DS4_DSPARK_VERIFY_CACHE` | presence diagnostic; unset: off; any presence including empty or 0 enables on each support-cache installation | CUDA only: copies every installed nonempty DSpark/support-cache range back to the host, byte-compares it with its source, and logs each mismatch plus a bad-count summary without changing the install result. | [ds4_cuda.cu:8257](ds4_cuda.cu#L8257) |
+| `DS4_DSPARK_VERIFY_HEAD_NO_TP` | presence rollback; unset: allow eligible CUDA output tensor parallelism; any presence including empty or 0 removes the TP path from eligibility | CUDA only: forces the DSpark speculative batched vocabulary head away from output-TP for correctness isolation; under CUDA TP+EP the attempt fails instead of using unavailable full output weights. | [ds4.c:26641](ds4.c#L26641) |
+| `DS4_DSPARK_VERIFY_NONCAUSAL` | presence diagnostic sampled once after the first successfully submitted CUDA noncausal-attention kernel; unset: verify 0 calls; any presence including empty or 0: verify that call and the next 2 | CUDA only: synchronizes and reads back Q/KV/output, computes the DSpark noncausal attention CPU reference, and logs max absolute/relative error; it reports only and does not fail the operation. | [ds4_cuda.cu:21736](ds4_cuda.cu#L21736) |
+| `DS4_DSPARK_VERIFY_PROFILE` | cached presence diagnostic; unset is off; any defined value including empty/0 profiles only the first eligible verifier invocation | Profile one full DSpark target-verifier invocation layer by layer. | [ds4.c:36757](ds4.c#L36757) |
+| `DS4_DSPARK_VERIFY_SELECTED_PROFILE` | presence flag; unset is off; any defined value, including empty or 0, enables unless DS4_DSPARK_DISABLE_VERIFY_SELECTED_PROFILE is also present (disable wins) | Profile selected-expert streaming inside the DSpark verifier. | [ds4.c:36659](ds4.c#L36659) |
+| `DS4_DSPARK_VERIFY_SPLIT_HEAD` | nonempty boolean with inverted default; unset/empty or exact 0: fused head; every other nonempty value: split head | Runs the DSpark suffix verifier output head and top-1 reduction in a separate GPU command section after the layer loop, for timing/correctness isolation; default keeps them fused into the layer command section. | [ds4.c:36725](ds4.c#L36725) |
+| `DS4_DSPARK_VERIFY_TOPS_CHECK` | presence flag; unset is off; any defined value, including empty or 0, is on; normal eligibility still applies | Read back verifier logits and compare GPU top IDs with CPU argmax. | [ds4.c:36873](ds4.c#L36873) |
+
+</details>
+
+<details>
+<summary><strong>General and shared (77)</strong></summary>
+
+| Variable | Accepted value and default | Effect | Source |
+| --- | --- | --- | --- |
+| `DS4_BATCHED_FFN` | Pure presence flag: any defined value, including empty or "0", enables. Unset leaves the default shared-expert-batched FFN path (or its configured fallback). It is read only by CPU layer-major prefill and takes precedence over shared-batch and token-parallel FFN choices. | Run the complete CPU prefill FFN in chunks through layer_ffn_batch instead of the default shared-expert-only batched path. | [ds4.c:14411](ds4.c#L14411) |
+| `DS4_BATCHED_ROPE_MAX` | Nonempty value parsed by strtol without full-string validation; integers 0..65536 are accepted, otherwise default 4096. Zero disables batched RoPE for every nonempty prompt. Effective only when prefix batch attention is selected and DS4_NO_BATCHED_ROPE is absent. | Set the largest CPU prefix-prefill token batch that applies RoPE and inverse RoPE with the batched kernels. | [ds4.c:13911](ds4.c#L13911) |
+| `DS4_BENCH_DISABLE_SNAPSHOT` | presence flag; unset allows snapshots for eligible frontiers | Disable benchmark state snapshots. | [ds4_bench.c:709](ds4_bench.c#L709) |
+| `DS4_BENCH_FORCE_SNAPSHOT` | presence flag; unset obeys the normal size/eligibility checks | Force benchmark state snapshots despite the normal limit. | [ds4_bench.c:712](ds4_bench.c#L712) |
+| `DS4_BENCH_SNAPSHOT_MAX_BYTES` | unsigned bytes or unlimited/inf; default DS4_BENCH_DEFAULT_SNAPSHOT_MAX_BYTES | Limit session snapshot size during benchmark sweeps. | [ds4_bench.c:70](ds4_bench.c#L70) |
+| `DS4_CHROME` | executable path; unset auto-detects Chrome/Chromium via standard paths and PATH | Select the browser executable used by web tooling. | [ds4_web.c:1014](ds4_web.c#L1014) |
+| `DS4_CLI_FORCE_SESSION` | Pure presence flag: any defined value, including empty or "0", forces the session path. Unset uses the session path only for distributed coordinators, TP leaders, temperature>0, or MTP depth>1; otherwise the CLI calls direct argmax generation. | Force ordinary CLI generation through run_sampled_generation/session APIs so single-node validation follows the same stateful path as TP/distributed runs. | [ds4_cli.c:1226](ds4_cli.c#L1226) |
+| `DS4_CPU_DISABLE_UNROLLED_ARGMAX` | presence rollback flag; unset keeps optimized/default path | Disable/roll back cpu disable unrolled argmax. | [ds4.c:41075](ds4.c#L41075) |
+| `DS4_CPU_DUMP_LOGITS` | filesystem path; unset disables read/write | Dump or select diagnostic data for cpu dump logits. | [ds4.c:39086](ds4.c#L39086) |
+| `DS4_CPU_DUMP_PREFILL_LOGITS` | filesystem path; unset disables read/write | Dump or select diagnostic data for cpu dump prefill logits. | [ds4.c:41606](ds4.c#L41606) |
+| `DS4_DECODE_PROFILE_DETAIL` | presence flag; unset=off | Print per-stage timing for the single-token CPU FFN path. | [ds4.c:12209](ds4.c#L12209) |
+| `DS4_DISABLE_GREEDY_TOP1_READBACK` | presence rollback; unset uses device top-1 plus a 4-byte readback for eligible single-tier greedy generation, including SSD streaming; any defined value including empty or 0 restores full-logits readback and CPU argmax | Restore the legacy per-token full-logits host readback for greedy generation A/B and emergency rollback. | [ds4.c:51964](ds4.c#L51964) |
+| `DS4_EXPERT_HOTLIST` | nonempty filesystem path; unset=off; currently Metal-only | Load an expert hotlist for Metal expert profiling/streaming. | [ds4.c:60637](ds4.c#L60637) |
+| `DS4_EXPERT_PROFILE` | presence diagnostic flag; unset=off | Collect timing/profile diagnostics for expert profile. | [ds4.c:60635](ds4.c#L60635) |
+| `DS4_FORCE_CUDA_PEER` | presence flag read once at CUDA init; unset uses automatic transfer selection; any defined value including 0 enables | Force cross-device transfers through cudaMemcpyPeerAsync for diagnostics. | [ds4_cuda.cu:364](ds4_cuda.cu#L364) |
+| `DS4_FORCE_HOST_BOUNCE` | presence flag read once at CUDA init; unset uses automatic transfer selection; any defined value including 0 enables | Force cross-device transfers through pinned host bounce buffers for diagnostics. | [ds4_cuda.cu:365](ds4_cuda.cu#L365) |
+| `DS4_LOCK_FILE` | path string; default /tmp/ds4.lock | Override the single-instance lock file. | [ds4.c:51999](ds4.c#L51999) |
+| `DS4_MMID_CASE1` | boolean-ish cached flag; default on; a value starting with 0 disables | Disable the single-expert MM-IDs specialized fast path for comparison. | [cuda/mmq/mmid.cu:290](cuda/mmq/mmid.cu#L290) |
+| `DS4_MMID_LARGE` | boolean-ish cached flag; default on; a value starting with 0 disables | Control the large-N global-memory MM-IDs path used beyond shared-memory capacity. | [cuda/mmq/mmid.cu:245](cuda/mmq/mmid.cu#L245) |
+| `DS4_MMQ_D2R` | boolean-ish cached flag; default on; a value starting with 0 disables | Control the direct-to-register Q2_K MoE down path. | [cuda/mmq/ds4_mmq.cu:525](cuda/mmq/ds4_mmq.cu#L525) |
+| `DS4_MMQ_D2R_IQ2` | boolean-ish cached flag; default on; a value starting with 0 disables | Control the direct-to-register IQ2 MoE gate/up path. | [cuda/mmq/ds4_mmq.cu:534](cuda/mmq/ds4_mmq.cu#L534) |
+| `DS4_MMQ_D2R_MIN_COLS` | positive integer; default 1024; invalid or nonpositive input restores the default | Set the minimum output-column count for the MMQ direct-to-register path. | [cuda/mmq/ds4_mmq.cu:629](cuda/mmq/ds4_mmq.cu#L629) |
+| `DS4_MMQ_D2R_STATS` | exact 1 enables; unset or every other value disables; cached and synchronizes the stream | Print partial-tile fill telemetry for the direct-to-register MMQ kernels. | [cuda/mmq/ds4_mmq_d2r.cu:33](cuda/mmq/ds4_mmq_d2r.cu#L33) |
+| `DS4_MMQ_DENSE_D2R` | boolean-ish flag; default on; exact 0 disables | Control the eligible aligned-Q8 dense prefill direct-to-register path. | [ds4_cuda.cu:19567](ds4_cuda.cu#L19567) |
+| `DS4_MMQ_NO_YIND` | presence rollback; unset keeps Y-indirect staging; any defined value including 0 disables it | Restore slot-gathered MoE gate/up activation quantization. | [cuda/mmq/ds4_mmq.cu:609](cuda/mmq/ds4_mmq.cu#L609) |
+| `DS4_MMQ_OUT_MEMSET` | exact 1 enables; unset or every other value disables; cached | Restore blanket MMQ output-buffer zeroing for diagnostics. | [cuda/mmq/ds4_mmq.cu:552](cuda/mmq/ds4_mmq.cu#L552) |
+| `DS4_MMQ_YBUF_MEMSET` | unset or 0 disables; 1 zero-fills; a value starting with p or P poison-fills with 0xFF | Control MMQ Q8_1 activation-staging initialization and its poison oracle. | [cuda/mmq/ds4_mmq.cu:578](cuda/mmq/ds4_mmq.cu#L578) |
+| `DS4_MMQ_YIND_VERIFY` | presence diagnostic; unset is off; any defined value including 0 enables | Byte-compare Y-indirect and slot-gathered MoE activation buffers. | [cuda/mmq/ds4_mmq.cu:620](cuda/mmq/ds4_mmq.cu#L620) |
+| `DS4_MOE_RECORD_SELECTED_HOTLIST` | nonempty output path; unset=off | Record per-layer selected-expert hit counts to a Metal hotlist file. | [ds4_metal.m:1741](ds4_metal.m#L1741) |
+| `DS4_MOE_RECORD_SELECTED_HOTLIST_FRESH` | presence flag; only relevant with HOTLIST; overrides MERGE | Start the selected-expert hotlist from empty state. | [ds4_metal.m:1642](ds4_metal.m#L1642) |
+| `DS4_MOE_RECORD_SELECTED_HOTLIST_MERGE` | presence flag; active only when FRESH is absent | Merge an existing selected-expert hotlist before recording. | [ds4_metal.m:1641](ds4_metal.m#L1641) |
+| `DS4_MOE_RECORD_SELECTED_IDS` | nonempty output path; unset=off | Record routed-MoE six-expert selections; also disables incompatible optimized paths. | [ds4.c:65290](ds4.c#L65290) |
+| `DS4_MOE_REPLAY_SELECTED_IDS` | nonempty input path; unset=off | Replay routed-MoE six-expert selections; also disables incompatible optimized paths. | [ds4.c:25308](ds4.c#L25308) |
+| `DS4_MTP_BATCH_VERIFY` | Pure presence flag: any defined value, including empty or "0", suppresses the exact two-row decode verifier. Unset selects exact decode-2 when draft_n==2 and either strict mode is active or the build is ROCm; other cases already use the generic verifier. | Diagnostic rollback from the exact Q8/one-token-equivalent MTP decode-2 verifier to the generic microbatch verifier. | [ds4.c:71021](ds4.c#L71021) |
+| `DS4_MTP_CAPTURE_PREFIX1` | Pure presence flag. In the generic verifier with exactly two drafts it enables prefix-1 state capture under strict mode; non-strict mode already captures prefix-1 without the variable. Unset under strict mode instead snapshots and replays a partial acceptance. | Let a one-of-two MTP partial acceptance commit the verifier's captured prefix directly, avoiding an exact one-token replay. | [ds4.c:71134](ds4.c#L71134) |
+| `DS4_MTP_CONF_LOG` | Pure presence flag; default off. It forces materialization of full draft logits, computes the top-2 margin, and after a successful generic microbatch verification prints drafted/committed counts, top candidates, margin, target-next and draft-next. Exact decode-2 success does not emit that generic log line. | Inspect MTP draft confidence and compare the recursive draft token with the target verifier result. | [ds4.c:70900](ds4.c#L70900) |
+| `DS4_MTP_EXACT_REPLAY` | Pure presence flag; default off. In the generic microbatch verifier it forces a pre-verifier frontier snapshot; after verification the snapshot is restored and every accepted draft is decoded sequentially to rebuild exact final state/logits. | Validate MTP acceptance while committing through the normal one-token decode path rather than retaining batched-verifier state. | [ds4.c:71139](ds4.c#L71139) |
+| `DS4_MTP_FORCE_SNAPSHOT` | Pure presence flag; default off. It forces a speculative-frontier snapshot before the generic verifier regardless of draft count or prefix-capture mode; it does not by itself force restoration or replay after a successful full acceptance. | Measure/debug snapshot behavior and guarantee a restorable pre-verifier frontier for generic MTP verification. | [ds4.c:71143](ds4.c#L71143) |
+| `DS4_MTP_FULL_LOGITS` | Pure presence flag; default off. When set, legacy and recursive MTP draft calls write the full vocabulary logits to s->mtp_logits; unset permits the faster top-token-only output unless confidence/margin logic independently needs logits. | Force full MTP draft-logit materialization for correctness comparison, inspection, or downstream confidence calculations. | [ds4.c:64260](ds4.c#L64260) |
+| `DS4_MTP_MIN_MARGIN` | non-negative float; default engine --mtp-margin value | Set confidence margin threshold for speculative MTP verification. | [ds4.c:70893](ds4.c#L70893) |
+| `DS4_MTP_PROBE` | Pure presence flag; default off. For legacy MTP it prepares drafts even when configured depth<=1, compares the previous draft with the next committed token, and prints cumulative hit counts/failures; generated output is unchanged. | Measure legacy MTP next-token draft accuracy without enabling speculative acceptance. | [ds4.c:64990](ds4.c#L64990) |
+| `DS4_MTP_SPEC_DISABLE` | Pure presence flag: any defined value, including empty or "0", disables MTP speculative argmax in CLI/chat/server loops. Unset permits it for greedy temperature<=0 generation with draft depth>1; unrelated split-KV speculation can still be independently requested. | Fall back from MTP multi-token speculative evaluation to normal one-token session evaluation. | [ds4_cli.c:580](ds4_cli.c#L580) |
+| `DS4_MTP_SPEC_LOG` | Pure presence flag; default off. It only emits diagnostics for first-draft misses, exact/generic verifier failures and sequential fallback misses/acceptance outcomes; it does not select a verifier. | Trace why MTP drafts were accepted, partially accepted, rejected, or sent to sequential fallback. | [ds4.c:70916](ds4.c#L70916) |
+| `DS4_MTP_STRICT` | Pure presence flag; engine quality mode also enables strictness automatically. Strict mode skips the non-strict low-margin shortcut, selects exact decode-2 for two drafts unless DS4_MTP_BATCH_VERIFY is set, and disables default prefix-1 capture unless explicitly restored. | Force the exact/quality-oriented MTP verification policy on otherwise non-quality runs. | [ds4.c:70891](ds4.c#L70891) |
+| `DS4_MTP_TIMING` | Pure presence flag; default off. When set, timestamps and prints draft, snapshot, verifier, prefix/replay and total durations for the path taken; algorithm selection is otherwise unchanged. | Profile end-to-end MTP speculative decoding and separate draft, verification and state-commit costs. | [ds4.c:70899](ds4.c#L70899) |
+| `DS4_NO_BATCHED_ATTN` | presence rollback flag; unset keeps default/optimized path | Disable/roll back no batched attn. | [ds4.c:14410](ds4.c#L14410) |
+| `DS4_NO_BATCHED_ROPE` | presence rollback flag; unset keeps default/optimized path | Disable/roll back no batched rope. | [ds4.c:13918](ds4.c#L13918) |
+| `DS4_NO_PARALLEL_ATTN_ROWS` | presence rollback flag; unset keeps default/optimized path | Disable/roll back no parallel attn rows. | [ds4.c:13904](ds4.c#L13904) |
+| `DS4_NO_ROUTED_TOKEN_PARALLEL` | presence rollback flag; unset keeps default/optimized path | Disable/roll back no routed token parallel. | [ds4.c:12588](ds4.c#L12588) |
+| `DS4_NO_SHARED_BATCH_FFN` | presence rollback flag; unset keeps default/optimized path | Disable/roll back no shared batch ffn. | [ds4.c:14413](ds4.c#L14413) |
+| `DS4_ORACLE_LOGITS` | filesystem path; unset disables read/write | Load reference logits for graph correctness comparison. | [ds4.c:39056](ds4.c#L39056) |
+| `DS4_PARALLEL_ATTN_ROWS` | Pure presence opt-in; any defined value enables the preference, but DS4_NO_PARALLEL_ATTN_ROWS overrides it. The path is eligible only for prefix prefill with cache n_raw==0 and pos0==0; unset uses per-token attention rows. | Batch/parallelize CPU prefix attention-row evaluation after cache/index preparation. | [ds4.c:13901](ds4.c#L13901) |
+| `DS4_PARALLEL_FFN` | Pure presence opt-in. It is effective only in CPU prefill when batched attention is enabled, DS4_BATCHED_FFN is absent, and the default shared-batch path has been disabled with DS4_NO_SHARED_BATCH_FFN; otherwise higher-priority paths win. | Run independent prompt-token FFNs through layer_ffn_tokens_parallel as the fallback after disabling shared batching. | [ds4.c:14412](ds4.c#L14412) |
+| `DS4_PREFILL_BATCH` | Nonempty value parsed by strtol without full-string validation; accepted range 1..4095, default 128 for unset/invalid/out-of-range values. It is used only when DS4_BATCHED_FFN selects full batched CPU FFN. | Set the token chunk size for layer_ffn_batch during CPU layer-major prefill. | [ds4.c:14414](ds4.c#L14414) |
+| `DS4_PREFILL_PROFILE_DETAIL` | presence flag; unset=off | Print detailed per-stage CPU prefill timing. | [ds4.c:12555](ds4.c#L12555) |
+| `DS4_PREFILL_PROFILE_TOKEN` | presence flag; effective within detailed prefill profiling | Print token-loop substage timings during CPU prefill. | [ds4.c:14130](ds4.c#L14130) |
+| `DS4_Q8_FOLD_SELFTEST` | positive call budget; unset/empty disables; a nonempty value parsing to 1 or less selects 512 calls | Byte-check folded Q8_1 activations against a fresh quantization; synchronizes eager streams. | [cuda/mmq/ds4_mmq.cu:5953](cuda/mmq/ds4_mmq.cu#L5953) |
+| `DS4_ROUTED_TOKEN_PARALLEL` | Pure presence flag that forces token-parallel routed MoE, even if DS4_NO_ROUTED_TOKEN_PARALLEL is also set. When unset, token parallelism is automatic for n_tok>=64 unless the NO flag is present; smaller batches use per-token routed MoE. | Choose token-parallel CPU routed-expert evaluation inside the default shared-batch FFN prefill path. | [ds4.c:12587](ds4.c#L12587) |
+| `DS4_SERVER_BATCH_LOG` | Pure presence flag read once when the decode worker starts; default off. Any defined value, including empty or "0", logs one record per coalesced decode batch with count, elapsed milliseconds and ok/error status. | Observe server-side decode coalescing size, latency and result without changing batching behavior. | [ds4_server.c:11090](ds4_server.c#L11090) |
+| `DS4_SERVER_DECODE_COALESCE_US` | integer 0..100000 microseconds; default 2000; 0 disables wait | Control server micro-batch coalescing delay. | [ds4_server.c:11069](ds4_server.c#L11069) |
+| `DS4_SSD_AUTO_CACHE_PCT` | integer 50..95; default 80 | Choose the RAM percentage used by automatic SSD expert-cache planning. | [ds4_ssd.c:81](ds4_ssd.c#L81) |
+| `DS4_TEST_METAL_EXACTN_ORACLE` | presence flag compiled only with DS4_TEST_HOOKS; unset is off; any defined value enables | Force allocation of the Metal exact-N verifier/oracle workspace in test builds. | [ds4.c:61991](ds4.c#L61991) |
+| `DS4_THREADS` | positive integer; default min(online CPUs,12), capped by DS4_MAX_THREADS; CLI thread request overrides env | Set CPU worker-pool size. | [ds4.c:1874](ds4.c#L1874) |
+| `DS4_TOKEN_TIMING` | Pure presence flag; default off. Any defined value times and prints each CPU token decode evaluation; sampling, emission and callbacks are outside the measured interval. | Report per-token CPU model-evaluation latency during direct argmax generation. | [ds4.c:41619](ds4.c#L41619) |
+| `DS4_TP_ABLATE` | comma/list string matched for hcpre,router,kv,compidx; unset=no ablation; must match on both ranks | Skip named TP encode chains for timing; output is semantically wrong. | [ds4.c:22520](ds4.c#L22520) |
+| `DS4_TP_EVENT_GATES` | presence flag; unset uses lower-latency slab flag gates when available | Fall back to Metal shared-event arrival gates. | [ds4_metal.m:10768](ds4_metal.m#L10768) |
+| `DS4_TP_GATE_PROFILE` | presence diagnostic flag; unset=off | Collect timing/profile diagnostics for tp gate profile. | [ds4_metal.m:10662](ds4_metal.m#L10662) |
+| `DS4_TP_GATE_TRACE` | presence diagnostic flag; unset=off | Emit trace diagnostics for tp gate trace. | [ds4_tp.c:911](ds4_tp.c#L911) |
+| `DS4_TP_KEEPALIVE_ITERS` | atoi unsigned iteration count; default 1200000 | Tune work per Metal TP keep-alive dispatch. | [ds4_metal.m:10626](ds4_metal.m#L10626) |
+| `DS4_TP_KEEPALIVE_TGS` | integer 1..2048; invalid/out of range uses 1 | Tune threadgroups per Metal TP keep-alive dispatch. | [ds4_metal.m:10612](ds4_metal.m#L10612) |
+| `DS4_TP_NO_KEEPALIVE` | presence flag; unset starts Metal TP keep-alive | Disable the Metal TP GPU keep-alive worker. | [ds4_metal.m:10798](ds4_metal.m#L10798) |
+| `DS4_TP_PREFILL_SPLIT_MIN` | atoi token threshold; default 32; values below 2 clamp to 2 | Set when TP prefill row-splits the replicated shared expert. | [ds4.c:29192](ds4.c#L29192) |
+| `DS4_TP_SUBGATE_PIPELINE` | nonempty integer; nonzero enables; default off; must match on both ranks | Enable TP prefill sub-chunk gate pipelining. | [ds4.c:29206](ds4.c#L29206) |
+| `DS4_TP_TIMEOUT_SEC` | atoi seconds stored unsigned; default DS4_TP_DEFAULT_TIMEOUT_SEC | Override TP control/data socket operation timeout. | [ds4_tp.c:1329](ds4_tp.c#L1329) |
+| `DS4_TRACE_TOP` | presence flag; unset=off | Print top-logit/token trace data during CPU generation. | [ds4.c:41588](ds4.c#L41588) |
+| `DS4_WS_REPACK_HASH` | exact 1 enables; unset or every other value disables unless overridden by CLI; cached | Print a per-artifact FNV-1a hash for workspace repack identity checks. | [cuda/mmq/ds4_repack.cu:530](cuda/mmq/ds4_repack.cu#L530) |
+| `DS4_WS_REPACK_THREADS` | positive integer; default min(6, hardware threads), capped at 16 and the job count | Set the CPU worker count for CUDA workspace artifact repacking. | [cuda/mmq/ds4_repack.cu:539](cuda/mmq/ds4_repack.cu#L539) |
+
+</details>
+
+### External runtime environment
+
+These names are not owned by the `DS4_*` namespace but are read directly by
+the binaries or vendored runtime code.
+
+| Variable | Accepted value and default | Effect | Source |
+| --- | --- | --- | --- |
+| `GGML_CUDA_DISABLE_GRAPHS` | Pure presence flag cached on first is_enabled() call; any defined value, including empty or "0", disables. Unset permits CUDA graphs unless the GPU architecture independently disables them; relevant only when compiled with USE_CUDA_GRAPH. | Disable vendored GGML CUDA graph capture/replay and execute through the ordinary CUDA path. | [cuda/mmq/common.cuh:1208](cuda/mmq/common.cuh#L1208) |
+| `HOME` | Filesystem directory string. In ds4-agent, unset or empty falls back to "." for the default cache and history roots; the web helper applies the same fallback for its browser profile. A nonempty value roots .ds4/kvcache, .ds4_agent_history and .ds4/browser. | Choose the user's persistent ds4-agent cache, line-history and Chrome-profile base directory. | [ds4_agent.c:4023](ds4_agent.c#L4023) |
+| `LINENOISE_ASSUME_TTY` | Pure presence test flag: any defined value, including empty or "0", treats non-TTY input as interactive while skipping real termios raw-mode setup/restoration. Unset follows isatty and uses plain blocking line input for pipes. | Exercise the live linenoise/agent editor, prompt and status layout in automated pipe-based tests without a real terminal. | [linenoise.c:593](linenoise.c#L593) |
+| `LINENOISE_COLS` | If defined, its value is returned directly through atoi with no validation: empty/nonnumeric becomes 0 and signed values are accepted. If unset, linenoise uses TIOCGWINSZ, then a cursor-position query, then fallback width 80. | Force a deterministic terminal column count for linenoise wrapping/layout tests. | [linenoise.c:684](linenoise.c#L684) |
+| `PATH` | Colon-separated executable search directories consulted only after DS4_CHROME, macOS app paths and fixed Chrome/Chromium paths fail. The first executable google-chrome, google-chrome-stable, chromium or chromium-browser wins; unset/empty/no match falls back to the literal command google-chrome, which execlp may search again. | Locate a Chrome/Chromium executable for the ds4 web/CDP tool. | [ds4_web.c:992](ds4_web.c#L992) |
+| `TERM` | Case-insensitive terminal name. Values dumb, cons25, or emacs select linenoise's simple prompt plus blocking line reader; unset, empty, or any other value selects the normal interactive editor when stdin is a TTY. | Avoid ANSI/raw interactive editing on terminal types known not to support the required escape sequences. | [linenoise.c:559](linenoise.c#L559) |
+
+## Test and fixture environment inputs
+
+These entries are consumed by repository test binaries or fixture scripts. Some
+production runtime controls are repeated here because a maintained fixture exposes
+them as part of its own test contract.
+
+### Test binaries and cleanup hooks
+
+| Variable | Accepted value and default | Effect | Source |
+| --- | --- | --- | --- |
+| `DS4_CUDA_TOPK_REGRESSION_SEC` | positive floating-point seconds; default 2.0; invalid or nonpositive input restores the default | Set the CUDA large-top-k elapsed-time regression limit. | [tests/cuda_long_context_smoke.c:72](tests/cuda_long_context_smoke.c#L72) |
+| `DS4_METAL_MOE_TILE_MAX` | cleanup-only historical spelling; no production consumer exists, so setting it has no runtime effect | Clears a legacy Metal MoE tile override while preparing the test environment. | [tests/ds4_test.c:7516](tests/ds4_test.c#L7516) |
+| `DS4_ROCM_ENABLE_Q4_PREFILL_TILE8` | cleanup-only legacy spelling; no runtime or test reader; setting it has no effect | Remove a stale opt-in name while preparing ROCm Q4 test cases; TILE8 is automatic. | [tests/test_rocm_q4_dense_pair.cpp:58](tests/test_rocm_q4_dense_pair.cpp#L58) |
+| `DS4_TEST_ALLOW_FALLBACK` | presence flag; unset: native mixed path required; any defined value including empty or 0 permits exactly the serialized-fallback counter outcome | Lets the CUDA mixed prefill/decode oracle accept serialized fallback while retaining bit-exact logit comparisons. | [tests/test_cuda_mixed_batch.c:204](tests/test_cuda_mixed_batch.c#L204) |
+| `DS4_TEST_BACKEND` | exact cpu selects CPU; every other value, including unset/empty, selects Metal on Apple and CUDA elsewhere | Chooses the backend used by model-backed tests in tests/ds4_test.c. | [tests/ds4_test.c:91](tests/ds4_test.c#L91) |
+| `DS4_TEST_BATCH_ONLY` | presence flag; unset: run batched and isolated-control phases; any defined value including empty or 0 stops after the batched archive/hash phase | Runs only the CUDA session-batch phase and skips replay against isolated control sessions. | [tests/test_cuda_session_batch.c:288](tests/test_cuda_session_batch.c#L288) |
+| `DS4_TEST_CONTEXT` | CUDA session/mixed fixtures default to 1024 and require 1024..65536; mixed-batch uses strict full decimal parsing, while session-batch uses atoi and therefore accepts numeric prefixes | Sets the context and placement hint for the CUDA session-batch and mixed prefill/decode oracles. | [tests/test_cuda_session_batch.c:125](tests/test_cuda_session_batch.c#L125) |
+| `DS4_TEST_DSPARK` | nonempty DSpark support-GGUF path; unset/empty skips the DSpark verify-depth test | Loads the DSpark support model for teacher-forced verification of committed speculative tokens. | [tests/ds4_test.c:8563](tests/ds4_test.c#L8563) |
+| `DS4_TEST_GPU_DEVICES` | GPU device-list string parsed with the normal auto-VRAM parser; unset/empty defaults to 0,2,4,6,1,3,5,7; parse failure is fatal | Selects and orders the CUDA TP/EP devices used by the mixed prefill/decode oracle. | [tests/test_cuda_mixed_batch.c:122](tests/test_cuda_mixed_batch.c#L122) |
+| `DS4_TEST_LOCAL_GOLDEN_FILE` | nonempty readable fixture path; unset/empty defaults to tests/test-vectors/flash-0731/local-golden.vec | Selects the local-golden vector file used for model-logit regression checks. | [tests/ds4_test.c:7501](tests/ds4_test.c#L7501) |
+| `DS4_TEST_LOGPROB_AUTO_METAL` | presence flag; unset forces DS4_METAL_DISABLE_METAL4=1; any presence including empty or 0 removes that rollback and permits automatic Metal selection | Runs official log-probability vectors with automatic Metal-path selection instead of the fixed pre-Metal4 baseline. | [tests/ds4_test.c:7235](tests/ds4_test.c#L7235) |
+| `DS4_TEST_LONG_PROMPT` | nonempty readable prompt-file path; unset/empty defaults to tests/long_context_story_prompt.txt | Selects the rendered story prompt for the long-context fact-recall test. | [tests/ds4_test.c:6970](tests/ds4_test.c#L6970) |
+| `DS4_TEST_LONG_WORDS` | atoi integer; unset/empty/nonnumeric defaults to 0; valid range is 0..DS4_TEST_CONTEXT-128 and numeric prefixes are accepted | Adds repeated words to alternating CUDA session-batch prompts to exercise long-prefill rows. | [tests/test_cuda_session_batch.c:135](tests/test_cuda_session_batch.c#L135) |
+| `DS4_TEST_METAL_ARGMAX_TOP1_TIMING` | presence flag; unset runs correctness only; any defined value including empty or 0 also runs the GGUF-free resident production-shape A/B | Measures generic argsort versus dedicated Metal top-1 and full-readback versus 4-byte greedy selection using resident synthetic logits, excluding model and SSD I/O. | [tests/test_metal_argmax_top1.c:545](tests/test_metal_argmax_top1.c#L545) |
+| `DS4_TEST_METAL_EXACTN_BATCH_HEAD` | nonempty value other than exact 0 enables; unset/empty/0 disables; false/off also enable | Enables the Metal exact-N batch-head path and requires its attempt/use counters for every eligible oracle case. | [tests/test_metal_exactn_oracle.c:401](tests/test_metal_exactn_oracle.c#L401) |
+| `DS4_TEST_METAL_EXACTN_ORACLE` | presence flag compiled only with DS4_TEST_HOOKS; absent from normal production builds | Force allocation of the exact-N Metal verifier/oracle workspace in tests. | [ds4.c:61991](ds4.c#L61991) |
+| `DS4_TEST_MIXED_INITIAL` | integer 128..context-1; default 128 | Set the initial prefill length for the CUDA mixed-batch oracle. | [tests/test_cuda_mixed_batch.c:111](tests/test_cuda_mixed_batch.c#L111) |
+| `DS4_TEST_MIXED_QUANTUM` | integer 1..context-1; default 128 | Set the number of prompt tokens added per CUDA mixed-batch round. | [tests/test_cuda_mixed_batch.c:113](tests/test_cuda_mixed_batch.c#L113) |
+| `DS4_TEST_MIXED_ROUNDS` | integer 1..64; default 3 | Set the number of CUDA mixed-batch oracle rounds. | [tests/test_cuda_mixed_batch.c:115](tests/test_cuda_mixed_batch.c#L115) |
+| `DS4_TEST_MODEL` | nonempty GGUF path; tests/ds4_test.c defaults to ds4flash.gguf, while standalone model-backed CUDA/Metal fixtures generally require a supplied path and fail or skip when absent | Selects the target model shared by model-backed test binaries. | [tests/ds4_test.c:14](tests/ds4_test.c#L14) |
+| `DS4_TEST_MPP_EQ_CASE` | comma-separated substring filter; unset/empty runs all cases; tokens are whitespace-trimmed and the filter is truncated to 255 bytes | Restricts Metal tensor-equivalence vectors to IDs containing at least one requested substring. | [tests/ds4_test.c:7805](tests/ds4_test.c#L7805) |
+| `DS4_TEST_MTP` | nonempty MTP support-GGUF path; unset/empty loads no MTP head; only the fast test engine uses it, with draft depth 4 | Enables the legacy MTP verify-depth regression; the test self-skips without this model. | [tests/ds4_test.c:104](tests/ds4_test.c#L104) |
+| `DS4_TEST_Q4_STREAM_ITERS` | integer 1..10000; default 2 | Set measured iterations for the Metal Q4 stream oracle. | [tests/test_metal_q4_streams.c:734](tests/test_metal_q4_streams.c#L734) |
+| `DS4_TEST_Q4_STREAM_SOAK` | integer 1..100000; default 8 | Set bounded overlap-soak iterations for the Metal Q4 stream oracle. | [tests/test_metal_q4_streams.c:736](tests/test_metal_q4_streams.c#L736) |
+| `DS4_TEST_Q4_STREAM_TIMING` | presence flag; unset: correctness/leak checks only; any defined value including empty or 0 also runs timing pairs | Adds FIFO-versus-overlap and native-versus-overlap timing measurements to the Metal Q4 stream oracle. | [tests/test_metal_q4_streams.c:737](tests/test_metal_q4_streams.c#L737) |
+| `DS4_TEST_Q4_STREAM_TIMING_BLOCKS` | integer 5..MAX_TIMING_BLOCKS; default 5 | Set the timing block count for the Metal Q4 stream oracle. | [tests/test_metal_q4_streams.c:739](tests/test_metal_q4_streams.c#L739) |
+| `DS4_TEST_Q4_STREAM_TIMING_ITERS` | integer 1..10000; default 20 | Set timing iterations for the Metal Q4 stream oracle. | [tests/test_metal_q4_streams.c:741](tests/test_metal_q4_streams.c#L741) |
+| `DS4_TEST_Q4_STREAM_WARMUP` | integer 1..64; default 1 | Set warmup iterations for the Metal Q4 stream oracle. | [tests/test_metal_q4_streams.c:732](tests/test_metal_q4_streams.c#L732) |
+| `DS4_TEST_REQUIRE_MODEL` | nonempty value other than exact 0 requires a readable model; unset/empty/0 permits a skip; false/off count as required | Turns a missing Metal exact-N oracle model from a developer skip into a release-gate failure. | [tests/test_metal_exactn_oracle.c:390](tests/test_metal_exactn_oracle.c#L390) |
+| `DS4_TEST_REQUIRE_ROCM_DEVICE` | nonempty value other than exact 0 requires a visible ROCm device; unset/empty/0 returns the fixture skip code; false/off count as required | Turns absence of a ROCm device from a skip into failure for the ROCm Q4 oracle. | [tests/test_rocm_q4_dense_pair.cpp:1569](tests/test_rocm_q4_dense_pair.cpp#L1569) |
+| `DS4_TEST_SERVER_PREFILL` | presence flag; unset: normal prefill; any defined value including empty or 0 installs a no-op display-progress callback | Exercises the progress-split prefill path used by ds4-server in CUDA session-batch and control sessions. | [tests/test_cuda_session_batch.c:110](tests/test_cuda_session_batch.c#L110) |
+| `DS4_TEST_SESSION_BATCH_ARM` | arbitrary nonempty label; unset/empty defaults to unspecified; it is logged only and does not alter execution | Labels the Metal session-batch experiment arm in setup diagnostics. | [tests/test_metal_session_batch.c:153](tests/test_metal_session_batch.c#L153) |
+| `DS4_TEST_SESSION_BATCH_TIMING` | nonempty boolean; unset/empty/exact 0 disables; every other value enables | Print timing data from the Metal session-batch oracle. | [tests/test_metal_session_batch.c:150](tests/test_metal_session_batch.c#L150) |
+| `DS4_TEST_SESSION_COUNT` | fixture-specific integer: CUDA session-batch defaults 8 and accepts atoi 2..16; CUDA mixed-batch defaults 8 with strict 3..16; Metal session-batch defaults 2 with strict 2..16 | Sets the number of simultaneous sessions exercised by the model-backed batch oracles. | [tests/test_cuda_session_batch.c:113](tests/test_cuda_session_batch.c#L113) |
+| `DS4_TEST_SSD_CACHE_EXPERTS` | strict unsigned integer 30..UINT32_MAX; unset/empty defaults to 30; read only when Metal session-batch SSD streaming is enabled | Sizes the Metal session-batch routed-expert cache used to exercise SSD union policies for N=2..5. | [tests/test_metal_session_batch.c:88](tests/test_metal_session_batch.c#L88) |
+| `DS4_TEST_SSD_STREAMING` | nonempty value other than exact 0 enables; unset/empty/0 disables; false/off also enable | Runs model-backed test engines through SSD streaming; the Metal session-batch fixture also uses cold mode and shared prefill workspace. | [tests/ds4_test.c:109](tests/ds4_test.c#L109) |
+| `DS4_TEST_SSD_STREAMING_CACHE_EXPERTS` | strtoul decimal prefix; unset/empty/nonnumeric becomes 0, values above UINT32_MAX (including a parsed negative) saturate, and trailing text is accepted | Sets the routed-expert cache count on SSD-streaming engines created by tests/ds4_test.c. | [tests/ds4_test.c:112](tests/ds4_test.c#L112) |
+| `DS4_TEST_SSD_STREAMING_CACHE_GB` | strtoull decimal GiB prefix; unset/empty/nonnumeric/zero becomes 0, byte overflow (including a parsed negative) saturates to UINT64_MAX, and trailing text is accepted | Sets the routed-expert cache byte budget on SSD-streaming engines created by tests/ds4_test.c. | [tests/ds4_test.c:114](tests/ds4_test.c#L114) |
+| `DS4_TEST_SSD_STREAMING_COLD` | nonempty boolean; unset/empty/exact 0 disables; every other value enables | Run test engines in cold SSD-streaming mode and skip hot-expert preload. | [tests/ds4_test.c:110](tests/ds4_test.c#L110) |
+| `DS4_TEST_SSD_STREAMING_PRELOAD_EXPERTS` | unsigned integer; default 0; numeric prefixes are accepted and overflow clamps to UINT32_MAX | Set the number of SSD-streaming experts preloaded by test engines. | [tests/ds4_test.c:116](tests/ds4_test.c#L116) |
+| `DS4_TEST_SSD_UNION_POLICY_SWITCH` | nonempty boolean; unset/empty/exact 0 disables; every other value enables | Exercise an SSD session-union policy transition in the Metal session-batch oracle. | [tests/test_metal_session_batch.c:152](tests/test_metal_session_batch.c#L152) |
+| `DS4_TEST_TP_DISCONNECT` | presence flag effective only in leader mode; unset: normal test; any defined value including empty or 0 enters the disconnect oracle | Waits for the TP worker to disconnect, then requires the next batch to fail and every session checkpoint to be invalidated. | [tests/test_metal_session_batch.c:268](tests/test_metal_session_batch.c#L268) |
+| `DS4_TEST_TP_LEADER_HOST` | nonempty host string required in worker mode; no default; ignored outside worker mode | Sets the TP leader address contacted by the Metal session-batch worker. | [tests/test_metal_session_batch.c:212](tests/test_metal_session_batch.c#L212) |
+| `DS4_TEST_TP_LISTEN_HOST` | nonempty host string; unset/empty defaults to 0.0.0.0; used only in leader mode | Sets the TP listen address for the Metal session-batch leader. | [tests/test_metal_session_batch.c:204](tests/test_metal_session_batch.c#L204) |
+| `DS4_TEST_TP_MODE` | unset/empty: no TP; exact leader or worker selects that role; every other nonempty value fails; incompatible with SSD-streaming mode | Selects standalone, TP-leader, or TP-worker execution for the Metal session-batch oracle. | [tests/test_metal_session_batch.c:168](tests/test_metal_session_batch.c#L168) |
+| `DS4_TEST_TP_PORT` | strict full decimal integer 1..65535; unset/empty defaults to 19452 | Sets the listen/connect port shared by Metal session-batch TP leader and worker. | [tests/test_metal_session_batch.c:63](tests/test_metal_session_batch.c#L63) |
+| `DS4_TEST_TP_TRANSPORT` | unset/empty/auto selects automatic transport; exact tcp or rdma selects that transport; other values fail | Chooses the TP transport for Metal session-batch leader/worker tests. | [tests/test_metal_session_batch.c:52](tests/test_metal_session_batch.c#L52) |
+| `DS4_TEST_VECTOR_FILE` | nonempty readable vector path; unset/empty defaults to tests/test-vectors/flash-0731/official.vec | Selects the official fixture used by log-probability and Metal tensor-equivalence tests. | [tests/ds4_test.c:7222](tests/ds4_test.c#L7222) |
+| `PROTO_Q8_DEBUG` | presence diagnostic; unset: summary only; any defined value including empty or 0 prints detailed error structure after a Q8 parity failure | Dumps bad-element tile and row/column histograms for the CUDA Q8 prototype when parity fails. | [cuda/mmq/test/proto_gemm_dense_q8_d2r.cu:647](cuda/mmq/test/proto_gemm_dense_q8_d2r.cu#L647) |
+
+### Test fixture scripts
+
+| Variable | Accepted value and default | Effect | Source |
+| --- | --- | --- | --- |
+| `DEEPSEEK_API_KEY` | secret string; required | Authenticate official test-vector fetch. | [tests/test-vectors/fetch_official_vectors.py:236](tests/test-vectors/fetch_official_vectors.py#L236) |
+| `DS4_BIN` | executable path; unset/empty defaults to ./ds4; the Q4 matrix requires it executable, the DSpark fixture skips if missing, and the GLM smoke lets command failure fail the test | Selects the ds4 binary launched by model-backed shell test fixtures. | [tests/cuda_q4_gb10_fast_matrix.sh:47](tests/cuda_q4_gb10_fast_matrix.sh#L47) |
+| `DS4_CUDA_DISABLE_DSPARK_EXACTN` | arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4 | Records and passes through the CUDA exact-N rollback used by the acceptance run. | [tests/dspark_acceptance_fixture.sh:253](tests/dspark_acceptance_fixture.sh#L253) |
+| `DS4_CUDA_DISABLE_DSPARK_EXACTN_BATCH_HEAD` | arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4 | Records and passes through the CUDA exact-N batch-head rollback. | [tests/dspark_acceptance_fixture.sh:255](tests/dspark_acceptance_fixture.sh#L255) |
+| `DS4_CUDA_DISABLE_DSPARK_EXACTN_GRAPHS` | arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4 | Records and passes through the CUDA exact-N graph rollback. | [tests/dspark_acceptance_fixture.sh:257](tests/dspark_acceptance_fixture.sh#L257) |
+| `DS4_CUDA_DISABLE_DSPARK_NONCAUSAL_ONLINE` | arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4 | Records and passes through the CUDA noncausal-online-attention rollback. | [tests/dspark_acceptance_fixture.sh:266](tests/dspark_acceptance_fixture.sh#L266) |
+| `DS4_CUDA_DSPARK_DEVICE_PROPOSER` | arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4 | Records and passes through the CUDA device-proposer opt-in. | [tests/dspark_acceptance_fixture.sh:258](tests/dspark_acceptance_fixture.sh#L258) |
+| `DS4_CUDA_DSPARK_EXACT2` | arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4 | Records and passes through the CUDA exact-2 verifier override. | [tests/dspark_acceptance_fixture.sh:250](tests/dspark_acceptance_fixture.sh#L250) |
+| `DS4_CUDA_DSPARK_EXACTN` | arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4 | Records and passes through the CUDA exact-N verifier opt-in. | [tests/dspark_acceptance_fixture.sh:252](tests/dspark_acceptance_fixture.sh#L252) |
+| `DS4_CUDA_DSPARK_EXACTN_BATCH_HEAD` | arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4 | Records and passes through the CUDA exact-N batch-head opt-in. | [tests/dspark_acceptance_fixture.sh:254](tests/dspark_acceptance_fixture.sh#L254) |
+| `DS4_CUDA_DSPARK_EXACTN_GRAPHS` | arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4 | Records and passes through the CUDA exact-N graph opt-in. | [tests/dspark_acceptance_fixture.sh:256](tests/dspark_acceptance_fixture.sh#L256) |
+| `DS4_CUDA_DSPARK_NO_DEVICE_PROPOSER` | arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4 | Records and passes through the CUDA device-proposer rollback. | [tests/dspark_acceptance_fixture.sh:259](tests/dspark_acceptance_fixture.sh#L259) |
+| `DS4_CUDA_DSPARK_PROPOSER_BLOCK_MAX` | arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4 | Records and passes through the CUDA proposer block-size cap. | [tests/dspark_acceptance_fixture.sh:269](tests/dspark_acceptance_fixture.sh#L269) |
+| `DS4_CUDA_ENABLE_DSPARK_NONCAUSAL_ONLINE` | arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4 | Records and passes through the CUDA noncausal-online-attention opt-in. | [tests/dspark_acceptance_fixture.sh:265](tests/dspark_acceptance_fixture.sh#L265) |
+| `DS4_CUDA_Q4_MATRIX_CTX` | nonempty decimal digits other than exact 0; unset/empty defaults to 4096; no upper bound; leading-zero zero strings such as 00 pass the script's guard | Sets --ctx for every Q4 GB10 smoke and score_official matrix arm. | [tests/cuda_q4_gb10_fast_matrix.sh:49](tests/cuda_q4_gb10_fast_matrix.sh#L49) |
+| `DS4_CUDA_Q4_MATRIX_DECODE_GRAPHS` | exact default, 0, or 1; unset/empty defaults to default; other values fail | Leaves decode graphs automatic, forces them off, or forces them on with capture logging for every matrix arm. | [tests/cuda_q4_gb10_fast_matrix.sh:56](tests/cuda_q4_gb10_fast_matrix.sh#L56) |
+| `DS4_CUDA_Q4_MATRIX_PROMPT` | prompt string; unset/empty defaults to Write a complete Python quicksort function with comments. | Sets the deterministic smoke prompt whose log-probability output is compared across Q4 fast-path arms. | [tests/cuda_q4_gb10_fast_matrix.sh:52](tests/cuda_q4_gb10_fast_matrix.sh#L52) |
+| `DS4_CUDA_Q4_MATRIX_SCORER` | executable path; unset/empty defaults to gguf-tools/quality-testing/score_official; a missing/nonexecutable path fails | Selects the scorer used to produce quality TSVs for each non-oracle Q4 matrix arm. | [tests/cuda_q4_gb10_fast_matrix.sh:48](tests/cuda_q4_gb10_fast_matrix.sh#L48) |
+| `DS4_CUDA_Q4_MATRIX_SKIP_PARITY` | exact 0 or 1; unset/empty defaults to 0; other values fail | When 1, skips the synthetic MMQ parity prerequisite and marks the resulting QA run incomplete. | [tests/cuda_q4_gb10_fast_matrix.sh:57](tests/cuda_q4_gb10_fast_matrix.sh#L57) |
+| `DS4_CUDA_Q4_MATRIX_SSD_CACHE` | unset/empty by default; required and passed verbatim as --ssd-streaming-cache-experts when streaming=1; must remain empty when streaming=0; no further validation | Sets the SSD expert cache as a count or NGB value for every streamed matrix arm. | [tests/cuda_q4_gb10_fast_matrix.sh:54](tests/cuda_q4_gb10_fast_matrix.sh#L54) |
+| `DS4_CUDA_Q4_MATRIX_SSD_PRELOAD` | unset/empty omits preload; a nonempty value is passed verbatim as --ssd-streaming-preload-experts and is allowed only when streaming=1 | Sets optional expert preload for streamed Q4 smoke and scoring arms. | [tests/cuda_q4_gb10_fast_matrix.sh:55](tests/cuda_q4_gb10_fast_matrix.sh#L55) |
+| `DS4_CUDA_Q4_MATRIX_SSD_STREAMING` | exact 0 or 1; unset/empty defaults to 0; other values fail | Runs all Q4 matrix arms resident or with SSD streaming and enforces matching cache/preload arguments. | [tests/cuda_q4_gb10_fast_matrix.sh:53](tests/cuda_q4_gb10_fast_matrix.sh#L53) |
+| `DS4_CUDA_Q4_MATRIX_TOKENS` | nonempty decimal digits other than exact 0; unset/empty defaults to 32; no upper bound; leading-zero zero strings such as 00 pass the script's guard | Sets the continuation length for each Q4 GB10 smoke arm. | [tests/cuda_q4_gb10_fast_matrix.sh:50](tests/cuda_q4_gb10_fast_matrix.sh#L50) |
+| `DS4_CUDA_Q4_MATRIX_TOP_K` | nonempty decimal digits other than exact 0 and numerically <=128; unset/empty defaults to 128; leading-zero zero strings such as 00 pass the guard | Sets --logprobs-top-k for the byte-comparable Q4 GB10 smoke dumps. | [tests/cuda_q4_gb10_fast_matrix.sh:51](tests/cuda_q4_gb10_fast_matrix.sh#L51) |
+| `DS4_DSPARK_FIXTURE_BACKEND` | unset/empty defaults to auto; exact auto, metal, cuda, or rocm accepted; every other value fails | Chooses the explicit backend flag for baseline and DSpark acceptance runs; auto passes none. | [tests/dspark_acceptance_fixture.sh:14](tests/dspark_acceptance_fixture.sh#L14) |
+| `DS4_DSPARK_FIXTURE_CONFIDENCE` | unset/empty omits the option and uses the runtime default; otherwise passed verbatim to --dspark-confidence without fixture-side validation | Overrides the DSpark confidence threshold for acceptance runs and records it in metadata. | [tests/dspark_acceptance_fixture.sh:13](tests/dspark_acceptance_fixture.sh#L13) |
+| `DS4_DSPARK_FIXTURE_C_ADD_MIN_ACCEPTED` | decimal digits including 0; unset/empty defaults to 8; non-digits fail | Sets the minimum accepted-draft count for the c_add case when the proposal-quality guard is active. | [tests/dspark_acceptance_fixture.sh:12](tests/dspark_acceptance_fixture.sh#L12) |
+| `DS4_DSPARK_FIXTURE_REQUIRE_ACTIVE` | exact 0 or 1; unset/empty defaults to 1; other values fail | When 1, requires aggregate proposed and accepted-draft counts to both be nonzero. | [tests/dspark_acceptance_fixture.sh:17](tests/dspark_acceptance_fixture.sh#L17) |
+| `DS4_DSPARK_FIXTURE_REQUIRE_CUDA_DEVICE_PROPOSER` | exact 0 or 1; unset/empty defaults to 0; other values fail; 1 also forces byte-identical output checking | Requires CUDA device-proposer attempts to equal uses, with nonzero use and zero fallback or policy mismatch. | [tests/dspark_acceptance_fixture.sh:22](tests/dspark_acceptance_fixture.sh#L22) |
+| `DS4_DSPARK_FIXTURE_REQUIRE_CUDA_EXACTN` | exact 0 or 1; unset/empty defaults to 0; other values fail; 1 also forces byte-identical output checking | Requires at least one CUDA exact-N attempt and zero exact-N error fallbacks. | [tests/dspark_acceptance_fixture.sh:19](tests/dspark_acceptance_fixture.sh#L19) |
+| `DS4_DSPARK_FIXTURE_REQUIRE_CUDA_EXACTN_BATCH_HEAD` | exact 0 or 1; unset/empty defaults to 0; other values fail; 1 implies REQUIRE_CUDA_EXACTN and identical output | Requires nonzero CUDA exact-N batch-head attempts/uses and zero batch-head fallbacks. | [tests/dspark_acceptance_fixture.sh:20](tests/dspark_acceptance_fixture.sh#L20) |
+| `DS4_DSPARK_FIXTURE_REQUIRE_CUDA_EXACTN_GRAPHS` | exact 0 or 1; unset/empty defaults to 0; other values fail; 1 implies REQUIRE_CUDA_EXACTN and identical output | Requires CUDA exact-N graph attempts, uses, captures, and replays, with zero no-slot or graph failures. | [tests/dspark_acceptance_fixture.sh:21](tests/dspark_acceptance_fixture.sh#L21) |
+| `DS4_DSPARK_FIXTURE_REQUIRE_DIRECT_COMMIT` | exact 0 or 1; unset/empty defaults to 0; other values fail | Requires at least one direct verifier-state commit; with REQUIRE_PARTIAL it also requires a direct partial commit. | [tests/dspark_acceptance_fixture.sh:9](tests/dspark_acceptance_fixture.sh#L9) |
+| `DS4_DSPARK_FIXTURE_REQUIRE_EXACT2` | exact 0 or 1; unset/empty defaults to 0; other values fail; 1 also forces byte-identical output checking | Requires at least one exact-2 attempt and zero exact-2 fallbacks. | [tests/dspark_acceptance_fixture.sh:18](tests/dspark_acceptance_fixture.sh#L18) |
+| `DS4_DSPARK_FIXTURE_REQUIRE_IDENTICAL` | exact 0 or 1; unset/empty defaults to 0; other values fail; several path-specific requirements force it to 1 | When 1, fails any byte difference between baseline and DSpark stdout; otherwise mismatches are reported but allowed. | [tests/dspark_acceptance_fixture.sh:10](tests/dspark_acceptance_fixture.sh#L10) |
+| `DS4_DSPARK_FIXTURE_REQUIRE_METAL_DEVICE_PROPOSER` | exact 0 or 1; unset/empty defaults to 0; other values fail; 1 also forces byte-identical output checking | Requires Metal device-proposer attempts to equal uses, with nonzero use and zero fallback or policy mismatch. | [tests/dspark_acceptance_fixture.sh:25](tests/dspark_acceptance_fixture.sh#L25) |
+| `DS4_DSPARK_FIXTURE_REQUIRE_METAL_EXACTN_BATCH_HEAD` | exact 0 or 1; unset/empty defaults to 0; other values fail; 1 also forces byte-identical output checking | Requires nonzero Metal exact-N batch-head attempts/uses and zero batch-head fallbacks. | [tests/dspark_acceptance_fixture.sh:23](tests/dspark_acceptance_fixture.sh#L23) |
+| `DS4_DSPARK_FIXTURE_REQUIRE_METAL_EXACTN_PARTIAL` | exact 0 or 1; unset/empty defaults to 0; other values fail; 1 also forces byte-identical output checking | Requires a Metal exact-N partial replay, matching verify-skip count, and zero union error fallbacks. | [tests/dspark_acceptance_fixture.sh:24](tests/dspark_acceptance_fixture.sh#L24) |
+| `DS4_DSPARK_FIXTURE_REQUIRE_PARTIAL` | unset/empty defaults to 0; exact 0 disables; any other value enables, with no 0/1 validation | Requires at least one partial-accept case and, together with REQUIRE_DIRECT_COMMIT, one direct partial commit. | [tests/dspark_acceptance_fixture.sh:8](tests/dspark_acceptance_fixture.sh#L8) |
+| `DS4_DSPARK_FIXTURE_REQUIRE_PROPOSAL_QUALITY` | unset/empty/auto selects auto; 0/false/no/off disables; 1/true/yes/on enables; values are lowercase and other strings fail; auto enables only without partial mode/confidence override and with tokens >=32 | Controls the c_add minimum-accepted-drafts quality guard. | [tests/dspark_acceptance_fixture.sh:11](tests/dspark_acceptance_fixture.sh#L11) |
+| `DS4_DSPARK_FIXTURE_SSD_STREAMING` | exact 0 or 1; unset/empty defaults to 0; other values fail | Adds --ssd-streaming to both baseline and DSpark runs when enabled. | [tests/dspark_acceptance_fixture.sh:15](tests/dspark_acceptance_fixture.sh#L15) |
+| `DS4_DSPARK_FIXTURE_SSD_STREAMING_CACHE_EXPERTS` | unset/empty omits the cache option; otherwise decimal digits including 0 are required, and a value is legal only with SSD streaming enabled | Passes an explicit --ssd-streaming-cache-experts value to both acceptance-run variants. | [tests/dspark_acceptance_fixture.sh:16](tests/dspark_acceptance_fixture.sh#L16) |
+| `DS4_DSPARK_FIXTURE_TOKENS` | token-count argument; unset/empty defaults to 32; the fixture does not validate it before passing --tokens, and auto quality treats nonnumeric or <32 as ineligible | Sets generated-token count for each baseline and DSpark acceptance case. | [tests/dspark_acceptance_fixture.sh:7](tests/dspark_acceptance_fixture.sh#L7) |
+| `DS4_DSPARK_MODEL` | target-model path; unset/empty falls back to DS4_TEST_MODEL, then ./ds4flash.gguf; a missing file causes a successful skip | Selects the target GGUF compared in baseline and DSpark acceptance runs. | [tests/dspark_acceptance_fixture.sh:5](tests/dspark_acceptance_fixture.sh#L5) |
+| `DS4_DSPARK_SSD_VERIFY_BLOCK_MAX` | unsigned integer rows; default/fallback 0 means automatic policy; numeric prefixes accepted; used both as verifier cap and as an exact-2 proposer-policy discriminator | Cap speculative rows verified from SSD and influence exact-2 proposal sizing. | [tests/dspark_acceptance_fixture.sh:271](tests/dspark_acceptance_fixture.sh#L271) |
+| `DS4_DSPARK_SUPPORT` | support-model path; unset/empty defaults to gguf/DeepSeek-V4-Flash-DSpark-support-0731.gguf; a missing file causes a successful skip | Selects the DSpark support GGUF passed through --mtp. | [tests/dspark_acceptance_fixture.sh:6](tests/dspark_acceptance_fixture.sh#L6) |
+| `DS4_DSPARK_VERIFY_NONCAUSAL` | presence diagnostic sampled once after the first successfully submitted CUDA noncausal-attention kernel; unset: verify 0 calls; any presence including empty or 0: verify that call and the next 2 | CUDA only: synchronizes and reads back Q/KV/output, computes the DSpark noncausal attention CPU reference, and logs max absolute/relative error; it reports only and does not fail the operation. | [tests/dspark_acceptance_fixture.sh:267](tests/dspark_acceptance_fixture.sh#L267) |
+| `DS4_GLM_BACKEND` | exact metal, cuda, or cpu; unset/empty defaults to metal; other values fail | Selects the backend flag used by the GLM long-context continuation smoke test. | [tests/glm_long_context_smoke.sh:29](tests/glm_long_context_smoke.sh#L29) |
+| `DS4_GLM_EXTRA_ARGS` | unset/empty adds no arguments; otherwise intentionally unquoted and therefore shell field-split and pathname-expanded | Adds backend/device options to the ds4 invocation used for every GLM long-context case. | [tests/glm_long_context_smoke.sh:91](tests/glm_long_context_smoke.sh#L91) |
+| `DS4_GLM_LONG_CONTEXT_CTX` | context argument; unset/empty defaults to 100000; forwarded to --ctx without script-side validation | Sets the context size for GLM long-context smoke invocations. | [tests/glm_long_context_smoke.sh:26](tests/glm_long_context_smoke.sh#L26) |
+| `DS4_GLM_LONG_CONTEXT_GEN` | generation-count argument; unset/empty defaults to 32; forwarded to -n without script-side validation | Sets the number of continuation tokens checked by each GLM long-context case. | [tests/glm_long_context_smoke.sh:28](tests/glm_long_context_smoke.sh#L28) |
+| `DS4_GLM_LONG_CONTEXT_REPEATS` | whitespace-separated list of prompt-padding counts; unset/empty defaults to the single count 130; each item must work as a shell integer | Chooses one or more audit-block counts used to construct long GLM prompts. | [tests/glm_long_context_smoke.sh:27](tests/glm_long_context_smoke.sh#L27) |
+| `DS4_GLM_MODEL` | model path used only when no nonempty positional MODEL is supplied; unset/empty defaults to models/GLM-5.2-UD-Q4_K_XL.gguf | Selects the GLM-5.2 GGUF used by the long-context continuation smoke test. | [tests/glm_long_context_smoke.sh:25](tests/glm_long_context_smoke.sh#L25) |
+| `DS4_METAL_DISABLE_DSPARK_EXACTN_BATCH_HEAD` | arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4 | Records and passes through the Metal exact-N batch-head rollback. | [tests/dspark_acceptance_fixture.sh:262](tests/dspark_acceptance_fixture.sh#L262) |
+| `DS4_METAL_DSPARK_DEVICE_PROPOSER` | arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4 | Records and passes through the Metal device-proposer opt-in. | [tests/dspark_acceptance_fixture.sh:263](tests/dspark_acceptance_fixture.sh#L263) |
+| `DS4_METAL_DSPARK_EXACT2` | arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4 | Records and passes through the Metal exact-2 verifier override. | [tests/dspark_acceptance_fixture.sh:251](tests/dspark_acceptance_fixture.sh#L251) |
+| `DS4_METAL_DSPARK_EXACTN_BATCH_HEAD` | arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4 | Records and passes through the Metal exact-N batch-head opt-in. | [tests/dspark_acceptance_fixture.sh:261](tests/dspark_acceptance_fixture.sh#L261) |
+| `DS4_METAL_DSPARK_EXACTN_UNION` | arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4 | Records and passes through the Metal exact-N union-verifier opt-in. | [tests/dspark_acceptance_fixture.sh:260](tests/dspark_acceptance_fixture.sh#L260) |
+| `DS4_METAL_DSPARK_EXACT_ROWS_ASYNC_TAILS` | arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4 | Records and passes through the Metal exact-row asynchronous-tail override. | [tests/dspark_acceptance_fixture.sh:268](tests/dspark_acceptance_fixture.sh#L268) |
+| `DS4_METAL_DSPARK_NO_DEVICE_PROPOSER` | arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4 | Records and passes through the Metal device-proposer rollback. | [tests/dspark_acceptance_fixture.sh:264](tests/dspark_acceptance_fixture.sh#L264) |
+| `DS4_METAL_DSPARK_PROPOSER_BLOCK_MAX` | arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4 | Records and passes through the Metal proposer block-size cap. | [tests/dspark_acceptance_fixture.sh:270](tests/dspark_acceptance_fixture.sh#L270) |
+| `DS4_TEST_MODEL` | fallback target-model path used only when DS4_DSPARK_MODEL is unset/empty; unset/empty then defaults to ./ds4flash.gguf | Provides the shared test-model fallback for the DSpark acceptance fixture. | [tests/dspark_acceptance_fixture.sh:5](tests/dspark_acceptance_fixture.sh#L5) |
+| `OPENROUTER_API_KEY` | secret string; required | Authenticate OpenRouter GLM test-vector fetch. | [tests/test-vectors/fetch_openrouter_glm_vectors.py:355](tests/test-vectors/fetch_openrouter_glm_vectors.py#L355) |
+| `TMPDIR` | temporary-directory base path; unset/empty defaults to /tmp | Chooses the parent directory for auto-created Q4 matrix, DSpark fixture, and GLM smoke work directories. | [tests/cuda_q4_gb10_fast_matrix.sh:108](tests/cuda_q4_gb10_fast_matrix.sh#L108) |
+
+## Tool and wrapper environment inputs
+
+These variables configure maintained download, service-wrapper, and offline tooling.
+A tool that accepts a variable name dynamically (for example `--api-key-env`) may read
+the caller-selected name in addition to the literal defaults listed here.
+
+| Variable | Accepted value and default | Effect | Source |
+| --- | --- | --- | --- |
+| `<NAME_FROM_--api-key-env>` | dynamic environment-variable name; no fixed identifier; overrides endpoint-derived key name | Allow a caller-selected credential environment variable. | [gguf-tools/quality-testing/collect_official.py:243](gguf-tools/quality-testing/collect_official.py#L243) |
+| `DEEPSEEK_API_KEY` | secret string; default credential for non-OpenRouter endpoint; required unless --api-key-env selects another name | Authenticate official DeepSeek continuation collection. | [gguf-tools/quality-testing/collect_official.py:242](gguf-tools/quality-testing/collect_official.py#L242) |
+| `DS4_BATCHED_SESSIONS` | unset/empty defaults to 16; otherwise passed verbatim to --batched-session; the wrapper does not validate it | Sets the maximum batched-session count for the managed CUDA tensor-parallel server. | [run-nvidia-tp-server.sh:14](run-nvidia-tp-server.sh#L14) |
+| `DS4_CTX` | unset/empty defaults to 100000; otherwise passed verbatim to --ctx; the wrapper does not validate it | Sets the managed server context size. | [run-nvidia-tp-server.sh:9](run-nvidia-tp-server.sh#L9) |
+| `DS4_GGUF_DIR` | path; default repository gguf/ directory | Choose the model download directory. | [download_model.sh:23](download_model.sh#L23) |
+| `DS4_KV_DIR` | directory path; unset/empty defaults to /data/ds4-kv | Sets --kv-disk-dir for the managed server disk-backed KV cache. | [run-nvidia-tp-server.sh:12](run-nvidia-tp-server.sh#L12) |
+| `DS4_KV_SPACE_MB` | unset/empty defaults to 8192; otherwise passed verbatim to --kv-disk-space-mb; the wrapper does not validate it | Sets the managed server disk-KV capacity in MiB. | [run-nvidia-tp-server.sh:13](run-nvidia-tp-server.sh#L13) |
+| `DS4_LOCK_FILE` | lock-file path; unset/empty defaults to /tmp/ds4.lock | Selects the PID/instance lock inspected by start, stop, restart, and status; an explicit environment value is inherited by ds4-server. | [run-nvidia-tp-server.sh:15](run-nvidia-tp-server.sh#L15) |
+| `DS4_MODEL` | model path; unset/empty defaults to /home/antirez/models/deepseek-v4-gguf/DeepSeek-V4-Flash-MXFP4Experts-F16HC-F16Compressor-F16Indexer-Q8Attn-Q8Shared-Q8Out-chat-v2-mxfp4-0731.gguf; unreadable paths fail | Sets the target GGUF passed to the managed CUDA tensor-parallel server. | [run-nvidia-tp-server.sh:8](run-nvidia-tp-server.sh#L8) |
+| `DS4_SERVER_HOST` | host string; unset/empty defaults to 0.0.0.0 | Sets the HTTP listen address passed to the managed server. | [run-nvidia-tp-server.sh:10](run-nvidia-tp-server.sh#L10) |
+| `DS4_SERVER_LOG` | log-file path; unset/empty defaults to /tmp/ds4-server.log | Receives detached-server stdout/stderr and supplies the readiness probe and failure tail. | [run-nvidia-tp-server.sh:16](run-nvidia-tp-server.sh#L16) |
+| `DS4_SERVER_PORT` | unset/empty defaults to 8000; otherwise passed verbatim to --port; the wrapper does not validate it | Sets the HTTP listen port passed to the managed server. | [run-nvidia-tp-server.sh:11](run-nvidia-tp-server.sh#L11) |
+| `DS4_START_TIMEOUT` | positive decimal integer with no leading zero; unset/empty defaults to 180; invalid values fail when starting | Sets how many seconds detached startup waits for the lock owner and listening log marker. | [run-nvidia-tp-server.sh:17](run-nvidia-tp-server.sh#L17) |
+| `DS4_STOP_TIMEOUT` | positive decimal integer with no leading zero; unset/empty defaults to 120; invalid values fail when stopping | Sets how many seconds graceful stop waits after SIGTERM before failing. | [run-nvidia-tp-server.sh:18](run-nvidia-tp-server.sh#L18) |
+| `FLATTEN_DOWNLOADS` | exact integer 1 enables; unset or 0 preserves normal shard paths | Move downloaded Hugging Face files from nested cache paths into the requested output directory. | [download_model.sh:244](download_model.sh#L244) |
+| `FORCE_HF_DOWNLOAD` | exact integer 1 enables; unset or 0 uses the available downloader automatically | Force download_model.sh to use hf download instead of curl when available. | [download_model.sh:216](download_model.sh#L216) |
+| `HF_TOKEN` | secret string; unset tries cached Hugging Face token or unauthenticated download | Authenticate Hugging Face downloads. | [download_model.sh:28](download_model.sh#L28) |
+| `HOME` | Filesystem directory string. In ds4-agent, unset or empty falls back to "." for the default cache and history roots; the web helper applies the same fallback for its browser profile. A nonempty value roots .ds4/kvcache, .ds4_agent_history and .ds4/browser. | Choose the user's persistent ds4-agent cache, line-history and Chrome-profile base directory. | [ds4_agent.c:4023](ds4_agent.c#L4023) |
+| `OPENROUTER_API_KEY` | secret string; default credential when endpoint contains openrouter.ai; required unless --api-key-env selects another name | Authenticate OpenRouter continuation collection. | [gguf-tools/quality-testing/collect_official.py:242](gguf-tools/quality-testing/collect_official.py#L242) |
+
+<!-- END GENERATED ENVIRONMENT VARIABLE INVENTORY -->
diff --git a/Makefile b/Makefile
index c58acb939b..784e10f22c 100644
--- a/Makefile
+++ b/Makefile
@@ -1,5 +1,6 @@
 CC ?= cc
 UNAME_S := $(shell uname -s)
+.DEFAULT_GOAL := all
 
 ifeq ($(UNAME_S),Darwin)
 NATIVE_CPU_FLAG ?= -mcpu=native
@@ -50,7 +51,7 @@ endif
 NVCCFLAGS ?= -O3 -g -lineinfo --use_fast_math $(NVCC_ARCH_FLAGS) -Xcompiler $(NATIVE_CPU_FLAG) -Xcompiler -pthread
 # Vendored llama.cpp mmq prefill tier (cuda/mmq/, see cuda/mmq/VENDOR.md).
 MMQ_INCLUDES := -Icuda/mmq
-MMQ_OBJS := cuda/mmq/ds4_ggml_stubs.o cuda/mmq/ds4_mmq.o cuda/mmq/ds4_mmq_d2r.o cuda/mmq/quantize.o cuda/mmq/mmid.o cuda/mmq/mmvq.o cuda/mmq/ds4_repack.o
+MMQ_OBJS := cuda/mmq/ds4_ggml_stubs.o cuda/mmq/ds4_mmq.o cuda/mmq/ds4_mmq_d2r.o cuda/mmq/ds4_mmq_q4_16warp.o cuda/mmq/quantize.o cuda/mmq/mmid.o cuda/mmq/mmvq.o cuda/mmq/ds4_repack.o
 CORE_OBJS = ds4.o ds4_image.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_cuda.o ds4_layer_pack.o $(MMQ_OBJS)
 CPU_CORE_OBJS = ds4_cpu.o ds4_image.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o
 CUDA_LDLIBS ?= -lm -Xcompiler -pthread -L$(CUDA_HOME)/targets/sbsa-linux/lib -L$(CUDA_HOME)/lib64 -lcudart -lcublas
@@ -58,19 +59,29 @@ HIPCC ?= $(shell command -v hipcc 2>/dev/null || echo /opt/rocm/bin/hipcc)
 ROCM_ARCH ?= gfx1151
 ROCM_HOST_CFLAGS ?= -fPIC
 ROCM_CFLAGS ?= -O3 -ffast-math -g -fno-finite-math-only -pthread -D__HIP_PLATFORM_AMD__ -Wno-unused-command-line-argument --offload-arch=$(ROCM_ARCH)
-ROCM_LDLIBS ?= -lm -pthread -lhipblas -lhipblaslt -lrocblas
+ROCM_LDLIBS ?= -lm -pthread -lhipblas -lrocblas
 ROCM_MMQ_Y ?= 64
 ROCM_MMQ_FLAGS := $(ROCM_CFLAGS) -std=c++17 -DGGML_USE_HIP -DDS4_HIP_MMQ_Y=$(ROCM_MMQ_Y) $(MMQ_INCLUDES)
 ROCM_MMQ_OBJS := cuda/mmq/ds4_ggml_stubs.rocm.o cuda/mmq/ds4_mmq.rocm.o cuda/mmq/quantize.rocm.o cuda/mmq/mmid.rocm.o cuda/mmq/mmvq.rocm.o cuda/mmq/d2r_stubs.rocm.o
+ROCM_CORE_OBJS := ds4.o ds4_image.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_rocm.o ds4_rocm_compat.o ds4_rocm_unavailable.o ds4_layer_pack.o $(ROCM_MMQ_OBJS)
 DS4_LINK ?= $(NVCC) $(NVCCFLAGS)
 DS4_LINK_LIBS ?= $(CUDA_LDLIBS)
 METAL_LDLIBS := $(LDLIBS)
 endif
 
-.PHONY: all help clean test test-rocm test-glm53-kda-rocm test-metal-session-batch test-mxfp4-cuda test-mxfp4-rocm test-cuda-session-batch test-cuda-mixed-batch dspark-acceptance dspark-verify-depth mtp-verify-depth cpu cuda cuda-spark cuda-generic cuda-regression strix-halo rocm
+.PHONY: all help clean test test-ssd environment-docs test-quantizer-indexer-q4 test-rocm test-glm53-kda-rocm test-metal-session-batch test-metal-session-batch-ssd test-metal-q4-streams test-metal-q4-prefill-pair test-metal-indexer-q4 test-metal-q4-attn-exactn test-metal-q4-attn-out-a-direct test-metal-q4-qb-f16-cache test-metal-q4-qb-f16-cache-timing test-metal-exactn-oracle test-metal-dspark-capture test-metal-argmax-top1 bench-metal-argmax-top1 test-metal-iq2-midonly test-metal-iq2-ssd-grouped-mm test-metal-iq2-live-index test-mxfp4-metal test-mxfp4-cuda test-mxfp4-rocm test-mmq-parity-cuda test-mmq-q4-grouped-q81-cuda test-mmq-q4-16warp-cuda test-rocm-q4-parity test-rocm-q4-dense test-rocm-q4-pair test-rocm-q4-prefill test-strix-rocm-q4-parity test-strix-rocm-q4-prefill test-strix-rocm-q4-prefill-long test-cuda-session-batch test-cuda-mixed-batch dspark-acceptance dspark-verify-depth rocm-dspark-acceptance rocm-dspark-verify-depth mtp-verify-depth cpu cuda cuda-spark cuda-generic cuda-regression strix-halo rocm cuda-iq2-moe-prefill-bench cuda-q4-prefill-bench rocm-iq2-moe-prefill-bench rocm-q4-prefill-bench
+
+gguf-tools/deepseek4-quantize: gguf-tools/deepseek4-quantize.c gguf-tools/quants.c gguf-tools/quants.h
+	$(MAKE) -C gguf-tools deepseek4-quantize
+
+tests/test_quantizer_indexer_q4: tests/test_quantizer_indexer_q4.c gguf-tools/quants.c gguf-tools/quants.h
+	$(CC) -O2 -Wall -Wextra -std=c99 -Igguf-tools -o $@ tests/test_quantizer_indexer_q4.c gguf-tools/quants.c $(LDLIBS)
+
+test-quantizer-indexer-q4: gguf-tools/deepseek4-quantize tests/test_quantizer_indexer_q4
+	./tests/test_quantizer_indexer_q4 ./gguf-tools/deepseek4-quantize
 
 ifeq ($(UNAME_S),Darwin)
-.PHONY: metal-decode-schedule-bench metal-prefill-variant-bench check-mxfp4-half-lut
+.PHONY: metal-decode-schedule-bench metal-prefill-variant-bench metal-q4-dense-pair-bench metal-q4-prefill-pair-bench metal-q4-mm-tail-cull-bench metal-q4-attn-out-a-direct-bench metal-iq2-moe-tail-cull-bench metal-iq2-moe-top8-pair-bench check-mxfp4-half-lut test-mxfp4-metal
 
 all: ds4 ds4-server ds4-bench ds4-eval ds4-agent
 
@@ -79,10 +90,35 @@ help:
 	@echo "  make              Build Metal ./ds4, ./ds4-server, ./ds4-bench, ./ds4-eval, and ./ds4-agent"
 	@echo "  make cpu          Build CPU-only ./ds4, ./ds4-server, ./ds4-bench, ./ds4-eval, and ./ds4-agent"
 	@echo "  make test         Build and run tests"
+	@echo "  make test-ssd     Run the model suite with cold SSD streaming"
+	@echo "  make test-quantizer-indexer-q4  Check direct F16-to-Q4_K indexer conversion"
+	@echo "  make environment-docs  Generate and verify the environment variable inventory"
+	@echo "  make test-metal-session-batch-ssd  Exact-logit Metal SSD union control/candidate oracle"
+	@echo "  make test-metal-q4-streams  Check resident Q4 Metal stream overlap"
+	@echo "  make test-metal-q4-prefill-pair  Runtime oracle for the M1-M4 Q4 prefill pair"
+	@echo "  make test-metal-indexer-q4  Check the production-shape Q4_K indexer projection"
+	@echo "  make test-metal-q4-attn-exactn  Bitwise/canary oracle for M1-M4 SSD-prefill Q4 attention output"
+	@echo "  make test-metal-q4-attn-out-a-direct  Production-shape oracle for M1-M4 Q4 output-A direct routing"
+	@echo "  make test-metal-q4-qb-f16-cache  Oracle for M1-M4 Q4 q_b sidecar and transient F16 paths"
+	@echo "  make test-metal-q4-qb-f16-cache-timing  Compare Q4 direct, sidecar, and transient production at N=4096"
+	@echo "  make test-metal-dspark-capture  Check fused DSpark HC capture bitwise"
+	@echo "  make test-metal-argmax-top1  Check the resident production-shape Metal decode argmax"
+	@echo "  make bench-metal-argmax-top1  Time full argsort versus resident top-1 without GGUF/SSD"
+	@echo "  make test-metal-iq2-midonly  Check M1 IQ2 addr mid-only output and sentinels"
+	@echo "  make test-metal-iq2-live-index  Check IQ2 SSD live-cache index policy and fallback"
+	@echo "  make test-rocm-q4-parity  Run ROCm Q4_K dense/pair/prefill oracle (or SKIP without HIP)"
+	@echo "  make test-rocm-q4-prefill Run ROCm Q4 tiled-prefill parity/canary oracle"
 	@echo "  make metal-decode-schedule-bench  Build the balanced Metal decode schedule benchmark"
 	@echo "  make metal-prefill-variant-bench  Build the balanced Metal prefill variant benchmark"
+	@echo "  make metal-q4-dense-pair-bench  Build the resident Q4 decode pair kernel benchmark"
+	@echo "  make metal-q4-prefill-pair-bench  Build the resident Q4 prefill pair F16-RHS benchmark"
+	@echo "  make metal-q4-mm-tail-cull-bench  Build the resident Q4 prefill tail-cull kernel benchmark"
+	@echo "  make metal-q4-attn-out-a-direct-bench  Build the resident Q4 attention output-A direct benchmark"
+	@echo "  make metal-iq2-moe-tail-cull-bench  Build the resident IQ2 pair MoE tail-cull benchmark"
+	@echo "  make metal-iq2-moe-top8-pair-bench  Build the resident GLM-shape IQ2 top-8 pair-fusion benchmark"
 	@echo "  make check-mxfp4-half-lut  Verify the checked-in MXFP4 half LUT matches the generator"
 	@echo "  make test-mxfp4-metal  Check the MXFP4 half LUT, then run Metal MXFP4 exactness tests"
+	@echo "  make test-metal-exactn-oracle  Compare Metal exact-N state with sequential decode"
 	@echo "  make dspark-verify-depth  Run DSpark speculative verification smoke if support GGUF is present"
 	@echo "  make mtp-verify-depth  Run legacy MTP speculative verification smoke if MTP GGUF is present"
 	@echo "  make clean        Remove build outputs"
@@ -114,6 +150,193 @@ tests/test_metal_session_batch: tests/test_metal_session_batch.o $(CORE_OBJS)
 test-metal-session-batch: tests/test_metal_session_batch
 	DS4_TEST_MODEL="$(DS4_TEST_MODEL)" ./tests/test_metal_session_batch
 
+test-metal-session-batch-ssd: tests/test_metal_session_batch
+	env -u DS4_METAL_ENABLE_Q4_SSD_SESSION_UNION \
+		-u DS4_METAL_REQUIRE_EXACT_ROWS_PERSISTENT_CACHE \
+		-u DS4_TEST_SSD_UNION_POLICY_SWITCH \
+		DS4_METAL_DISABLE_Q4_SSD_SESSION_UNION=1 \
+		DS4_METAL_REQUIRE_Q4_SSD_SESSION_UNION=1 \
+		DS4_METAL_DISABLE_EXACT_ROWS_PERSISTENT_CACHE=1 \
+		DS4_TEST_SSD_STREAMING=1 DS4_TEST_SESSION_COUNT=5 \
+		DS4_TEST_SSD_CACHE_EXPERTS="$(DS4_TEST_SSD_CACHE_EXPERTS)" \
+		DS4_TEST_SESSION_BATCH_TIMING=1 \
+		DS4_TEST_SESSION_BATCH_ARM=control \
+		DS4_TEST_MODEL="$(DS4_TEST_MODEL)" \
+		./tests/test_metal_session_batch
+	env -u DS4_METAL_DISABLE_Q4_SSD_SESSION_UNION \
+		-u DS4_METAL_ENABLE_Q4_SSD_SESSION_UNION \
+		-u DS4_METAL_DISABLE_EXACT_ROWS_PERSISTENT_CACHE \
+		DS4_METAL_REQUIRE_Q4_SSD_SESSION_UNION=1 \
+		DS4_METAL_REQUIRE_EXACT_ROWS_PERSISTENT_CACHE=1 \
+		DS4_TEST_SSD_STREAMING=1 DS4_TEST_SESSION_COUNT=5 \
+		DS4_TEST_SSD_UNION_POLICY_SWITCH=1 \
+		DS4_TEST_SSD_CACHE_EXPERTS="$(DS4_TEST_SSD_CACHE_EXPERTS)" \
+		DS4_TEST_SESSION_BATCH_TIMING=1 \
+		DS4_TEST_SESSION_BATCH_ARM=candidate \
+		DS4_TEST_MODEL="$(DS4_TEST_MODEL)" \
+		./tests/test_metal_session_batch
+
+tests/test_metal_q4_streams.o: tests/test_metal_q4_streams.c ds4.h ds4_gpu.h
+	$(CC) $(CFLAGS) -DDS4_TEST_HOOKS -I. -c -o $@ $<
+
+tests/test_metal_q4_streams: tests/test_metal_q4_streams.o ds4_metal_test_hooks.o ds4_image.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_metal.o ds4_layer_pack.o
+	$(CC) $(CFLAGS) -o $@ $^ $(METAL_LDLIBS)
+
+test-metal-q4-streams: tests/test_metal_q4_streams
+	env -u DS4_METAL_MODEL_UNTRACKED ./tests/test_metal_q4_streams
+	DS4_METAL_MODEL_UNTRACKED=1 ./tests/test_metal_q4_streams
+
+tests/test_metal_q4_prefill_pair.o: tests/test_metal_q4_prefill_pair.c ds4_gpu.h
+	$(CC) $(CFLAGS) -I. -c -o $@ $<
+
+tests/test_metal_q4_prefill_pair: tests/test_metal_q4_prefill_pair.o ds4_image.o ds4_metal.o
+	$(CC) $(CFLAGS) -o $@ $^ $(METAL_LDLIBS)
+
+test-metal-q4-prefill-pair: tests/test_metal_q4_prefill_pair
+	env -u DS4_METAL_ENABLE_Q4_PREFILL_PAIR_F16_RHS \
+		-u DS4_METAL_DISABLE_Q4_PREFILL_PAIR_F16_RHS \
+		-u DS4_METAL_REQUIRE_Q4_PREFILL_PAIR_F16_RHS \
+		-u DS4_METAL_DISABLE_Q4_DENSE_PAIR \
+		-u DS4_METAL_DISABLE_CONTIG_F32_F16_COPY \
+		-u DS4_METAL_MODEL_UNTRACKED \
+		-u DS4_METAL_UNRETAINED_COMMAND_BUFFERS \
+		./tests/test_metal_q4_prefill_pair
+	env -u DS4_METAL_ENABLE_Q4_PREFILL_PAIR_F16_RHS \
+		-u DS4_METAL_DISABLE_Q4_PREFILL_PAIR_F16_RHS \
+		-u DS4_METAL_REQUIRE_Q4_PREFILL_PAIR_F16_RHS \
+		-u DS4_METAL_DISABLE_Q4_DENSE_PAIR \
+		-u DS4_METAL_DISABLE_CONTIG_F32_F16_COPY \
+		-u DS4_METAL_MODEL_UNTRACKED \
+		DS4_METAL_UNRETAINED_COMMAND_BUFFERS=1 \
+		./tests/test_metal_q4_prefill_pair
+
+tests/test_metal_indexer_q4.o: tests/test_metal_indexer_q4.c ds4_gpu.h
+	$(CC) $(CFLAGS) -I. -c -o $@ $<
+
+tests/test_metal_indexer_q4: tests/test_metal_indexer_q4.o ds4_image.o ds4_metal.o
+	$(CC) $(CFLAGS) -o $@ $^ $(METAL_LDLIBS)
+
+test-metal-indexer-q4: tests/test_metal_indexer_q4
+	./tests/test_metal_indexer_q4
+
+tests/test_metal_q4_attn_exactn.o: tests/test_metal_q4_attn_exactn.c ds4_gpu.h
+	$(CC) $(CFLAGS) -I. -c -o $@ $<
+
+tests/test_metal_q4_attn_exactn: tests/test_metal_q4_attn_exactn.o ds4_image.o ds4_metal.o
+	$(CC) $(CFLAGS) -o $@ $^ $(METAL_LDLIBS)
+
+test-metal-q4-attn-exactn: tests/test_metal_q4_attn_exactn
+	env -u DS4_METAL_ENABLE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN \
+		-u DS4_METAL_DISABLE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN \
+		-u DS4_METAL_REQUIRE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN \
+		-u DS4_METAL_DISABLE_Q4_SSD_PREFILL_ATTN_OUT_SCALE_META \
+		-u DS4_METAL_REQUIRE_Q4_SSD_PREFILL_ATTN_OUT_SCALE_META \
+		-u DS4_METAL_DISABLE_Q4_MV_CLASSIC \
+		./tests/test_metal_q4_attn_exactn
+
+tests/test_metal_q4_attn_out_a_direct.o: tests/test_metal_q4_attn_out_a_direct.c ds4_gpu.h
+	$(CC) $(CFLAGS) -I. -c -o $@ $<
+
+tests/test_metal_q4_attn_out_a_direct: tests/test_metal_q4_attn_out_a_direct.o ds4_image.o ds4_metal.o
+	$(CC) $(CFLAGS) -o $@ $^ $(METAL_LDLIBS)
+
+test-metal-q4-attn-out-a-direct: tests/test_metal_q4_attn_out_a_direct
+	env -u DS4_METAL_DISABLE_Q4_ATTN_OUT_A_DIRECT \
+		-u DS4_METAL_REQUIRE_Q4_ATTN_OUT_A_DIRECT \
+		-u DS4_METAL_DISABLE_Q4_ATTN_OUT_B_F16_RHS \
+		-u DS4_METAL_REQUIRE_Q4_ATTN_OUT_B_F16_RHS \
+		./tests/test_metal_q4_attn_out_a_direct
+
+tests/test_metal_q4_qb_f16_cache.o: tests/test_metal_q4_qb_f16_cache.c ds4_gpu.h
+	$(CC) $(CFLAGS) -I. -c -o $@ $<
+
+tests/test_metal_q4_qb_f16_cache: tests/test_metal_q4_qb_f16_cache.o ds4_image.o ds4_metal.o
+	$(CC) $(CFLAGS) -o $@ $^ $(METAL_LDLIBS)
+
+test-metal-q4-qb-f16-cache: tests/test_metal_q4_qb_f16_cache
+	env -u DS4_METAL_DISABLE_Q4_ATTN_Q_B_F16_CACHE \
+		-u DS4_METAL_ENABLE_Q4_ATTN_Q_B_F16_CACHE_WITH_SSD_STREAMING \
+		-u DS4_METAL_DISABLE_Q4_ATTN_Q_B_F16_RHS \
+		-u DS4_METAL_DISABLE_Q4_ATTN_Q_B_TRANSIENT_F16 \
+		-u DS4_METAL_Q4_ATTN_Q_B_TRANSIENT_F16_MIN_TOKENS \
+		-u DS4_METAL_UNRETAINED_COMMAND_BUFFERS \
+		-u DS4_TEST_METAL_Q4_QB_F16_CACHE_TIMING \
+		-u DS4_TEST_METAL_Q4_QB_F16_CACHE_TIMING_TOKENS \
+		DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MIN_TOKENS=32 \
+		DS4_METAL_REQUIRE_Q4_ATTN_Q_B_F16_CACHE=1 \
+		./tests/test_metal_q4_qb_f16_cache
+	env -u DS4_METAL_DISABLE_Q4_ATTN_Q_B_F16_CACHE \
+		-u DS4_METAL_ENABLE_Q4_ATTN_Q_B_F16_CACHE_WITH_SSD_STREAMING \
+		-u DS4_METAL_DISABLE_Q4_ATTN_Q_B_F16_RHS \
+		-u DS4_METAL_DISABLE_Q4_ATTN_Q_B_TRANSIENT_F16 \
+		-u DS4_METAL_Q4_ATTN_Q_B_TRANSIENT_F16_MIN_TOKENS \
+		-u DS4_TEST_METAL_Q4_QB_F16_CACHE_TIMING \
+		-u DS4_TEST_METAL_Q4_QB_F16_CACHE_TIMING_TOKENS \
+		DS4_METAL_UNRETAINED_COMMAND_BUFFERS=1 \
+		DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MIN_TOKENS=32 \
+		DS4_METAL_REQUIRE_Q4_ATTN_Q_B_F16_CACHE=1 \
+		./tests/test_metal_q4_qb_f16_cache
+
+test-metal-q4-qb-f16-cache-timing: tests/test_metal_q4_qb_f16_cache
+	env -u DS4_METAL_DISABLE_Q4_ATTN_Q_B_F16_CACHE \
+		-u DS4_METAL_ENABLE_Q4_ATTN_Q_B_F16_CACHE_WITH_SSD_STREAMING \
+		-u DS4_METAL_DISABLE_Q4_ATTN_Q_B_F16_RHS \
+		-u DS4_METAL_DISABLE_Q4_ATTN_Q_B_TRANSIENT_F16 \
+		-u DS4_METAL_Q4_ATTN_Q_B_TRANSIENT_F16_MIN_TOKENS \
+		DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MIN_TOKENS=32 \
+		DS4_METAL_REQUIRE_Q4_ATTN_Q_B_F16_CACHE=1 \
+		DS4_TEST_METAL_Q4_QB_F16_CACHE_TIMING=1 \
+		DS4_TEST_METAL_Q4_QB_F16_CACHE_TIMING_TOKENS=4096 \
+		./tests/test_metal_q4_qb_f16_cache
+
+tests/test_metal_dspark_capture.o: tests/test_metal_dspark_capture.c ds4_gpu.h
+	$(CC) $(CFLAGS) -I. -c -o $@ $<
+
+tests/test_metal_dspark_capture: tests/test_metal_dspark_capture.o ds4_image.o ds4_metal.o
+	$(CC) $(CFLAGS) -o $@ $^ $(METAL_LDLIBS)
+
+test-metal-dspark-capture: tests/test_metal_dspark_capture
+	./tests/test_metal_dspark_capture
+
+tests/test_metal_argmax_top1.o: tests/test_metal_argmax_top1.c ds4_gpu.h
+	$(CC) $(CFLAGS) -fno-fast-math -I. -c -o $@ $<
+
+tests/test_metal_argmax_top1: tests/test_metal_argmax_top1.o ds4_image.o ds4_metal.o
+	$(CC) $(CFLAGS) -o $@ $^ $(METAL_LDLIBS)
+
+test-metal-argmax-top1: tests/test_metal_argmax_top1
+	env -u DS4_TEST_METAL_ARGMAX_TOP1_TIMING ./tests/test_metal_argmax_top1
+
+bench-metal-argmax-top1: tests/test_metal_argmax_top1
+	DS4_TEST_METAL_ARGMAX_TOP1_TIMING=1 ./tests/test_metal_argmax_top1
+
+tests/test_metal_iq2_midonly.o: tests/test_metal_iq2_midonly.c ds4_gpu.h
+	$(CC) $(CFLAGS) -I. -c -o $@ $<
+
+tests/test_metal_iq2_midonly: tests/test_metal_iq2_midonly.o ds4_image.o ds4_metal.o
+	$(CC) $(CFLAGS) -o $@ $^ $(METAL_LDLIBS)
+
+test-metal-iq2-midonly: tests/test_metal_iq2_midonly
+	./tests/test_metal_iq2_midonly
+
+tests/test_metal_iq2_ssd_grouped_mm.o: tests/test_metal_iq2_ssd_grouped_mm.c ds4_gpu.h
+	$(CC) $(CFLAGS) -I. -c -o $@ $<
+
+tests/test_metal_iq2_ssd_grouped_mm: tests/test_metal_iq2_ssd_grouped_mm.o ds4_image.o ds4_metal.o
+	$(CC) $(CFLAGS) -o $@ $^ $(METAL_LDLIBS)
+
+test-metal-iq2-ssd-grouped-mm: tests/test_metal_iq2_ssd_grouped_mm
+	./tests/test_metal_iq2_ssd_grouped_mm
+
+tests/test_metal_iq2_live_index.o: tests/test_metal_iq2_live_index.c ds4_gpu.h
+	$(CC) $(CFLAGS) -I. -c -o $@ $<
+
+tests/test_metal_iq2_live_index: tests/test_metal_iq2_live_index.o ds4_image.o ds4_metal.o
+	$(CC) $(CFLAGS) -o $@ $^ $(METAL_LDLIBS)
+
+test-metal-iq2-live-index: tests/test_metal_iq2_live_index
+	./tests/test_metal_iq2_live_index
+
 speed-bench/metal_decode_schedule_bench.o: speed-bench/metal_decode_schedule_bench.c ds4.h
 	$(CC) $(CFLAGS) -I. -c -o $@ $<
 
@@ -130,10 +353,60 @@ speed-bench/metal_prefill_variant_bench: speed-bench/metal_prefill_variant_bench
 
 metal-prefill-variant-bench: speed-bench/metal_prefill_variant_bench
 
+ds4_metal_test_hooks.o: ds4.c ds4.h ds4_gpu.h ds4_gpu_mgpu.h ds4_image.h ds4_layer_pack.h
+	$(CC) $(CFLAGS) -Wno-unused-function -DDS4_TEST_HOOKS -c -o $@ ds4.c
+
+tests/test_metal_exactn_oracle.o: tests/test_metal_exactn_oracle.c ds4.h
+	$(CC) $(CFLAGS) -DDS4_TEST_HOOKS -I. -c -o $@ $<
+
+tests/test_metal_exactn_oracle: tests/test_metal_exactn_oracle.o ds4_metal_test_hooks.o ds4_image.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_metal.o ds4_layer_pack.o
+	$(CC) $(CFLAGS) -o $@ $^ $(METAL_LDLIBS)
+
+test-metal-exactn-oracle: tests/test_metal_exactn_oracle
+	DS4_TEST_REQUIRE_MODEL=1 \
+	DS4_TEST_MODEL="$(DS4_TEST_MODEL)" \
+	./tests/test_metal_exactn_oracle
+
+speed-bench/metal_q4_dense_pair_bench: speed-bench/metal_q4_dense_pair_bench.m $(METAL_SRCS)
+	$(CC) $(OBJCFLAGS) -o $@ $< $(METAL_LDLIBS)
+
+metal-q4-dense-pair-bench: speed-bench/metal_q4_dense_pair_bench
+
+speed-bench/metal_q4_prefill_pair_bench: speed-bench/metal_q4_prefill_pair_bench.m $(METAL_SRCS)
+	$(CC) $(OBJCFLAGS) -o $@ $< $(METAL_LDLIBS)
+
+metal-q4-prefill-pair-bench: speed-bench/metal_q4_prefill_pair_bench
+
+speed-bench/metal_q4_mm_tail_cull_bench: speed-bench/metal_q4_mm_tail_cull_bench.m $(METAL_SRCS)
+	$(CC) $(OBJCFLAGS) -o $@ $< $(METAL_LDLIBS)
+
+metal-q4-mm-tail-cull-bench: speed-bench/metal_q4_mm_tail_cull_bench
+
+speed-bench/metal_q4_attn_out_a_direct_bench: speed-bench/metal_q4_attn_out_a_direct_bench.m $(METAL_SRCS)
+	$(CC) $(OBJCFLAGS) -o $@ $< $(METAL_LDLIBS)
+
+metal-q4-attn-out-a-direct-bench: speed-bench/metal_q4_attn_out_a_direct_bench
+
+speed-bench/metal_iq2_moe_tail_cull_bench.o: speed-bench/metal_iq2_moe_tail_cull_bench.c ds4_gpu.h
+	$(CC) $(CFLAGS) -I. -c -o $@ $<
+
+speed-bench/metal_iq2_moe_tail_cull_bench: speed-bench/metal_iq2_moe_tail_cull_bench.o ds4_image.o ds4_metal.o
+	$(CC) $(CFLAGS) -o $@ $^ $(METAL_LDLIBS)
+
+metal-iq2-moe-tail-cull-bench: speed-bench/metal_iq2_moe_tail_cull_bench
+
+speed-bench/metal_iq2_moe_top8_pair_bench.o: speed-bench/metal_iq2_moe_top8_pair_bench.c speed-bench/metal_iq2_moe_tail_cull_bench.c ds4_gpu.h
+	$(CC) $(CFLAGS) -I. -c -o $@ $<
+
+speed-bench/metal_iq2_moe_top8_pair_bench: speed-bench/metal_iq2_moe_top8_pair_bench.o ds4_image.o ds4_metal.o
+	$(CC) $(CFLAGS) -o $@ $^ $(METAL_LDLIBS)
+
+metal-iq2-moe-top8-pair-bench: speed-bench/metal_iq2_moe_top8_pair_bench
+
 tests/test_mxfp4_metal.o: tests/test_mxfp4_metal.c ds4_gpu.h
 	$(CC) $(CFLAGS) -I. -c -o $@ $<
 
-tests/test_mxfp4_metal: tests/test_mxfp4_metal.o ds4_metal.o
+tests/test_mxfp4_metal: tests/test_mxfp4_metal.o ds4_image.o ds4_metal.o
 	$(CC) $(CFLAGS) -o $@ $^ $(METAL_LDLIBS)
 
 check-mxfp4-half-lut:
@@ -159,12 +432,26 @@ help:
 	@echo "  make cuda-spark          Build CUDA for DGX Spark / GB10"
 	@echo "  make cuda-generic        Build CUDA for a generic local CUDA GPU"
 	@echo "  make cuda CUDA_ARCH=sm_N Build CUDA with an explicit nvcc -arch value"
+	@echo "  make test-mmq-parity-cuda CUDA_ARCH=sm_N  Run quantized CUDA kernel parity tests"
+	@echo "  make test-mmq-q4-grouped-q81-cuda CUDA_ARCH=sm_N  Run focused grouped Q8_1 byte-parity tests"
+	@echo "  make test-mmq-q4-16warp-cuda CUDA_ARCH=sm_N  Run focused Q4 16-warp bitwise/canary oracle"
+	@echo "  make test-rocm-q4-parity        Run ROCm Q4_K dense/pair/prefill oracle"
+	@echo "  make test-strix-rocm-q4-prefill Require gfx1151 and run tiled-prefill oracle"
+	@echo "  make test-strix-rocm-q4-parity  Require a visible gfx1151 device and run the Q4 tests"
 	@echo "  make strix-halo          Build ROCm for Strix Halo / gfx1151"
 	@echo "  make rocm                Alias for make strix-halo"
+	@echo "  make rocm-dspark-acceptance   Build ROCm and run the DSpark acceptance fixture"
+	@echo "  make rocm-dspark-verify-depth Build ROCm and run the DSpark verifier invariant"
 	@echo "  make test-mxfp4-rocm     Build and run the synthetic ROCm MXFP4 MoE test"
+	@echo "  make rocm-iq2-moe-prefill-bench  Build the resident ROCm IQ2/Q2 WMMA A/B harness"
+	@echo "  make rocm-q4-prefill-bench  Build the resident ROCm Q4 projection/WMMA A/B harness"
+	@echo "  make cuda-iq2-moe-prefill-bench CUDA_ARCH=sm_N  Build the resident CUDA IQ2/Q2 profiling harness"
+	@echo "  make cuda-q4-prefill-bench CUDA_ARCH=sm_N  Build the resident CUDA Q4 dense/pair/q_b/output-A/output-B harness"
 	@echo "  make test-rocm           Core regression suite on ROCm-only hosts"
 	@echo "  make cpu                 Build CPU-only ./ds4, ./ds4-server, ./ds4-bench, ./ds4-eval, and ./ds4-agent"
 	@echo "  make test                Build and run tests"
+	@echo "  make test-ssd            Run the model suite with cold SSD streaming"
+	@echo "  make environment-docs  Generate and verify the environment variable inventory"
 	@echo "  make dspark-verify-depth Run DSpark speculative verification smoke if support GGUF is present"
 	@echo "  make mtp-verify-depth    Run legacy MTP speculative verification smoke if MTP GGUF is present"
 	@echo "  make clean               Remove build outputs"
@@ -184,8 +471,8 @@ cuda:
 	$(MAKE) -B ds4 ds4-server ds4-bench ds4-eval ds4-agent CUDA_ARCH="$(CUDA_ARCH)"
 
 strix-halo:
-	$(MAKE) -B ds4 ds4-server ds4-bench ds4-eval ds4-agent \
-		CORE_OBJS="ds4.o ds4_image.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_rocm.o ds4_rocm_compat.o ds4_rocm_unavailable.o ds4_layer_pack.o $(ROCM_MMQ_OBJS)" \
+	$(MAKE) -B ds4 ds4-server ds4-bench ds4-eval ds4-agent ds4_test \
+		CORE_OBJS="$(ROCM_CORE_OBJS)" \
 		CFLAGS="$(CFLAGS) $(ROCM_HOST_CFLAGS) -DDS4_ROCM_BUILD" \
 		DS4_LINK="$(HIPCC) $(ROCM_CFLAGS)" \
 		DS4_LINK_LIBS="$(ROCM_LDLIBS)"
@@ -200,7 +487,7 @@ test-rocm:
 	$(MAKE) -B ds4_test ds4_agent_test ds4-eval q4k-dot-test mxfp4-dot-test \
 		tests/test_layer_pack tests/test_engine_mgpu_placement tests/test_gpu_args tests/test_prompt_prefix \
 		ds4 ds4-server ds4-bench ds4-agent \
-		CORE_OBJS="ds4.o ds4_image.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_rocm.o ds4_rocm_compat.o ds4_rocm_unavailable.o ds4_layer_pack.o $(ROCM_MMQ_OBJS)" \
+		CORE_OBJS="$(ROCM_CORE_OBJS)" \
 		CFLAGS="$(CFLAGS) $(ROCM_HOST_CFLAGS) -DDS4_ROCM_BUILD" \
 		DS4_LINK="$(HIPCC) $(ROCM_CFLAGS)" \
 		DS4_LINK_LIBS="$(ROCM_LDLIBS)"
@@ -213,6 +500,41 @@ test-rocm:
 	./tests/test_gpu_args_cli.sh
 	./tests/test_prompt_prefix
 
+rocm-dspark-acceptance:
+	@if [ ! -f "$(DS4_DSPARK_MODEL)" ]; then \
+		echo "rocm-dspark-acceptance: missing model $(DS4_DSPARK_MODEL)" >&2; \
+		exit 1; \
+	elif [ ! -f "$(DS4_DSPARK_SUPPORT)" ]; then \
+		echo "rocm-dspark-acceptance: missing DSpark support $(DS4_DSPARK_SUPPORT)" >&2; \
+		exit 1; \
+	fi
+	$(MAKE) -B ds4 \
+		CORE_OBJS="$(ROCM_CORE_OBJS)" \
+		CFLAGS="$(CFLAGS) $(ROCM_HOST_CFLAGS) -DDS4_ROCM_BUILD" \
+		DS4_LINK="$(HIPCC) $(ROCM_CFLAGS)" \
+		DS4_LINK_LIBS="$(ROCM_LDLIBS)"
+	DS4_DSPARK_MODEL="$(DS4_DSPARK_MODEL)" \
+	DS4_DSPARK_SUPPORT="$(DS4_DSPARK_SUPPORT)" \
+	DS4_DSPARK_FIXTURE_BACKEND=rocm \
+	sh tests/dspark_acceptance_fixture.sh
+
+rocm-dspark-verify-depth:
+	@if [ ! -f "$(DS4_TEST_MODEL)" ]; then \
+		echo "rocm-dspark-verify-depth: missing model $(DS4_TEST_MODEL)" >&2; \
+		exit 1; \
+	elif [ ! -f "$(DS4_DSPARK_SUPPORT)" ]; then \
+		echo "rocm-dspark-verify-depth: missing DSpark support $(DS4_DSPARK_SUPPORT)" >&2; \
+		exit 1; \
+	fi
+	$(MAKE) -B ds4_test \
+		CORE_OBJS="$(ROCM_CORE_OBJS)" \
+		CFLAGS="$(CFLAGS) $(ROCM_HOST_CFLAGS) -DDS4_ROCM_BUILD" \
+		DS4_LINK="$(HIPCC) $(ROCM_CFLAGS)" \
+		DS4_LINK_LIBS="$(ROCM_LDLIBS)"
+	DS4_TEST_MODEL="$(DS4_TEST_MODEL)" \
+	DS4_TEST_DSPARK="$(DS4_DSPARK_SUPPORT)" \
+	./ds4_test --dspark-verify-depth
+
 ds4: ds4_cli.o ds4_help.o ds4_prompt_prefix.o linenoise.o ds4_gpu_args.o $(CORE_OBJS)
 	$(DS4_LINK) -o $@ $^ $(DS4_LINK_LIBS)
 
@@ -249,9 +571,65 @@ tests/test_mxfp4_cuda: tests/test_mxfp4_cuda.cu $(MMQ_OBJS)
 
 test-mxfp4-cuda: tests/test_mxfp4_cuda
 	./tests/test_mxfp4_cuda
+
+cuda/mmq/test/test_mmq_parity: cuda/mmq/test/test_mmq_parity.cu cuda/mmq/ds4_mmq.h $(MMQ_OBJS)
+	$(NVCC) $(NVCCFLAGS) -std=c++17 $(MMQ_INCLUDES) -o $@ $< $(MMQ_OBJS) $(CUDA_LDLIBS)
+
+test-mmq-parity-cuda: cuda/mmq/test/test_mmq_parity
+	./cuda/mmq/test/test_mmq_parity
+
+test-mmq-q4-grouped-q81-cuda: cuda/mmq/test/test_mmq_parity
+	./cuda/mmq/test/test_mmq_parity --q4-grouped-q81
+
+test-mmq-q4-16warp-cuda: cuda/mmq/test/test_mmq_parity
+	./cuda/mmq/test/test_mmq_parity --q4-16warp
+
+speed-bench/gpu_iq2_moe_prefill_bench_rocm.o: speed-bench/gpu_iq2_moe_prefill_bench.c ds4_gpu.h
+	$(CC) $(filter-out -ffast-math,$(CFLAGS)) $(ROCM_HOST_CFLAGS) -std=c11 -DDS4_ROCM_BUILD -DDS4_BENCH_ROCM -I. -c -o $@ $<
+
+speed-bench/gpu_iq2_moe_prefill_bench_rocm: speed-bench/gpu_iq2_moe_prefill_bench_rocm.o ds4_image.o ds4_rocm.o $(ROCM_MMQ_OBJS)
+	$(HIPCC) $(ROCM_CFLAGS) -o $@ $^ $(ROCM_LDLIBS)
+
+rocm-iq2-moe-prefill-bench:
+	$(MAKE) --no-print-directory -B speed-bench/gpu_iq2_moe_prefill_bench_rocm ROCM_ARCH="$(ROCM_ARCH)"
+
+speed-bench/rocm_q4_prefill_bench.o: speed-bench/rocm_q4_prefill_bench.cpp ds4_gpu.h
+	$(HIPCC) $(ROCM_CFLAGS) -DDS4_ROCM_BUILD -std=c++17 -fno-fast-math -I. -c -o $@ $<
+
+speed-bench/rocm_q4_prefill_bench: speed-bench/rocm_q4_prefill_bench.o ds4_image.o ds4_rocm.o $(ROCM_MMQ_OBJS) ds4_rocm_compat.o ds4_rocm_unavailable.o
+	$(HIPCC) $(ROCM_CFLAGS) -o $@ $^ $(ROCM_LDLIBS)
+
+rocm-q4-prefill-bench:
+	$(MAKE) --no-print-directory -B speed-bench/rocm_q4_prefill_bench ROCM_ARCH="$(ROCM_ARCH)"
+
+speed-bench/gpu_iq2_moe_prefill_bench_cuda.o: speed-bench/gpu_iq2_moe_prefill_bench.c ds4_gpu.h
+	$(CC) $(filter-out -ffast-math,$(CFLAGS)) -std=c11 -DDS4_BENCH_CUDA -I. -c -o $@ $<
+
+speed-bench/gpu_iq2_moe_prefill_bench_cuda: speed-bench/gpu_iq2_moe_prefill_bench_cuda.o ds4_image.o ds4_cuda.o $(MMQ_OBJS)
+	$(NVCC) $(NVCCFLAGS) -std=c++17 $(MMQ_INCLUDES) -o $@ $^ $(CUDA_LDLIBS)
+
+cuda-iq2-moe-prefill-bench:
+	$(MAKE) --no-print-directory -B speed-bench/gpu_iq2_moe_prefill_bench_cuda CUDA_ARCH="$(CUDA_ARCH)"
+
+speed-bench/cuda_q4_prefill_bench.o: speed-bench/cuda_q4_prefill_bench.cu ds4_gpu.h cuda/mmq/ds4_mmq.h cuda/mmq/ds4_mmq_q4_16warp.cuh
+	$(NVCC) $(NVCCFLAGS) -std=c++17 -DDS4_BENCH_CUDA -I. -c -o $@ $<
+
+speed-bench/cuda_q4_prefill_bench: speed-bench/cuda_q4_prefill_bench.o ds4_image.o ds4_cuda.o $(MMQ_OBJS)
+	$(NVCC) $(NVCCFLAGS) -std=c++17 $(MMQ_INCLUDES) -o $@ $^ $(CUDA_LDLIBS)
+
+cuda-q4-prefill-bench:
+	@if [ -z "$(strip $(CUDA_ARCH))" ]; then \
+		echo "error: specify CUDA_ARCH, for example: make cuda-q4-prefill-bench CUDA_ARCH=sm_121"; \
+		exit 2; \
+	fi
+	$(MAKE) --no-print-directory -B speed-bench/cuda_q4_prefill_bench CUDA_ARCH="$(CUDA_ARCH)"
 endif
 
-ds4.o: ds4.c ds4.h ds4_ssd.h ds4_distributed.h ds4_gpu.h
+environment-docs:
+	python3 scripts/generate_environment_variables.py
+	python3 scripts/generate_environment_variables.py --check
+
+ds4.o: ds4.c ds4.h ds4_ssd.h ds4_distributed.h ds4_gpu.h ds4_image.h
 	$(CC) $(CFLAGS) -c -o $@ ds4.c
 
 ds4_image.o: ds4_image.c ds4_image.h third_party/iris/jpeg.h third_party/iris/png.h
@@ -311,7 +689,7 @@ rax.o: rax.c rax.h rax_malloc.h
 linenoise.o: linenoise.c linenoise.h
 	$(CC) $(CFLAGS) -c -o $@ linenoise.c
 
-ds4_cpu.o: ds4.c ds4.h ds4_ssd.h ds4_distributed.h ds4_gpu.h
+ds4_cpu.o: ds4.c ds4.h ds4_ssd.h ds4_distributed.h ds4_gpu.h ds4_image.h
 	$(CC) $(CFLAGS) -Wno-unused-function -DDS4_NO_GPU -c -o $@ ds4.c
 
 ds4_cli_cpu.o: ds4_cli.c ds4.h ds4_ssd.h ds4_distributed.h ds4_help.h ds4_prompt_prefix.h linenoise.h
@@ -332,7 +710,7 @@ ds4_eval_cpu.o: ds4_eval.c ds4.h ds4_ssd.h ds4_distributed.h ds4_help.h
 ds4_agent_cpu.o: ds4_agent.c ds4.h ds4_ssd.h ds4_distributed.h ds4_help.h ds4_prompt_prefix.h ds4_kvstore.h ds4_web.h linenoise.h
 	$(CC) $(CFLAGS) -DDS4_NO_GPU -c -o $@ ds4_agent.c
 
-ds4_metal.o: ds4_metal.m ds4_gpu.h $(METAL_SRCS)
+ds4_metal.o: ds4_metal.m ds4_gpu.h ds4_image.h $(METAL_SRCS)
 	$(CC) $(OBJCFLAGS) -c -o $@ ds4_metal.m
 
 tests/test_glm53_kda.o: tests/test_glm53_kda.c ds4_gpu.h
@@ -365,10 +743,10 @@ tests/test_deepseek4_vision_image: tests/test_deepseek4_vision_image.o ds4_image
 	$(CC) $(CFLAGS) -o $@ $^ -lm
 
 ifeq ($(UNAME_S),Darwin)
-$(GLM53_KDA_TEST): tests/test_glm53_kda.o ds4_metal.o
+$(GLM53_KDA_TEST): tests/test_glm53_kda.o ds4_image.o ds4_metal.o
 	$(CC) $(CFLAGS) -o $@ $^ $(METAL_LDLIBS)
 else
-$(GLM53_KDA_TEST): tests/test_glm53_kda.o ds4_cuda.o $(MMQ_OBJS)
+$(GLM53_KDA_TEST): tests/test_glm53_kda.o ds4_image.o ds4_cuda.o $(MMQ_OBJS)
 	$(NVCC) $(NVCCFLAGS) -o $@ $^ $(CUDA_LDLIBS)
 endif
 
@@ -379,7 +757,7 @@ test-glm53-kda: $(GLM53_KDA_TEST)
 tests/test_glm53_kda_rocm.o: tests/test_glm53_kda.c ds4_gpu.h
 	$(CC) $(filter-out -ffast-math,$(CFLAGS)) $(ROCM_HOST_CFLAGS) -DDS4_ROCM_BUILD -I. -c -o $@ $<
 
-$(GLM53_KDA_ROCM_TEST): tests/test_glm53_kda_rocm.o ds4_rocm.o
+$(GLM53_KDA_ROCM_TEST): tests/test_glm53_kda_rocm.o ds4_image.o ds4_rocm.o
 	$(HIPCC) $(ROCM_CFLAGS) -o $@ $^ $(ROCM_LDLIBS)
 
 test-glm53-kda-rocm: $(GLM53_KDA_ROCM_TEST)
@@ -391,15 +769,18 @@ ds4_cuda.o: ds4_cuda.cu ds4_gpu.h ds4_gpu_mgpu.h ds4_glm53_vision_gpu.cuh ds4_de
 # Vendored mmq pieces (see cuda/mmq/VENDOR.md).  ds4_mmq.cu transitively
 # pulls in mmq.cuh which has heavy template instantiation -- each piece
 # compiles in its own TU and links in.
-cuda/mmq/ds4_ggml_stubs.o: cuda/mmq/ds4_ggml_stubs.cu cuda/mmq/ds4_ggml_stubs.h cuda/mmq/common.cuh
+cuda/mmq/ds4_ggml_stubs.o: cuda/mmq/ds4_ggml_stubs.cu cuda/mmq/ds4_mmq.h cuda/mmq/ds4_ggml_stubs.h cuda/mmq/common.cuh
 	$(NVCC) $(NVCCFLAGS) -std=c++17 $(MMQ_INCLUDES) -c -o $@ $<
 
-cuda/mmq/ds4_mmq.o: cuda/mmq/ds4_mmq.cu cuda/mmq/ds4_mmq.h cuda/mmq/ds4_mmq_d2r.cuh cuda/mmq/mmq.cuh cuda/mmq/common.cuh cuda/mmq/ds4_ggml_stubs.h cuda/mmq/quantize.cuh cuda/mmq/mmid.cuh cuda/mmq/vecdotq.cuh cuda/mmq/mma.cuh
+cuda/mmq/ds4_mmq.o: cuda/mmq/ds4_mmq.cu cuda/mmq/ds4_mmq.h cuda/mmq/ds4_mmq_d2r.cuh cuda/mmq/ds4_mmq_q4_16warp.cuh cuda/mmq/mmq.cuh cuda/mmq/common.cuh cuda/mmq/ds4_ggml_stubs.h cuda/mmq/quantize.cuh cuda/mmq/mmid.cuh cuda/mmq/vecdotq.cuh cuda/mmq/mma.cuh
 	$(NVCC) $(NVCCFLAGS) -std=c++17 $(MMQ_INCLUDES) -c -o $@ $<
 
 cuda/mmq/ds4_mmq_d2r.o: cuda/mmq/ds4_mmq_d2r.cu cuda/mmq/ds4_mmq_d2r.cuh cuda/mmq/mmq.cuh cuda/mmq/common.cuh cuda/mmq/ds4_ggml_stubs.h cuda/mmq/vecdotq.cuh cuda/mmq/mma.cuh
 	$(NVCC) $(NVCCFLAGS) -std=c++17 $(MMQ_INCLUDES) -c -o $@ $<
 
+cuda/mmq/ds4_mmq_q4_16warp.o: cuda/mmq/ds4_mmq_q4_16warp.cu cuda/mmq/ds4_mmq_q4_16warp.cuh cuda/mmq/mmq.cuh cuda/mmq/common.cuh cuda/mmq/ds4_ggml_stubs.h cuda/mmq/vecdotq.cuh cuda/mmq/mma.cuh
+	$(NVCC) $(NVCCFLAGS) -std=c++17 $(MMQ_INCLUDES) -c -o $@ $<
+
 cuda/mmq/quantize.o: cuda/mmq/quantize.cu cuda/mmq/quantize.cuh cuda/mmq/common.cuh cuda/mmq/ds4_ggml_stubs.h cuda/mmq/mmq.cuh
 	$(NVCC) $(NVCCFLAGS) -std=c++17 $(MMQ_INCLUDES) -c -o $@ $<
 
@@ -436,13 +817,13 @@ cuda/mmq/d2r_stubs.rocm.o: cuda/mmq/test/d2r_stubs.cu cuda/mmq/ds4_mmq_d2r.cuh c
 tests/test_mxfp4_rocm.o: tests/test_mxfp4_rocm.c ds4_gpu.h
 	$(CC) $(filter-out -ffast-math,$(CFLAGS)) $(ROCM_HOST_CFLAGS) -DDS4_ROCM_BUILD -I. -c -o $@ $<
 
-tests/test_mxfp4_rocm: tests/test_mxfp4_rocm.o ds4_rocm.o
+tests/test_mxfp4_rocm: tests/test_mxfp4_rocm.o ds4_image.o ds4_rocm.o
 	$(HIPCC) $(ROCM_CFLAGS) -o $@ $^ $(ROCM_LDLIBS)
 
 tests/bench_mxfp4_rocm.o: tests/bench_mxfp4_rocm.c ds4_gpu.h
 	$(CC) $(filter-out -ffast-math,$(CFLAGS)) $(ROCM_HOST_CFLAGS) -DDS4_ROCM_BUILD -I. -c -o $@ $<
 
-tests/bench_mxfp4_rocm: tests/bench_mxfp4_rocm.o ds4_rocm.o
+tests/bench_mxfp4_rocm: tests/bench_mxfp4_rocm.o ds4_image.o ds4_rocm.o
 	$(HIPCC) $(ROCM_CFLAGS) -o $@ $^ $(ROCM_LDLIBS)
 
 test-mxfp4-rocm: tests/test_mxfp4_rocm
@@ -454,7 +835,65 @@ ds4_rocm_compat.o: ds4_rocm_compat.cu ds4_gpu.h ds4_gpu_mgpu.h ds4_gpu_args.h
 ds4_rocm_unavailable.o: ds4_rocm_unavailable.cu
 	$(HIPCC) $(ROCM_CFLAGS) -c -o $@ ds4_rocm_unavailable.cu
 
-tests/cuda_long_context_smoke: tests/cuda_long_context_smoke.o ds4_cuda.o $(MMQ_OBJS)
+tests/test_rocm_q4_dense_pair.o: tests/test_rocm_q4_dense_pair.cpp ds4_gpu.h
+	$(HIPCC) $(ROCM_CFLAGS) -DDS4_ROCM_BUILD -std=c++17 -fno-fast-math -I. -c -o $@ $<
+
+tests/test_rocm_q4_dense_pair: tests/test_rocm_q4_dense_pair.o ds4_image.o ds4_rocm.o $(ROCM_MMQ_OBJS) ds4_rocm_compat.o ds4_rocm_unavailable.o
+	$(HIPCC) $(ROCM_CFLAGS) -o $@ $^ $(ROCM_LDLIBS)
+
+# Keep the public test target usable on development hosts without ROCm.  The
+# binary itself exits 77 when HIP is installed but no device is visible; an
+# explicitly required Strix run converts that condition into a hard failure.
+ROCM_Q4_TEST_ARGS ?= --all
+test-rocm-q4-parity:
+	@rocm_test_hipcc="$(strip $(HIPCC))"; \
+	if [ -z "$$rocm_test_hipcc" ]; then \
+		rocm_test_hipcc="$$(command -v hipcc 2>/dev/null || true)"; \
+	fi; \
+	rocm_test_probe="$${rocm_test_hipcc%% *}"; \
+	if [ -z "$$rocm_test_probe" ] || ! command -v "$$rocm_test_probe" >/dev/null 2>&1; then \
+		if [ -n "$(strip $(DS4_TEST_REQUIRE_ROCM_DEVICE))" ] && [ "$(strip $(DS4_TEST_REQUIRE_ROCM_DEVICE))" != "0" ]; then \
+			echo "ROCm Q4 dense/pair/prefill oracle: FAIL (hipcc not found, device required)"; \
+			exit 1; \
+		fi; \
+		echo "ROCm Q4 dense/pair/prefill oracle: SKIP (hipcc not found)"; exit 0; \
+	fi; \
+	$(MAKE) --no-print-directory tests/test_rocm_q4_dense_pair HIPCC="$$rocm_test_hipcc" || exit $$?; \
+	if [ -n "$(strip $(DS4_TEST_REQUIRE_ROCM_DEVICE))" ] && [ "$(strip $(DS4_TEST_REQUIRE_ROCM_DEVICE))" != "0" ]; then \
+		DS4_TEST_REQUIRE_ROCM_DEVICE="$(strip $(DS4_TEST_REQUIRE_ROCM_DEVICE))" \
+			./tests/test_rocm_q4_dense_pair $(ROCM_Q4_TEST_ARGS); \
+	else \
+		env -u DS4_TEST_REQUIRE_ROCM_DEVICE \
+			./tests/test_rocm_q4_dense_pair $(ROCM_Q4_TEST_ARGS); \
+	fi; \
+	rc=$$?; \
+	if [ $$rc -eq 77 ]; then \
+		echo "ROCm Q4 dense/pair/prefill oracle: SKIP (no visible HIP device)"; \
+		exit 0; \
+	fi; \
+	exit $$rc
+
+test-rocm-q4-dense:
+	$(MAKE) --no-print-directory test-rocm-q4-parity ROCM_Q4_TEST_ARGS=--dense
+
+test-rocm-q4-pair:
+	$(MAKE) --no-print-directory test-rocm-q4-parity ROCM_Q4_TEST_ARGS=--pair
+
+test-rocm-q4-prefill:
+	$(MAKE) --no-print-directory test-rocm-q4-parity ROCM_Q4_TEST_ARGS=--prefill
+
+test-strix-rocm-q4-parity:
+	$(MAKE) --no-print-directory -B test-rocm-q4-parity ROCM_ARCH=gfx1151 DS4_TEST_REQUIRE_ROCM_DEVICE=1
+
+test-strix-rocm-q4-prefill:
+	$(MAKE) --no-print-directory -B test-rocm-q4-parity ROCM_ARCH=gfx1151 \
+		DS4_TEST_REQUIRE_ROCM_DEVICE=1 ROCM_Q4_TEST_ARGS=--prefill
+
+test-strix-rocm-q4-prefill-long:
+	$(MAKE) --no-print-directory -B test-rocm-q4-parity ROCM_ARCH=gfx1151 \
+		DS4_TEST_REQUIRE_ROCM_DEVICE=1 ROCM_Q4_TEST_ARGS=--prefill-long
+
+tests/cuda_long_context_smoke: tests/cuda_long_context_smoke.o ds4_image.o ds4_cuda.o $(MMQ_OBJS)
 	$(NVCC) $(NVCCFLAGS) -o $@ $^ $(CUDA_LDLIBS)
 
 tests/test_layer_pack.o: tests/test_layer_pack.c ds4_layer_pack.h
@@ -488,22 +927,22 @@ ifneq ($(UNAME_S),Darwin)
 tests/test_gpu_xdev.o: tests/test_gpu_xdev.c ds4_gpu.h ds4_gpu_mgpu.h
 	$(CC) $(CFLAGS) -I. -I$(CUDA_HOME)/include -c -o $@ $<
 
-tests/test_gpu_xdev: tests/test_gpu_xdev.o ds4_cuda.o $(MMQ_OBJS)
+tests/test_gpu_xdev: tests/test_gpu_xdev.o ds4_image.o ds4_cuda.o $(MMQ_OBJS)
 	$(NVCC) $(NVCCFLAGS) -o $@ $^ $(CUDA_LDLIBS)
 
 tests/test_gpu_model_cache.o: tests/test_gpu_model_cache.c ds4_gpu.h
 	$(CC) $(CFLAGS) -I. -I$(CUDA_HOME)/include -c -o $@ $<
 
-tests/test_gpu_model_cache: tests/test_gpu_model_cache.o ds4_cuda.o $(MMQ_OBJS)
+tests/test_gpu_model_cache: tests/test_gpu_model_cache.o ds4_image.o ds4_cuda.o $(MMQ_OBJS)
 	$(NVCC) $(NVCCFLAGS) -o $@ $^ $(CUDA_LDLIBS)
 
 tests/test_gpu_lookup_cache_strict.o: tests/test_gpu_lookup_cache_strict.c ds4_gpu.h ds4_gpu_mgpu.h
 	$(CC) $(CFLAGS) -I. -I$(CUDA_HOME)/include -c -o $@ $<
 
-tests/test_gpu_lookup_cache_strict: tests/test_gpu_lookup_cache_strict.o ds4_cuda.o $(MMQ_OBJS)
+tests/test_gpu_lookup_cache_strict: tests/test_gpu_lookup_cache_strict.o ds4_image.o ds4_cuda.o $(MMQ_OBJS)
 	$(NVCC) $(NVCCFLAGS) -o $@ $^ $(CUDA_LDLIBS)
 
-ds4_cuda_test_hooks.o: ds4.c ds4.h ds4_gpu.h ds4_gpu_mgpu.h ds4_layer_pack.h
+ds4_cuda_test_hooks.o: ds4.c ds4.h ds4_gpu.h ds4_gpu_mgpu.h ds4_image.h ds4_layer_pack.h
 	$(CC) $(CFLAGS) -Wno-unused-function -DDS4_TEST_HOOKS -I$(CUDA_HOME)/include -c -o $@ ds4.c
 
 tests/test_engine_mgpu_refusal.o: tests/test_engine_mgpu_refusal.c ds4.h ds4_gpu_mgpu.h
@@ -515,7 +954,7 @@ tests/test_engine_mgpu_refusal: tests/test_engine_mgpu_refusal.o ds4_gpu_args.o
 tests/test_engine_mgpu_runtime.o: tests/test_engine_mgpu_runtime.c ds4.h ds4_gpu_mgpu.h
 	$(CC) $(CFLAGS) -DDS4_TEST_HOOKS -I. -I$(CUDA_HOME)/include -c -o $@ $<
 
-tests/test_engine_mgpu_runtime: tests/test_engine_mgpu_runtime.o ds4_cuda_test_hooks.o ds4_gpu_args.o ds4_kvstore.o rax.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_cuda.o ds4_layer_pack.o $(MMQ_OBJS)
+tests/test_engine_mgpu_runtime: tests/test_engine_mgpu_runtime.o ds4_cuda_test_hooks.o ds4_gpu_args.o ds4_kvstore.o rax.o ds4_image.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_cuda.o ds4_layer_pack.o $(MMQ_OBJS)
 	$(NVCC) $(NVCCFLAGS) -o $@ $^ $(CUDA_LDLIBS)
 
 tests/test_engine_correctness.o: tests/test_engine_correctness.c ds4.h ds4_gpu_mgpu.h
@@ -536,7 +975,7 @@ test-cuda-session-batch: tests/test_cuda_session_batch
 tests/test_cuda_mixed_batch.o: tests/test_cuda_mixed_batch.c ds4.h ds4_gpu_args.h ds4_gpu_mgpu.h
 	$(CC) $(CFLAGS) -DDS4_TEST_HOOKS -I. -I$(CUDA_HOME)/include -c -o $@ $<
 
-tests/test_cuda_mixed_batch: tests/test_cuda_mixed_batch.o ds4_cuda_test_hooks.o ds4_gpu_args.o ds4_kvstore.o rax.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_cuda.o ds4_layer_pack.o $(MMQ_OBJS)
+tests/test_cuda_mixed_batch: tests/test_cuda_mixed_batch.o ds4_cuda_test_hooks.o ds4_gpu_args.o ds4_kvstore.o rax.o ds4_image.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_cuda.o ds4_layer_pack.o $(MMQ_OBJS)
 	$(NVCC) $(NVCCFLAGS) -o $@ $^ $(CUDA_LDLIBS)
 
 test-cuda-mixed-batch: tests/test_cuda_mixed_batch
@@ -565,18 +1004,27 @@ tests/test_prompt_prefix: tests/test_prompt_prefix.o ds4_prompt_prefix.o
 
 test: ds4_test ds4_agent_test ds4-eval q4k-dot-test mxfp4-dot-test \
 	tests/test_layer_pack tests/test_engine_mgpu_placement tests/test_gpu_args \
+	tests/test_quantizer_indexer_q4 gguf-tools/deepseek4-quantize \
 	tests/test_deepseek4_vision_image tests/test_prompt_prefix $(SAMPLING_TEST) ds4 ds4-server ds4-bench ds4-agent
 	./ds4-eval --self-test-extractors
 	./ds4_agent_test
-	./ds4_test
+	# Avoid adding the Q4 resident sidecar's 2.69 GiB to this broad suite.
+	# This does not enable SSD streaming; use `make test-ssd` for oversized models.
+	DS4_METAL_DISABLE_Q4_ATTN_Q_B_F16_CACHE=1 ./ds4_test
 	./tests/test_layer_pack
 	./tests/test_engine_mgpu_placement
 	./tests/test_gpu_args
 	./tests/test_gpu_args_cli.sh
 	./tests/test_prompt_prefix
 	./tests/test_sampling
+	./tests/test_quantizer_indexer_q4 ./gguf-tools/deepseek4-quantize
 	./tests/test_deepseek4_vision_image
 
+test-ssd:
+	DS4_TEST_SSD_STREAMING=1 \
+	DS4_TEST_SSD_STREAMING_COLD=1 \
+	$(MAKE) test
+
 dspark-acceptance: ds4
 	DS4_DSPARK_MODEL="$(DS4_DSPARK_MODEL)" \
 	DS4_DSPARK_SUPPORT="$(DS4_DSPARK_SUPPORT)" \
@@ -611,4 +1059,5 @@ mxfp4-dot-test: tests/test_mxfp4_dot.c
 	./tests/test_mxfp4_dot
 
 clean:
-	rm -f ds4 ds4-server ds4-bench ds4-eval ds4-agent ds4_cpu ds4_native ds4_server_test ds4_test ds4_agent_test gguf-tools/quality-testing/score_official gguf-tools/quality-testing/score_official.o speed-bench/metal_decode_schedule_bench speed-bench/metal_prefill_variant_bench speed-bench/*.o tests/test_q4k_dot tests/test_mxfp4_dot tests/test_mxfp4_metal tests/test_mxfp4_rocm tests/test_mxfp4_cuda tests/test_metal_session_batch tests/test_glm53_kda tests/test_glm53_kda_rocm tests/test_glm53_vision_engine tests/test_glm53_vision_prompt tests/test_deepseek4_vision_image tests/test_prompt_prefix tests/test_gpu_xdev tests/test_gpu_model_cache tests/test_gpu_lookup_cache_strict tests/test_engine_mgpu_refusal tests/test_engine_mgpu_runtime tests/test_engine_correctness tests/test_sampling tests/test_cuda_session_batch tests/test_cuda_mixed_batch tests/*.o *.o tests/cuda_long_context_smoke tests/cuda_long_context_smoke.o
+	rm -f speed-bench/metal_iq2_moe_top8_pair_bench
+	rm -f ds4 ds4-server ds4-bench ds4-eval ds4-agent ds4_cpu ds4_native ds4_server_test ds4_test ds4_agent_test gguf-tools/quality-testing/score_official gguf-tools/quality-testing/score_official.o speed-bench/metal_decode_schedule_bench speed-bench/metal_prefill_variant_bench speed-bench/metal_q4_dense_pair_bench speed-bench/metal_q4_prefill_pair_bench speed-bench/metal_q4_mm_tail_cull_bench speed-bench/metal_q4_attn_out_a_direct_bench speed-bench/metal_iq2_moe_tail_cull_bench speed-bench/gpu_iq2_moe_prefill_bench_rocm speed-bench/gpu_iq2_moe_prefill_bench_cuda speed-bench/rocm_q4_prefill_bench speed-bench/cuda_q4_prefill_bench speed-bench/*.o tests/test_q4k_dot tests/test_mxfp4_dot tests/test_quantizer_indexer_q4 tests/test_mxfp4_metal tests/test_mxfp4_rocm tests/bench_mxfp4_rocm tests/test_mxfp4_cuda tests/test_rocm_q4_dense_pair tests/test_metal_session_batch tests/test_metal_q4_streams tests/test_metal_q4_prefill_pair tests/test_metal_indexer_q4 tests/test_metal_q4_attn_exactn tests/test_metal_q4_attn_out_a_direct tests/test_metal_q4_qb_f16_cache tests/test_metal_exactn_oracle tests/test_metal_dspark_capture tests/test_metal_argmax_top1 tests/test_metal_iq2_midonly tests/test_metal_iq2_ssd_grouped_mm tests/test_metal_iq2_live_index tests/test_glm53_kda tests/test_glm53_kda_rocm tests/test_glm53_vision_engine tests/test_glm53_vision_prompt tests/test_deepseek4_vision_image tests/test_prompt_prefix tests/test_gpu_xdev tests/test_gpu_model_cache tests/test_gpu_lookup_cache_strict tests/test_engine_mgpu_refusal tests/test_engine_mgpu_runtime tests/test_engine_correctness tests/test_sampling tests/test_cuda_session_batch tests/test_cuda_mixed_batch tests/*.o *.o cuda/mmq/*.o cuda/mmq/test/*.o tests/cuda_long_context_smoke tests/cuda_long_context_smoke.o
diff --git a/QA_BEFORE_RELEASES.md b/QA_BEFORE_RELEASES.md
index 746db73e5c..c25d39b3f2 100644
--- a/QA_BEFORE_RELEASES.md
+++ b/QA_BEFORE_RELEASES.md
@@ -232,8 +232,8 @@ Use the normal Flash GGUF that 128 GB users run.
 ### DSpark / DeepSpec Runtime
 
 DSpark is opt-in, but it mutates the verifier, target-hidden capture, support
-model loading, and scheduler paths.  Run these whenever DSpark support,
-speculative verification, confidence/scheduler policy, target hidden capture,
+model loading, and proposal paths. Run these whenever DSpark support,
+speculative verification, confidence policy, target hidden capture,
 tiny routed-MoE verifier kernels, or shared `--mtp-model` support-model code changes:
 
 Use the 0731 DSpark support GGUF only with a Flash 0731 target. A support model
@@ -267,6 +267,247 @@ than a failure. `--dspark-strict` remains the byte-identical target-only mode.
   `DS4_DSPARK_FIXTURE_CONFIDENCE=0 DS4_DSPARK_FIXTURE_TOKENS=8 DS4_DSPARK_FIXTURE_REQUIRE_PARTIAL=1 DS4_DSPARK_MODEL=/Users/antirez/ds4/gguf/DeepSeek-V4-Flash-IQ2XXS-w2Q2K-AProjQ8-SExpQ8-OutQ8-chat-v2-imatrix-0731.gguf DS4_DSPARK_SUPPORT=/Users/antirez/ds4/gguf/DeepSeek-V4-Flash-DSpark-support-0731.gguf make dspark-acceptance`.
 - DSpark verifier invariant smoke:
   `DS4_TEST_MODEL=/Users/antirez/ds4/gguf/DeepSeek-V4-Flash-IQ2XXS-w2Q2K-AProjQ8-SExpQ8-OutQ8-chat-v2-imatrix-0731.gguf DS4_DSPARK_SUPPORT=/Users/antirez/ds4/gguf/DeepSeek-V4-Flash-DSpark-support-0731.gguf make dspark-verify-depth`.
+- For Metal DSpark verifier/proposer/replay changes, run this same-machine A/B
+  matrix with `DS4_DSPARK_STATS=1`, greedy decoding, the same prompt and token
+  limit, and no other environment changes:
+
+  | Target expert cache | Expected DSpark depth | Legacy control | Candidate |
+  | ---: | ---: | --- | --- |
+  | 16 | 2 | `DS4_METAL_DSPARK_PROPOSER_BLOCK_MAX=0 DS4_METAL_DSPARK_ACCEPTANCE_ONLY_VERIFY=0 DS4_METAL_DSPARK_HEADLESS_REPLAY=0` | Leave proposer/headless unset; keep acceptance-only `=0` |
+  | 32 | 5 | `DS4_DSPARK_SSD_VERIFY_BLOCK_MAX=5 DS4_METAL_DSPARK_ACCEPTANCE_ONLY_VERIFY=0 DS4_METAL_DSPARK_HEADLESS_REPLAY=0` | Keep the verifier cap, set acceptance-only `=1`, and leave proposer/headless unset |
+
+  Use `--ssd-streaming-cache-experts 16` or `32` to match the row. The 0731
+  top-6 verifier needs 30 effective slots for five draft rows; 32 leaves a small
+  margin. Require byte-identical stdout between control and candidate and
+  `errors=0`, `verifier_unavailable=0`, `proposed>0`, and
+  `accepted_draft>0`. Record generation t/s, acceptance, `propose`, `verify`,
+  `replay`, `prop_capped`, `prop_scheduled_rows`, `metal_accept_only`,
+  `metal_verify_rows_saved`, and `metal_replay_headless`. In the candidate,
+  eligible `N >= 3` verification cycles should save one target row; aligned
+  ratio-4 boundaries intentionally remain on the legacy path. The
+  depth-2 run exercises proposer capping and headless replay while retaining
+  the legacy verifier; the depth-5 run should retain the checkpoint's native
+  five proposal rows and exercise acceptance-only verification.
+  On low-memory Metal, repeat the depth-5 candidate once with
+  `DS4_METAL_DSPARK_PIN_MAIN_PROJ=1`. Require a startup log confirming the
+  locked byte count, identical stdout and acceptance, and compare
+  `prop_setup`, total `propose`, page faults, and generation t/s. A lock
+  failure or a slower median keeps this optimization opt-in.
+- For the experimental Metal SSD exact-2 verifier, repeat the depth-2 row
+  above with `DS4_METAL_DSPARK_EXACT2=0` as the control and `=1` as the only
+  candidate change. Set `DS4_DSPARK_FIXTURE_REQUIRE_EXACT2=1` only on the
+  candidate. Require byte-identical stdout against both control and
+  target-only output, `exact2_attempt>0`, `exact2_full>0`,
+  `exact2_fallback=0`, and `errors=0`. Record generation t/s, `verify`, and
+  `replay`; then repeat for at least 100 generated tokens to catch cumulative
+  state drift. Do not infer that the generic five-row batch state is directly
+  committable from this two-row result.
+- For Metal exact-union or the AProjQ4/HC decode fusions, first run the
+  model-backed oracle with the target AProjQ4 GGUF:
+  `DS4_TEST_MODEL=/path/to/deepseek-v4-flash-aprojq4.gguf make test-metal-exactn-oracle`.
+  Require its N=2..5 cases to be byte-identical to sequential decode for
+  serialized KV/compressor state, logits, and the four-token continuation.
+  The matrix must include full accepts for N=2,3,4,5, all N=5 partial prefixes
+  1..4, and EOS in the first and a middle row. This is a correctness gate, not
+  evidence of a speedup.
+- The Q8 Q-A/KV compound rows below require a separate AProjQ8 target whose
+  metadata includes both ratio-4 and ratio-128 compressor layers. An AProjQ4
+  oracle cannot exercise that compound and is a failed coverage gate even if
+  greedy output remains correct.
+- Then run isolated, same-machine greedy A/B pairs with identical prompt,
+  context, cache, token limit, and `DS4_DSPARK_STATS=1`. Change only the gate
+  named by the row:
+
+  | Metal fusion | Reference control | Candidate |
+  | --- | --- | --- |
+  | HC RMSNorm + F16 mixer on M1-M4 | `DS4_METAL_DISABLE_PRE_M5_HC_NORM_MIX_FUSE=1` | Leave the disable switch unset |
+  | HC RMSNorm + F16 mixer on another Apple generation | Leave both HC norm/mix switches unset | `DS4_METAL_ENABLE_HC_NORM_MIX_FUSE=1` |
+  | HC producer + split/Sinkhorn/destination RMSNorm on M1-M5 | `DS4_METAL_DISABLE_HC_PRODUCER_PRE_NORM_FUSE=1` | Leave the disable switch unset |
+  | Q4 Q-A/KV + compressor store in exact-union | `DS4_METAL_DSPARK_EXACTN_UNION=1` with the Q4 enable switch unset | Keep exact-union `=1`; set `DS4_METAL_ENABLE_Q4_QKV_COMPRESSOR_FUSE=1` |
+  | Q4 Q-A/KV + compressor store in ordinary `FULL` decode | Leave `DS4_METAL_ENABLE_Q4_QKV_COMPRESSOR_FUSE` unset | Set `DS4_METAL_ENABLE_Q4_QKV_COMPRESSOR_FUSE=1` |
+  | Q8 Q-A/KV + compressor store in SSD `FULL` (AProjQ8) | Leave the Q8 enable/require switches unset | Set `DS4_METAL_ENABLE_Q8_QKV_COMPRESSOR_FUSE=1 DS4_METAL_REQUIRE_Q8_QKV_COMPRESSOR_FUSE=1` |
+  | Q8 Q-A/KV + compressor store in SSD exact-union (AProjQ8) | `DS4_METAL_DSPARK_EXACTN_UNION=1` with the Q8 enable/require switches unset | Keep exact-union `=1`; set `DS4_METAL_ENABLE_Q8_QKV_COMPRESSOR_FUSE=1 DS4_METAL_REQUIRE_Q8_QKV_COMPRESSOR_FUSE=1` |
+  | Q4 attention-output tiny batch in the generic verifier | Set `DS4_METAL_DSPARK_EXACTN_UNION=0 DS4_METAL_DSPARK_EXACTN=0 DS4_METAL_DSPARK_EXACT2=0`; leave tiny enable/require unset | Keep all three exact gates `=0`; set `DS4_METAL_REQUIRE_Q4_ATTN_OUT_TINY_BATCH=1` and require at least one proposed block of depth 3–5 (the acceptance-only suffix evaluates one fewer row) |
+  | F16 attention+indexer quad compressor store in `FULL` decode | `DS4_METAL_DISABLE_COMPRESSOR_QUAD_STORE=1` | Leave the disable switch unset |
+  | F16 attention+indexer quad compressor store in exact-union | `DS4_METAL_DSPARK_EXACTN_UNION=1 DS4_METAL_DISABLE_COMPRESSOR_QUAD_STORE=1` | Keep exact-union `=1`; leave the quad disable switch unset |
+  | Exact ratio-4 one-row compressor pool on M1-M5 | `DS4_METAL_DISABLE_COMPRESSOR_EXACT_POOL_RATIO4=1` | Leave the disable switch unset |
+  | Q4 attention-output B + HC expansion | `DS4_METAL_DISABLE_Q4_ATTN_OUT_HC_FUSE=1` | Leave the disable switch unset |
+  | FlashAttention pad/block PSO memo | `DS4_METAL_DISABLE_PRE_M5_FLASH_ATTN_PAD_BLK_MEMO=1` | Leave the disable switch unset |
+  | FlashAttention batched/vector PSO memo | `DS4_METAL_DISABLE_PRE_M5_FLASH_ATTN_BATCHED_MEMO=1` | Leave the disable switch unset |
+  | Exact-union asynchronous routed tails | `DS4_METAL_DSPARK_EXACTN_UNION=1` with `DS4_METAL_DSPARK_EXACT_ROWS_ASYNC_TAILS` unset | Keep exact-union `=1`; set `DS4_METAL_DSPARK_EXACT_ROWS_ASYNC_TAILS=1` |
+
+  The Q4 Q-A/KV compound is opt-in in both exact-union and ordinary `FULL`
+  decode; it is enabled only when the explicit enable variable is present.
+  Require byte-identical stdout and `errors=0`; for exact-union also
+  require `exactn_union_attempt>0` and `exactn_union_error_fallback=0`.
+  Partial-accept fallback is expected when the draft diverges. Record
+  `exactn_union_full`, `exactn_union_partial_fallback`, `propose`, `verify`,
+  `replay`, stage timings, page faults, and generation t/s. A candidate that
+  is correct but slower remains disabled or opt-in according to its gate.
+  For asynchronous tails, also repeat the model-backed oracle with the switch
+  set and run enough exact-union cycles to cross cache eviction and raw-ring
+  wrap boundaries. The candidate removes a CPU wait but retains private expert
+  buffers until command-buffer completion; serialized state and process memory
+  after synchronization must match the synchronous control.
+  Before the model-backed runs, build `ds4_test` and run
+  `./ds4_test --metal-kernels`. This covers the isolated compound HC, F16 quad
+  compressor-store, exact ratio-4 pool, and tie-heavy Metal routing kernels.
+  For the exact one-row pool candidate, repeat once with
+  `DS4_METAL_REQUIRE_COMPRESSOR_EXACT_POOL_RATIO4=1`; the run must exercise the
+  specialization instead of silently falling back. Also exercise the global
+  kill switches plus the matching pre-M5 or M5 HC/pool rollback on the target
+  machine. Treat the FlashAttention memo rows as host-dispatch A/B tests: the
+  selected specialization and output must remain identical, and any timing
+  comparison must use repeated warm runs.
+- For the default M1 IQ2 address-table mid-only path, first run
+  `make test-metal-iq2-midonly`. It must cover 12,288 full-shape top-6 mid
+  words in both unmasked and complementary masked address-table modes with
+  both mid mismatch counters at zero, no canonical unwritten rows, zero
+  candidate gate/up writes, and zero guard mismatches. Then use the same greedy
+  IQ2_XXS/Q2_K SSD-streaming model, prompt, cache state, and token count for
+  three decode runs: leave all switches unset for the automatic candidate,
+  set `DS4_METAL_REQUIRE_M1_IQ2_MID_ONLY=1` for fail-closed coverage, and set
+  `DS4_METAL_DISABLE_M1_IQ2_MID_ONLY=1` for the canonical control and
+  kill-switch fallback.
+  Enable the routed-MoE stage profiler on one candidate layer and require path
+  `iq2_stream_addr_mid_only_4096x2048` or
+  `iq2_stream_addr_mask_mid_only_4096x2048`; absence of both is failed model
+  coverage. Require byte-identical greedy output and top-logprobs, and report
+  prefill separately from decode: this one-token routed producer is not a
+  prefill optimization. Compare repeated hot-cache medians, then repeat a
+  cold-cache sanity run to exclude a change in SSD cache behavior.
+- For the removed Metal 512-column streaming top-k path, there is no runtime
+  candidate gate. Compare the current binary with a build immediately before
+  its removal only if historical timing is needed. First require
+  `./ds4_test --metal-kernels` to pass, including tie-heavy routing cases, then
+  require identical selected expert ids and greedy output. Correct deterministic
+  ordering takes precedence over a timing difference.
+- For the default CPU unrolled argmax, run `tests/test_sampling`, then compare
+  an otherwise identical greedy workload with
+  `DS4_CPU_DISABLE_UNROLLED_ARGMAX=1` (scalar control) and with the variable
+  unset (candidate). Require identical tokens for ordinary, excluded-id,
+  cross-lane-tie, and vocabulary-tail cases; record median generation t/s over
+  repeated runs without claiming a speedup from the implementation alone.
+- For the experimental resident-CUDA exact-2 verifier, use three controlled
+  runs with verifier cap two on the same single-GPU host: native proposer plus
+  legacy verifier
+  (`DS4_CUDA_DSPARK_EXACT2=0 DS4_CUDA_DSPARK_PROPOSER_BLOCK_MAX=0 DS4_DSPARK_SSD_VERIFY_BLOCK_MAX=2`),
+  two-row proposer plus legacy verifier
+  (`DS4_CUDA_DSPARK_EXACT2=0 DS4_CUDA_DSPARK_PROPOSER_BLOCK_MAX=2 DS4_DSPARK_SSD_VERIFY_BLOCK_MAX=2`),
+  and two-row proposer plus exact-2
+  (`DS4_CUDA_DSPARK_EXACT2=1 DS4_CUDA_DSPARK_PROPOSER_BLOCK_MAX=2 DS4_DSPARK_SSD_VERIFY_BLOCK_MAX=2`).
+  This separates the non-causal proposer-width change from the verifier and
+  replay change. Then compare uncapped legacy DSpark against exact-2 as an
+  end-to-end policy test. Keep SSD streaming and TP disabled. Require
+  byte-identical stdout, `errors=0`, and `verifier_unavailable=0` from every
+  run; set `DS4_DSPARK_FIXTURE_REQUIRE_EXACT2=1` on the exact-2 run so the
+  fixture enforces `exact2_attempt>0` and `exact2_fallback=0`.
+  Record `prop_scheduled_rows/cycles`, `propose`, `verify`, `replay`, `net_saved`,
+  `miss_first`, `no_draft`, `avg_accept`, and generation t/s from every run.
+- For resident CUDA exact-N, keep exact-2 disabled and compare
+  `DS4_CUDA_DSPARK_EXACTN=0` against `=1` with the native five-row proposer
+  and `DS4_DSPARK_SSD_VERIFY_BLOCK_MAX=5`. Repeat N=2,3,4,5 with explicit
+  proposer/verifier caps, then exercise the kill switch with both
+  `DS4_CUDA_DSPARK_EXACTN=1` and
+  `DS4_CUDA_DISABLE_DSPARK_EXACTN=1`. Require byte-identical greedy stdout,
+  `errors=0`, `verifier_unavailable=0`, `cuda_exactn_attempt>0`, and at least
+  one `cuda_exactn_full`; partial cases must increment the partial and
+  aggregate fallback counters, never the error counter, and continue
+  identically through legacy replay. Include
+  EOS as the first and a middle draft, a raw-ring wrap boundary, a context
+  capacity cut, and prefill workspaces below five rows. Record
+  `cuda_exactn_rows`, its full/partial/error counters, `snapshot`, `verify`,
+  `replay`, acceptance, and generation t/s. Run with CUDA decode graphs both
+  enabled and disabled. Do not promote the gate without a CUDA device build
+  and serialized KV/compressor-state oracle; host syntax tests do not execute
+  this path. On candidate fixture runs set
+  `DS4_DSPARK_FIXTURE_REQUIRE_CUDA_EXACTN=1`; it requires aggregate
+  `cuda_exactn_attempt>0` and `cuda_exactn_error_fallback=0`. It reports but
+  does not reject aggregate `cuda_exactn_fallback`, because valid partial
+  matches increment both the partial and aggregate fallback counters before
+  legacy replay.
+- For CUDA DSpark non-causal proposer attention, compare the reference with
+  `DS4_CUDA_ENABLE_DSPARK_NONCAUSAL_ONLINE=0` against the candidate with `=1`.
+  Repeat at proposal depths two and five, across every raw-ring start index,
+  and once with both the enable variable and
+  `DS4_CUDA_DISABLE_DSPARK_NONCAUSAL_ONLINE=1` to prove the kill switch restores
+  the reference dispatch. On the short diagnostic runs also set
+  `DS4_DSPARK_VERIFY_NONCAUSAL=1`; record all three reported `max_abs` and
+  `max_rel` comparisons and reject non-finite values or a material error
+  regression. Then run the acceptance fixture without the diagnostic host
+  readbacks and require byte-identical target stdout, `errors=0`, and
+  `verifier_unavailable=0`. Record proposal time, acceptance, generation t/s,
+  and the startup dispatch log. Draft logits or acceptance may differ slightly
+  because online softmax changes the floating-point reduction order; that is
+  not permission for the verified target continuation to differ.
+- For CUDA HC and tiny routed-MoE kernel changes, keep
+  `DS4_CUDA_DSPARK_EXACT2` unset and repeat the resident acceptance fixture
+  with these explicit A/B pairs: HC control
+  `DS4_CUDA_DISABLE_HC_SPLIT_NORM_FUSED=1` versus candidate with that variable
+  absent; routed-MoE control `DS4_CUDA_DSPARK_TINY_ALIGNED_VEC=0` versus
+  candidate `=1`. Require byte-identical stdout, `errors=0`, and
+  `verifier_unavailable=0`; record `prop_chain`, `verify_layer`, total
+  proposal/verify time, acceptance, and generation t/s. Also run
+  `--decode-consistency 64` and the logprob-vector regression before enabling
+  a numerically different kernel by default.
+- For the CUDA AProjQ4 ports, run an isolated A/B for each dispatch:
+  Q-A/KV pair control `DS4_CUDA_DISABLE_Q4_DENSE_PAIR=1` versus candidate with
+  that variable absent; HC norm/mix control
+  `DS4_CUDA_DISABLE_HC_NORM_MIX_FUSE=1` versus candidate
+  `DS4_CUDA_ENABLE_HC_NORM_MIX_FUSE=1 DS4_CUDA_NO_F16_CUBLAS_ONE=1`; and Q4
+  attention-output/HC control `DS4_CUDA_DISABLE_Q4_ATTN_OUT_HC_FUSE=1` versus
+  the graph-compatible canonical candidate with that variable absent. Run a separate
+  non-captured diagnostic with `DS4_CUDA_Q4_ATTN_OUT_HC_ORACLE=1`; require
+  the summary to be present with `calls>0`, `skips=0`, and
+  `epilogue_mismatches=0`, while `q8k_mismatches` records the expected
+  numerical distance from the optional one-dispatch Q8_K experiment. A
+  zero-call summary is a failed coverage gate. Only
+  test `DS4_CUDA_Q4_ATTN_OUT_HC_Q8K_EXPERIMENT=1` as a promotion candidate if
+  its oracle mismatches are also zero. Repeat the pair and attention-output
+  cases with `DS4_CUDA_MMQ=0` to exercise the canonical Q8_K fallback
+  separately from the default MMVQ/Q8_1 path. Require byte-identical stdout
+  and full-logit/tensor equivalence before promoting an opt-in gate. Run with
+  decode graphs both enabled and disabled, and record target, proposer,
+  verifier, replay, acceptance, and generation t/s. A CUDA build and hardware
+  run are mandatory; a host-only build does not compile the device kernels.
+- When DSpark, support-model mapping, or SSD streaming changes, repeat both
+  the acceptance fixture and verifier invariant on every advertised graph
+  backend. Apply the backend and SSD options to the target-only baseline as
+  well as the DSpark run:
+
+  ```sh
+  DS4_DSPARK_MODEL=/path/to/flash-0731.gguf \
+  DS4_DSPARK_SUPPORT=/path/to/DeepSeek-V4-Flash-DSpark-support-0731.gguf \
+  DS4_DSPARK_FIXTURE_BACKEND=cuda \
+  DS4_DSPARK_FIXTURE_SSD_STREAMING=1 \
+  DS4_DSPARK_FIXTURE_SSD_STREAMING_CACHE_EXPERTS=32 \
+  DS4_DSPARK_FIXTURE_CONFIDENCE=0 \
+  make dspark-acceptance
+
+  DS4_TEST_MODEL=/path/to/flash-0731.gguf \
+  DS4_DSPARK_SUPPORT=/path/to/DeepSeek-V4-Flash-DSpark-support-0731.gguf \
+  DS4_TEST_SSD_STREAMING=1 \
+  DS4_TEST_SSD_STREAMING_CACHE_EXPERTS=32 \
+  make dspark-verify-depth
+  ```
+
+  On Strix Halo use the same variables with
+  `DS4_DSPARK_FIXTURE_BACKEND=rocm make rocm-dspark-acceptance` and
+  `make rocm-dspark-verify-depth`. Do not use the generic targets after a ROCm
+  build: on non-Apple hosts their default object set is CUDA. For the 0731
+  Flash layout, ROCm needs at least 30 expert slots; use 32 in release tests.
+- The fixture must report aggregate `proposed>0`, `accepted_draft>0`,
+  `verifier_unavailable=0`, and `errors=0`; stdout must remain byte-identical
+  to the target-only SSD baseline. The verifier smoke must report
+  `max_chunk>1`, `nspec>64`, and `worst_argmax_gap<=2`.
+- Preserve baseline and DSpark `generation` t/s from the same fixture run,
+  with the same host, model, cache, runtime settings, thermal state, and
+  background load. A DSpark path that is materially slower than the target-only
+  SSD path without a documented correctness tradeoff is a release blocker.
+- On ROCm, also run one `DS4_DSPARK_PROBE=1`
+  generation and require the non-causal attention and stage-chain probes to
+  pass. This covers the HIP draft-attention kernel before the end-to-end
+  verifier gate.
 - If shared support-model or verifier structures changed, also run legacy MTP:
   `make mtp-verify-depth` with `DS4_TEST_MTP` set to a one-stage MTP support
   GGUF, or confirm the target skips only because the optional file is missing.
@@ -274,7 +515,7 @@ than a failure. `--dspark-strict` remains the byte-identical target-only mode.
   `replay_fallbacks`, `errors=0`, `verify_layer`, `net_saved`, and
   `output_match` for both 32-token and 64-token runs. At least one direct commit
   must occur. A faster run with lower proposal quality is a regression unless
-  it was an intentional scheduler change.
+  it was an intentional confidence-policy change.
 - If verifier MoE kernels changed, run one diagnostic `c_add` profile with
   `DS4_DSPARK_VERIFY_SELECTED_PROFILE=1` or the Metal MoE stage profiler and
   record the selected-expert footprint or stage timing in the DSpark log.
@@ -737,6 +978,122 @@ Do not use high-performance Hugging Face Xet mode while vLLM is resident.
   receiving explicit permission to use `192.168.60.250` for this QA pass.
 - Run:
   `make cuda-regression`.
+- On a single GB10 (`sm_121`), validate the imported Q2 decode fast paths with
+  the AProjQ8/OutQ8 Flash GGUF.  Compare the default against a rollback process
+  that sets all of:
+  `DS4_CUDA_NO_DIRECT_Q2_PREFILL=1`,
+  `DS4_CUDA_NO_F16_PAIR_COMPRESSOR_STORE=1`,
+  `DS4_CUDA_NO_F16_PAIR_COMPRESSOR_TRANSPOSE=1`,
+  `DS4_CUDA_NO_F16_PAIR_COMPRESSOR_TRANSPOSE_PREFETCH8=1`,
+  `DS4_CUDA_NO_Q8_FUSED_ALIGNED=1`,
+  `DS4_CUDA_NO_Q8_ALIGNED_PERSISTENT=1`,
+  `DS4_CUDA_NO_Q8_ALIGNED_DENSE_SCRATCH=1`, and
+  `DS4_CUDA_NO_HC_SPLIT_NORM_SPLIT4096=1`.  Use separate processes, require
+  byte-identical greedy stdout and per-token logprobs, then run the same pair
+  under Compute Sanitizer.  Record prefill, decode, and steady decode rather
+  than copying the upstream PR numbers into a release claim.
+- Repeat the GB10 comparison with AProjQ4/OutQ8.  First run
+  `make test-mmq-parity-cuda CUDA_ARCH=sm_121`; its Q4 cases must report zero
+  bit mismatches for persistent scratch, grouped attention-A, and the
+  opt-in K1024 persistent kernel.  For the model A/B, use
+  `DS4_CUDA_NO_Q4_GB10_FAST=1` in the control and leave it unset in the
+  candidate. Run a separate generic-verifier candidate with
+  `DS4_CUDA_DISABLE_DSPARK_EXACTN=1`,
+  `DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_BATCH=1`,
+  `DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_BATCH=1`, and
+  `DS4_CUDA_Q4_GROUPED_ATTN_A_ORACLE=1`; require `batch_candidates>0`,
+  `batch_calls>0`, `batch_mismatches=0`, and `batch_skips=0`, plus
+  byte-identical stdout. Build the resident prefill harness and run
+  `./speed-bench/cuda_q4_prefill_bench --path mmq --case outa --tokens
+  127,128,129,257,512,2048,4096 --samples 16 --warmup 4`; require bitwise
+  equality between `pack8_mmq_unpack` and `grouped_8_grids`, finite/canary/CPU
+  oracle success, and record the paired median. Run
+  `make test-mmq-q4-grouped-q81-cuda CUDA_ARCH=sm_121`, then isolate the new
+  Q8_1 front-end with
+  `--grouped-q81-kernel --tokens 512,1024,2048,4096,6144,8192`; require
+  byte-identical direct parity, bitwise final output, and a repeatable paired
+  median win between `grouped_generic_q81` and
+  `grouped_k4096_g8x2_q81`. Then compare full-model
+  prefills with the default environment against the dominant
+  `DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL=1` rollback, and separately against
+  the narrow `DS4_CUDA_NO_Q4_GROUPED_ATTN_A_Q81=1` rollback. Keep the
+  single-grid and 16-warp experiments unset in this promotion comparison.
+  Benchmark the K1024
+  persistent kernel as a
+  separate fail-closed arm with both
+  `DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT=1` and
+  `DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT=1`; its rollback is
+  `DS4_CUDA_NO_Q4_K1024_PERSISTENT=1`. Then run a non-captured oracle process
+  with `DS4_CUDA_DECODE_GRAPHS=0`,
+  `DS4_CUDA_Q4_K1024_PERSISTENT_ORACLE=1`, and
+  `DS4_CUDA_Q4_K1024_PERSISTENT_STATS=1`; require `candidates>0`, `uses>0`,
+  `oracle_calls>0`, `oracle_mismatches=0`, and `oracle_skips=0`. The parity
+  test must also show a nonzero REQUIRE failure with the local kill set,
+  proving admission fails before enqueue, and a canonical reference forced by
+  that same kill. The counters are host dispatches and intentionally exclude
+  CUDA graph replays. The persistent OutQ8
+  vocabulary, compressor, HC split, direct routed-MoE paths, Q4 scratch,
+  grouped attention-A, and canonical B+HC epilogue remain relevant, while
+  the Q8-only attention-projection consumers are intentionally ineligible.
+- Validate the experimental HC-to-consumer Q8_1 producer fold in separate
+  processes. Use `DS4_CUDA_NO_Q8_FOLD=1` for the control and
+  `DS4_CUDA_ENABLE_Q8_FOLD=1` for the candidate, first with the normal graph
+  setting and then with `DS4_CUDA_DECODE_GRAPHS=0`. For the non-captured arm,
+  also set `DS4_CUDA_Q8_FOLD_ORACLE=1` and require `hits>0`, `byte_calls>0`,
+  `output_calls>0`, `byte_mismatches=0`, `output_mismatches=0`, and `skips=0`.
+  The reached consumer must be reported as aligned Q8 or IQ2 MoE rather than
+  inferred from producer counters alone. Require byte-identical greedy stdout
+  and per-token logprobs, then repeat the control/candidate pair under Compute
+  Sanitizer. Keep the oracle off for the graph-on timing arm: capture is an
+  intentional fail-closed miss and is checked for safety, not fold coverage.
+  Run these arms through the ordinary serialized inference dispatcher; the
+  opt-in fold does not support concurrent host-thread submission to one CUDA
+  stream.
+- If the umbrella AProjQ4 A/B changes logits, do not attribute that change to
+  "the Q4 fast path" as a unit. Run the fail-closed component matrix from a
+  clean `cuda-spark` build. The output directory is intentionally explicit so
+  the six independent-process arms, two oracle arms, raw logs, and diffs are
+  retained:
+
+  ```sh
+  make clean && make cuda-spark
+  make gguf-tools/quality-testing/score_official CUDA_ARCH=sm_121
+
+  DS4_CUDA_Q4_MATRIX_SSD_STREAMING=1 \
+  DS4_CUDA_Q4_MATRIX_SSD_CACHE=16GB \
+  DS4_CUDA_Q4_MATRIX_DECODE_GRAPHS=1 \
+  tests/cuda_q4_gb10_fast_matrix.sh \
+    /path/to/DeepSeek-V4-Flash-AProjQ4-OutQ8.gguf \
+    gguf-tools/quality-testing/data/flash/manifest.tsv \
+    /tmp/q4-gb10-graphs-on
+
+  DS4_CUDA_Q4_MATRIX_SSD_STREAMING=1 \
+  DS4_CUDA_Q4_MATRIX_SSD_CACHE=16GB \
+  DS4_CUDA_Q4_MATRIX_DECODE_GRAPHS=0 \
+  tests/cuda_q4_gb10_fast_matrix.sh \
+    /path/to/DeepSeek-V4-Flash-AProjQ4-OutQ8.gguf \
+    gguf-tools/quality-testing/data/flash/manifest.tsv \
+    /tmp/q4-gb10-graphs-off
+  ```
+
+  The matrix first proves that the three local rollback switches reproduce
+  `DS4_CUDA_NO_Q4_GB10_FAST=1`; failure of `local_control` means the matrix is
+  incomplete and no component claim is valid. It then enables exactly one of
+  persistent Q8_1 scratch, grouped attention-A, or the graph-compatible B+HC
+  call, with K1024 persistent kept disabled because it is a separate opt-in.
+  The grouped and HC oracle summaries must have `calls>0`, `skips=0`, and zero
+  relevant mismatches. `summary.txt` must say `promotion_gate=pass`. When it
+  is blocked, use the named `*_differences` arms and their `.diff` or
+  `.comparison.txt` files to identify a component; if no single arm differs
+  but `default_fast` does, report an interaction rather than blaming an
+  individual kernel. The tensor oracles and synthetic parity test are the
+  bit-exact component gates; the top-128 smoke dump and every scorer TSV row
+  are complementary end-to-end drift detectors, not a full-logit proof.
+- Exercise CUDA DSpark at verifier/proposer depth 5 with the fast paths enabled
+  and disabled.  Require identical final output, zero verifier errors, and
+  matching full/partial acceptance histograms.  Test both the generic batch
+  verifier (direct Q2 path) and CUDA exact-N (one-row decode paths); do not
+  infer speculative speedup from the target-only benchmark.
 - For native MXFP4 changes, run
   `make test-mxfp4-cuda CUDA_ARCH=native` on the multi-GPU CUDA host only after
   receiving explicit permission for `192.168.60.250`, and
@@ -811,6 +1168,20 @@ a substitute for CUDA or Metal release testing.
 - Do not use the mixed q2-q4 or Q4 Flash GGUFs for routine Strix Halo QA yet.
   They are dangerous on this machine for now because the ROCm path can hit
   system OOM instead of failing cleanly.
+- When ROCm Q4 code changes, run `make test-strix-rocm-q4-parity` and
+  `make test-strix-rocm-q4-prefill` before attempting a model. The prefill
+  oracle must compare the TILE8 default with
+  `DS4_ROCM_DISABLE_Q4_PREFILL_TILE8=1` at K=256, 1024, and 4096 and at token
+  counts covering a partial tile and the 128-token production chunk. Require
+  bitwise dense, pair, Q4-attention-B, and Q8-attention-B parity with intact
+  canaries. A REQUIRE-plus-DISABLE arm must fail before modifying output.
+- Keep ROCm grouped attention-A decode opt-in until a model A/B wins. Its
+  fail-closed test uses `DS4_ROCM_ENABLE_Q4_GROUPED_ATTN_A=1`,
+  `DS4_ROCM_REQUIRE_Q4_GROUPED_ATTN_A=1`, and
+  `DS4_ROCM_Q4_GROUPED_ATTN_A_STATS=1`; require dispatches and groups above
+  zero, with zero fallbacks/failures and bitwise equality to the per-group
+  reference. This synthetic coverage does not supersede the Q4-model OOM
+  warning above.
 - Run a short CLI prompt:
   `./ds4 -m gguf/DeepSeek-V4-Flash-IQ2XXS-w2Q2K-AProjQ8-SExpQ8-OutQ8-chat-v2-imatrix-0731.gguf --ctx 4096 --nothink -p "Reply with exactly: OK"`.
 - For DeepSeek Flash and GLM 5.3 Flash decode, confirm the default path uses
diff --git a/README.md b/README.md
index 503f00c630..07d8fe1761 100644
--- a/README.md
+++ b/README.md
@@ -87,6 +87,9 @@ next sections.
   guide for contributors. **Read this before sending a pull request**.
 - [QA_BEFORE_RELEASES.md](QA_BEFORE_RELEASES.md): the complete release test
   matrix, including the remote Metal, CUDA, and ROCm machines.
+- [ENVIRONMENT_VARIABLES.md](ENVIRONMENT_VARIABLES.md): complete runtime,
+  test, and tooling environment-variable inventory, with a curated quick
+  reference for supported rollback, fail-closed, and diagnostic switches.
 - [gguf-tools/README.md](gguf-tools/README.md): offline GGUF generation,
   imatrix collection, quantization tooling, and quality checks.
 - [gguf-tools/imatrix/README.md](gguf-tools/imatrix/README.md): how the
@@ -382,10 +385,11 @@ GGUF of about 5.6 GiB. It is not a standalone model. Download it once:
 The support file can be used with the 0731 Flash `ds4f-q2`, `ds4f-q2-q4`, and
 `ds4f-q4` models listed above. It is checkpoint-specific
 and must not be paired with an older Flash model. For now **DeepSeek V4 PRO**
-is not supported. On Metal, the main model may be resident or use
-`--ssd-streaming`; the support model still adds its own weights and runtime
-state to the memory requirement. DSpark replaces the legacy one-stage MTP
-support model for that run rather than stacking with it.
+is not supported. On Metal, CUDA, and ROCm, the main model may be resident or
+use `--ssd-streaming`; the support model is kept separately mapped or
+device-cached and adds its own weights and runtime state to the memory
+requirement. DSpark replaces the legacy one-stage MTP support model for that
+run rather than stacking with it.
 
 Run it with the normal sampling defaults:
 
@@ -428,13 +432,623 @@ The same DSpark flags work with `ds4-agent` and with non-batched
 `ds4-server` requests. Session-batched serving currently uses ordinary target
 decoding.
 
+On a single accelerator, the main model can instead stream its routed experts
+from SSD while the DSpark support model remains mapped or device-cached
+separately. On Metal the support mapping is file-backed and pageable; CUDA and
+ROCm prepare a separate device cache. Select the backend with `--metal`,
+`--cuda`, or `--rocm`:
+
+```sh
+./ds4 -m ds4flash.gguf \
+  --mtp-model gguf/DeepSeek-V4-Flash-DSpark-support-0731.gguf \
+  --dspark --metal --ssd-streaming \
+  --ssd-streaming-cache-experts 16 --temp 0
+```
+
+Use `--cuda` in a CUDA build. On ROCm, use `--rocm` and a verification-safe
+cache, for example `--ssd-streaming-cache-experts 32`.
+
+For memory-constrained Metal systems, use a small graph workspace as well as a
+small expert cache. A practical 16 GiB starting point is:
+
+```sh
+./ds4 -m ds4flash.gguf \
+  --mtp-model gguf/DeepSeek-V4-Flash-DSpark-support-0731.gguf \
+  --dspark --metal --ssd-streaming \
+  --ssd-streaming-cache-experts 16 \
+  --ctx 4096 --prefill-chunk 128 --temp 0
+```
+
+When DSpark+SSD runs on a Mac with at most 24 GiB and neither
+`--prefill-chunk` nor `DS4_METAL_PREFILL_CHUNK` is set, the runtime selects 128
+automatically. Set `DS4_DSPARK_LOW_MEMORY_PREFILL_CHUNK=0` to retain the normal
+workspace policy, or set it to another row count.
+
+The Metal SSD verifier already supports the checkpoint's full five-draft
+speculative block. With top-6 routing it needs at least 30 effective target
+expert-cache slots; `--ssd-streaming-cache-experts 32` is the practical
+five-draft setting. Smaller caches automatically limit the verifier to the
+number of complete top-k rows that fit (a 16-expert cache normally selects two
+rows). The Metal proposer follows that effective verifier/cache cap, avoiding
+work on a suffix that cannot be consumed. Set
+`DS4_METAL_DSPARK_PROPOSER_BLOCK_MAX=0` to restore the checkpoint's native
+five-row proposer for an A/B control, or set a positive value to cap it
+explicitly. Override the verifier policy independently with
+`DS4_DSPARK_SSD_VERIFY_BLOCK_MAX=N`.
+
+Metal can experimentally mirror the final target-hidden prefill row from the
+HC weighted-sum kernel itself, avoiding a separate 16 KiB blit and
+compute/blit encoder transition on each captured target layer. Enable it with
+`DS4_METAL_ENABLE_DSPARK_CAPTURE_FUSED_LAST=1`; the historical
+weighted-sum-plus-blit sequence remains the default because short M1 Pro SSD
+A/B runs were bit-identical but did not show a repeatable throughput win.
+`DS4_METAL_DISABLE_DSPARK_CAPTURE_FUSED_LAST=1` is the dominant kill switch.
+
+An experimental single-device Metal verifier can use the current target
+logits for the first draft and evaluate only the remaining `N-1` target rows.
+Enable it with `DS4_METAL_DSPARK_ACCEPTANCE_ONLY_VERIFY=1`; it remains opt-in
+because the smaller batch did not improve throughput on the measured M1 Pro
+SSD path. Two-draft blocks retain the legacy verifier
+because its one-row SSD routed-FFN path does not yet use the tiny-batch expert
+table. A five-draft block starting exactly on a ratio-4 compressor boundary
+also retains the legacy path so acceptance arithmetic does not switch to the
+aligned compressor kernel. After verification rolls back,
+the exact replay also skips the output head and logits readback for accepted
+prefix tokens whose logits would be discarded; set
+`DS4_METAL_DSPARK_HEADLESS_REPLAY=0` to restore the legacy replay. With
+`DS4_DSPARK_STATS=1`, `metal_accept_only`, `metal_verify_rows_saved`, and
+`metal_replay_headless` show how often these paths were exercised.
+
+On very small unified-memory Macs, an additional diagnostic can keep only the
+stage-0 `main_norm` and `main_proj` support tensors resident. Set
+`DS4_METAL_DSPARK_PIN_MAIN_PROJ=1`; the 0731 support file locks about 51 MiB,
+not the full 5.6 GiB GGUF. A failed lock is non-fatal and leaves the existing
+pageable path active. Keep this opt-in until a same-machine A/B shows lower
+`prop_setup`/generation time without reducing target-only throughput.
+
+An experimental two-draft Metal SSD verifier can commit a full accept without
+the normal rollback/replay pass:
+
+```sh
+DS4_METAL_DSPARK_EXACT2=1 DS4_DSPARK_STATS=1 \
+./ds4 -m ds4flash.gguf \
+  --mtp-model gguf/DeepSeek-V4-Flash-DSpark-support-0731.gguf \
+  --dspark --metal --ssd-streaming \
+  --ssd-streaming-cache-experts 16 --ctx 4096 --prefill-chunk 128 \
+  --temp 0
+```
+
+It uses the canonical one-row decode kernels in layer order, restores and
+replays token zero on a partial accept, and defaults both proposal and verify
+width to two. Keep it opt-in until a long same-machine run is byte-identical,
+has `exact2_attempt>0` and `exact2_fallback=0`, and improves throughput. The
+generic Metal verifier can already evaluate five drafts together with a
+32-expert cache, but its batch state is not numerically interchangeable with
+ordinary decode and therefore still requires rollback plus exact replay.
+
+`DS4_METAL_DSPARK_EXACTN_UNION=1` enables a separate experimental Metal SSD
+verifier for two through five draft tokens. It executes canonical one-row
+target decode in layer order, loads the union of the rows' routed experts once
+per layer, and commits its verifier state directly after a full accept. On a
+partial match it restores the frontier once, skips the boundary oracle,
+exact-two path, and legacy token-by-token verifier, then exactly replays only
+the already verified prefix. With `DS4_DSPARK_STATS=1`,
+`exactn_union_partial_replay` and `exactn_union_verify_skip` should advance
+together; `exactn_union_partial_replay_ms` isolates the required commit replay.
+Set `DS4_DSPARK_FIXTURE_REQUIRE_METAL_EXACTN_PARTIAL=1` to require at least one
+such partial match, equal replay/skip counts, no exact-union error fallback,
+and byte-identical fixture output. The model-backed
+`test-metal-exactn-oracle` is byte-identical to sequential decode for N=2..5,
+including every N=5 partial prefix, EOS in the first or a middle row, serialized
+KV/compressor state, logits, and a four-token continuation. Five drafts plus
+the target token already available at the start of the cycle cover the
+six-token speculative-cycle limit. Exact-union remains opt-in: correctness
+does not imply a throughput improvement on a particular memory configuration.
+
+For an independent Q8 output-head A/B inside exact-union, set
+`DS4_METAL_DSPARK_EXACTN_BATCH_HEAD=1`. HC collapse and normalization remain on
+the canonical one-row kernels, while one bit-exact decode-row dispatch projects
+all two through five verifier rows to vocabulary logits. Non-Q8 output weights
+are ineligible and a dispatch failure falls back to the ordinary per-row heads.
+`DS4_METAL_DISABLE_DSPARK_EXACTN_BATCH_HEAD=1` is the unconditional kill switch
+and wins if both variables are set. The
+`metal_exactn_batch_head_attempt`, `metal_exactn_batch_head_use`, and
+`metal_exactn_batch_head_fallback` counters identify the selected path; set
+`DS4_DSPARK_FIXTURE_REQUIRE_METAL_EXACTN_BATCH_HEAD=1` to require a nonzero,
+fallback-free use with byte-identical output.
+
+The generic model-backed exact-N oracle keeps this Q8-only experiment disabled,
+so it remains valid for target models with another output quantization. To add
+model-backed batch-head coverage, use an OutQ8 target explicitly:
+
+```sh
+DS4_TEST_METAL_EXACTN_BATCH_HEAD=1 \
+DS4_TEST_MODEL=/path/to/target-OutQ8.gguf \
+make test-metal-exactn-oracle
+```
+
+The Metal proposer also has an independent experiment for confidence/Markov
+synchronization overhead. Set `DS4_METAL_DSPARK_DEVICE_PROPOSER=1` when the
+final confidence projection and both Markov matrices are Q8_0. On an eligible
+single-device, tier-zero run it keeps the previous token, confidence decisions,
+and Markov argmax chain on Metal, reuses the first confidence already computed
+by the proposer, and returns one result for the complete draft block. Unlike
+the CUDA experiment, the Metal path is eligible with SSD streaming; tensor
+placement and proposal-quality mode remain excluded. It stops at the first
+rejected confidence row and preserves the smaller-token argmax tie break.
+Unsupported layouts, an incomplete result, or a CPU sigmoid-policy mismatch
+fall back to the existing per-row implementation.
+
+`DS4_METAL_DSPARK_NO_DEVICE_PROPOSER=1` is the unconditional kill switch;
+`DS4_DSPARK_NO_GPU_MARKOV=1` and `DS4_DSPARK_NO_MARKOV=1` also keep the path
+disabled. This remains opt-in because Q8 confidence accumulation moves from the
+host CPU to Metal and therefore needs a same-machine greedy oracle and A/B.
+With `DS4_DSPARK_STATS=1`, require
+`metal_device_proposer_attempt == metal_device_proposer_use > 0`,
+`metal_device_proposer_fallback=0`, and
+`metal_device_proposer_policy_mismatch=0`. The acceptance fixture enforces
+those conditions and byte-identical output with
+`DS4_DSPARK_FIXTURE_REQUIRE_METAL_DEVICE_PROPOSER=1`.
+
+Exact-union normally waits for every layer's routed-tail command buffer before
+releasing its private expert-address scope. For an isolated A/B,
+`DS4_METAL_DSPARK_EXACT_ROWS_ASYNC_TAILS=1` commits that tail without the CPU
+wait, retains all scope resources until command-buffer completion, and lets the
+next layer's router boundary provide the required ordering. The switch has no
+effect outside exact-union and is also opt-in; unset it for the synchronous
+control. Validate serialized state and greedy output as well as verifier time,
+because removing a host wait is not by itself evidence of an end-to-end gain.
+
+The AProjQ4 Metal decode path has several exact dispatch fusions relevant to
+this verifier:
+
+- HC RMSNorm plus the narrow F16 HC mixer is the M1-M4 default, including SSD
+  split phases such as `TO_ROUTER`. Use
+  `DS4_METAL_DISABLE_PRE_M5_HC_NORM_MIX_FUSE=1` for the reference control;
+  `DS4_METAL_ENABLE_HC_NORM_MIX_FUSE=1` is the explicit non-default gate on
+  other Apple generations.
+- The Q4 Q-A/KV projections can share a dispatch with eligible F16 compressor
+  projection/store work. It remains opt-in in both exact-union and ordinary
+  `FULL` decode via `DS4_METAL_ENABLE_Q4_QKV_COMPRESSOR_FUSE=1`; the first M1
+  Pro SSD A/B reduced dispatch count but did not improve verifier time. In
+  either scope,
+  `DS4_METAL_DISABLE_Q4_QKV_COMPRESSOR_FUSE=1` selects the existing fallback.
+- The eligible Q4 attention-output B projection can perform the following HC
+  expansion in the same dispatch. Use
+  `DS4_METAL_DISABLE_Q4_ATTN_OUT_HC_FUSE=1` as its isolated A/B control.
+- For an AProjQ4 multi-row attention-output batch with either attention B in
+  Q8 or Q4_K, the opt-in
+  `DS4_METAL_ENABLE_Q4_ATTN_OUT_TINY_BATCH=1` evaluates two through five rows
+  in two dispatches while retaining the canonical one-row reduction order for
+  every row. This covers the generic suffix verifier; the exact-union tape is
+  intentionally still row-by-row and does not select this helper. Other
+  output formats, unsupported shapes, a disabled Q4 classic matvec, or a
+  dispatch setup failure return to the existing row-wise path.
+  `DS4_METAL_DISABLE_Q4_ATTN_OUT_TINY_BATCH=1` is the unconditional kill
+  switch and wins when both variables are set. For a fail-closed model-backed
+  generic-verifier test, `DS4_METAL_REQUIRE_Q4_ATTN_OUT_TINY_BATCH=1` implies
+  the enable gate for N=2..5 and turns an ineligible shape, the kill switch, or
+  a dispatch failure into a hard error instead of a silent row-wise fallback.
+
+The AProjQ8 Q-A/KV plus compressor compound remains the M1-M5 default for
+eligible resident `FULL` decode. For an SSD-streaming A/B, including the
+exact-union `TO_ROUTER` collection prefix, set
+`DS4_METAL_ENABLE_Q8_QKV_COMPRESSOR_FUSE=1`. Ratio-4 layers combine the Q8
+Q-A/KV pair with both attention and indexer F16 compressor pairs; ratio-128
+layers combine it with the attention pair. The kernel preserves the canonical
+NSG=4 Q8 and NR0=2 F16 reduction trees. A diagnostic Q8 NSG override or the
+experimental NR0=4 compressor schedule therefore selects the separate
+dispatches. `DS4_METAL_REQUIRE_Q8_QKV_COMPRESSOR_FUSE=1` turns such a fallback
+into a visible error for model-backed tests. The existing ratio-specific
+pre-M5/M5 QKV compound disable variables remain authoritative. Keep the SSD
+extension opt-in until warm and cold A/B runs show a gain: it removes a launch
+per row and layer but reads the same model bytes, and a compound grid can
+change the order in which distant GGUF pages are faulted.
+
+Additional PR #755 ports keep their established kernels as shape/resource
+fallbacks:
+
+- On Apple M1 through M5, an eligible one-row HC producer combines the F16
+  RMSNorm/mixer, HC split and Sinkhorn-weighted sum, and destination RMSNorm in
+  one compound dispatch for both attention and FFN producers. The global
+  rollback is `DS4_METAL_DISABLE_HC_PRODUCER_PRE_NORM_FUSE=1`; the narrower
+  controls are `DS4_METAL_DISABLE_PRE_M5_HC_PRODUCER_PRE_NORM_FUSE=1` and
+  `DS4_METAL_DISABLE_M5_HC_PRODUCER_PRE_NORM_FUSE=1`. The existing
+  `DS4_METAL_DISABLE_PRE_M5_DECODE_PORTS=1` umbrella also disables it before
+  M5. `DS4_METAL_ENABLE_HC_PRODUCER_PRE_NORM_FUSE=1` permits a focused trial
+  on another eligible Metal device.
+- For an eligible ratio-4 layer on Apple M1 through M5, the standalone F16 compressor path can
+  project the attention and indexer KV/gate pairs and append both recurrent
+  states in one quad dispatch. It is the default in ordinary `FULL` decode and
+  the exact-union collection prefix when the larger Q4 compound dispatch did
+  not already store those states. Use
+  `DS4_METAL_DISABLE_COMPRESSOR_QUAD_STORE=1` for the reference path;
+  `DS4_METAL_DISABLE_PRE_M5_COMPRESSOR_QUAD_STORE=1` is an additional
+  compatibility rollback. `DS4_METAL_ENABLE_COMPRESSOR_QUAD_STORE=1` permits
+  a focused trial on another Metal device and widens the phase scope for
+  diagnostics.
+- The exact ratio-4, one-compressed-row pool specialization is the M1-M5
+  default for supported 128- and 512-element head shapes. Disable it globally
+  with `DS4_METAL_DISABLE_COMPRESSOR_EXACT_POOL_RATIO4=1`, or use the
+  pre-M5/M5 controls
+  `DS4_METAL_DISABLE_PRE_M5_COMPRESSOR_EXACT_POOL_RATIO4=1` and
+  `DS4_METAL_DISABLE_M5_COMPRESSOR_EXACT_POOL_RATIO4=1`. For a diagnostic run,
+  `DS4_METAL_REQUIRE_COMPRESSOR_EXACT_POOL_RATIO4=1` turns an unavailable
+  exact dispatch into a visible failure instead of silently selecting the
+  legacy reduction sequence.
+
+Metal FlashAttention pipeline selection also keeps a generation-aware
+one-entry host memo for hot specializations. This changes pipeline lookup, not
+kernel arithmetic. Disable the pad/block memo with
+`DS4_METAL_DISABLE_PRE_M5_FLASH_ATTN_PAD_BLK_MEMO=1` and the batched/vector
+memo with `DS4_METAL_DISABLE_PRE_M5_FLASH_ATTN_BATCHED_MEMO=1` when isolating
+host-side dispatch overhead.
+
+The former 512-column streaming Metal top-k specialization has been removed;
+its ordering was not deterministic for every input. The regular deterministic
+top-k implementation is now used instead and has no runtime re-enable switch.
+Use a previous binary only as a performance control, and require identical
+selected ids on tie-heavy inputs before comparing timing.
+
+Apple M1 defaults to a specialized SSD-streaming decode path for the exact
+IQ2_XXS/Q2_K routed-MoE shape with 256 experts, top-6 routing, and a
+4096-to-2048 gate/up projection. It replaces the IQ2 address-table pair-SwiGLU
+producer, including complementary resident/missing cache masks.
+It preserves the canonical dot-product,
+reduction, clamp, activation, and route-weight order but writes `mid` directly
+instead of materializing the otherwise unused gate/up rows. Every other
+device, shape, streaming mode, unsupported mask/accumulate mode, or unavailable
+pipeline keeps the canonical producer. Set
+`DS4_METAL_DISABLE_M1_IQ2_MID_ONLY=1` to restore the canonical producer.
+For fail-closed model coverage, `DS4_METAL_REQUIRE_M1_IQ2_MID_ONLY=1` rejects
+an ineligible supported address-table dispatch; the kill switch still takes
+precedence. The former `DS4_METAL_ENABLE_M1_IQ2_MID_ONLY=1` opt-in is accepted
+as a harmless compatibility setting because the path is now automatic.
+`make test-metal-iq2-midonly` compares all 12,288 top-6
+output words bitwise at full shape for both unmasked and complementary masked
+address tables, verifies that the candidates leave gate/up sentinels untouched,
+and checks output guards. The routed-MoE stage profiler reports
+`iq2_stream_addr_mid_only_4096x2048` or
+`iq2_stream_addr_mask_mid_only_4096x2048` when the model path is actually
+covered.
+
+These gates change dispatch and intermediate-memory traffic, not model
+arithmetic. Compare byte-identical output, exact-union counters, stage timings,
+and generation rate on the same machine; do not infer a speedup from a lower
+dispatch count alone.
+
+CPU greedy decoding and the verifier's excluding-argmax scan use an unrolled
+eight-lane implementation by default, including scalar tail handling and
+first-index tie semantics. Set `DS4_CPU_DISABLE_UNROLLED_ARGMAX=1` to restore
+the scalar scan for an isolated A/B. `tests/test_sampling` compares both paths,
+including cross-lane ties, excluded ids, and non-multiple-of-eight vocabulary
+sizes.
+
+Exact file views for the two token embedding rows and repeatedly used Q8
+support tensors are automatic; the compatibility kill switches are
+`DS4_METAL_DISABLE_TOKEN_EMBED_EXACT_VIEW=1` and
+`DS4_METAL_DISABLE_SUPPORT_Q8_DECODE_EXACT_VIEWS=1`.
+
+DSpark attempts a proposal on every eligible cycle. Proposal cadence is not
+adaptively throttled, so the reported acceptance rate covers the full runtime
+sample. Quality and strict DSpark modes remain target-only.
+
+Tune the expert-cache count for the available accelerator memory. ROCm needs
+enough slots for a whole verification block (30 for the 0731 model; use at
+least 32), and currently supports the IQ2_XXS/Q2_K or all-Q2_K routed-expert
+layouts. CUDA uses a transient selected-expert cache for each target block.
+The DSpark support weights are included in the startup memory budget even when
+the Metal file-backed mapping remains pageable. This combination is
+single-device only; CPU, distributed or
+multi-GPU placement, tensor parallelism, and legacy MTP support models remain
+incompatible with DSpark plus SSD streaming.
+
+Resident single-GPU CUDA skips verifier captures that rollback/replay cannot
+consume, batches frontier snapshot/restore copies behind one device fence,
+computes the output head only for the final replayed token, pads the five-row
+Q8 proposer head to the tensor-core shape, and fuses proposer Q RMSNorm with
+RoPE. CUDA and ROCm also avoid the Metal-only mid-token submission split: on
+those backends the same flush is a device-wide synchronization and only drains
+the launch pipeline. The two kernel-selection kill switches for before/after
+measurements are
+`DS4_CUDA_DSPARK_NO_PADDED_HEAD=1` and
+`DS4_CUDA_DSPARK_NO_Q_NORM_ROPE_FUSION=1`.
+
+CUDA fuses HC split, weighted sum, and RMSNorm across multiple batch rows,
+including the DSpark proposer and verifier; use
+`DS4_CUDA_DISABLE_HC_SPLIT_NORM_FUSED=1` for an A/B fallback to the separate
+kernels. AProjQ4 CUDA decode can also share the activation quantization for
+the Q-A/KV dense pair; `DS4_CUDA_DISABLE_Q4_DENSE_PAIR=1` selects the two
+standalone projections. The canonical Q4 path submits to the decode stream so
+these projections and the attention-output tail can participate in CUDA
+decode graphs.
+
+Single-token resident CUDA can also experiment with folding the canonical
+Q8_1 activation emitted by the 4096-wide HC split plus RMSNorm stage into its
+next MMVQ consumer. This remains opt-in pending GB10/DGX validation: set
+`DS4_CUDA_ENABLE_Q8_FOLD=1`; `DS4_CUDA_NO_Q8_FOLD=1` is the dominant kill
+switch. The sidecar is one-shot and keyed by model map, physical device,
+stream, source pointer, and session epoch. Capture, scratch growth, a model-map
+or device transition, and every lookup mismatch reject or invalidate it and
+fall back to the established quantizer. For a diagnostic run, disable decode
+graphs and add `DS4_CUDA_Q8_FOLD_ORACLE=1`; the oracle compares canonical Q8_1
+bytes and the reached aligned-Q8 or IQ2 MoE consumer output, then always keeps
+the freshly quantized reference. Require nonzero fold hits, `byte_calls`, and
+`output_calls`, with zero mismatches and skips before considering promotion.
+The experiment supports ds4's serialized, single-inference-host-thread CUDA
+runtime only; embeddings that submit concurrently to the same CUDA stream
+must leave it disabled.
+
+On a single DGX Spark/GB10, the AProjQ4 path also mirrors the safe parts of
+the aligned-Q8 decode work while retaining canonical Q4_K MMVQ/Q8_1
+arithmetic:
+
+- dense and paired Q4 projections reuse the persistent 1-MiB Q8_1 scratch;
+  `DS4_CUDA_NO_Q4_DENSE_SCRATCH=1` restores pool allocation;
+- attention-output A evaluates all output groups through one channel-grouped
+  MMVQ dispatch per token, preserving the one-row reduction tree of every
+  group. For DSpark verification widths 2--8,
+  `DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_BATCH=1` flattens `(token, group)` into
+  MMVQ channels and replaces the per-token loop with one grouped MMVQ
+  dispatch while keeping `ncols_dst=1`;
+  `DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH=1` restores the per-token grouped loop
+  and `DS4_CUDA_NO_Q4_GROUPED_ATTN_A=1` restores the per-group loop;
+- for prefill widths above eight, the default eligible GB10 path quantizes the
+  strided `[token][group][K]` input in one launch and writes each group directly
+  into `[token][group][rank]`. It removes the eight F32 pack/unpack copies while
+  keeping one established stream-K MMQ reduction per group. On the production
+  `groups=8`, `K=4096`, `rank=1024` shape, a fixed-layout eight-warp Q8_1
+  producer is also the default: each warp emits two canonical 128-value DS4
+  records, reducing quantizer CTA count by four while preserving every output
+  byte. `DS4_CUDA_NO_Q4_GROUPED_ATTN_A_Q81=1` restores only the generic
+  strided Q8_1 producer, and
+  `DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_Q81=1` fails closed if the specialized
+  producer is not selected. Set
+  `DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL=1` for the dominant local rollback;
+  exact `DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_PREFILL=0` is also a compatibility
+  opt-out. Add `DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_PREFILL=1` for fail-closed
+  tests. The separate single-grid/grid.z submission remains opt-in. The
+  unrelated 16-warp MMQ experiment also remains opt-in and is not used by
+  this producer;
+- attention-output B keeps its canonical MMVQ result and the ordinary HC
+  epilogue inside the graph-compatible fused call.  The row-packed epilogue
+  remains oracle-only until a GB10 device run proves it bit-exact;
+- the exact Q-b shape `32768x1024` has an experimental persistent-CTA kernel
+  behind `DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT=1`, with
+  `DS4_CUDA_NO_Q4_K1024_PERSISTENT=1` taking precedence. Tests can add
+  `DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT=1` to fail instead of silently using
+  canonical MMVQ when the persistent dispatch is unavailable; this admission
+  now fails before quantization, output clearing, or any kernel enqueue. Set
+  `DS4_CUDA_Q4_K1024_PERSISTENT_STATS=1` for host-dispatch candidate/use/
+  fallback counters. For a bitwise model-backed check, run with
+  `DS4_CUDA_DECODE_GRAPHS=0 DS4_CUDA_Q4_K1024_PERSISTENT_ORACLE=1`; the oracle
+  forces the candidate, compares it with canonical MMVQ, and always retains
+  canonical output.
+
+`DS4_CUDA_NO_Q4_GB10_FAST=1` is the umbrella rollback for these new GB10
+choices; it does not disable the older cross-CUDA Q-A/KV pair itself. For a
+fail-closed grouped attention comparison, set
+`DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_BATCH=1`,
+`DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_BATCH=1`, and
+`DS4_CUDA_Q4_GROUPED_ATTN_A_ORACLE=1`. The oracle computes the established per-group
+MMVQ reference (or the established per-token grouped loop for a multi-token
+candidate), reports aggregate calls/mismatches/skips plus separate
+batch_candidates/batch_calls/batch_mismatches/batch_skips, and retains
+canonical output. A valid multi-token test has nonzero candidates/calls and
+zero batch mismatches/skips. The
+oracle disables decode-graph capture. For a multi-token candidate, if it
+encounters another active capture or cannot allocate comparison scratch, it
+directly enqueues the canonical reference instead of consuming an unchecked
+candidate. The scratch, grouped, and persistent paths are also covered by
+`make test-mmq-parity-cuda CUDA_ARCH=sm_121`.
+
+CUDA Q4_K MMQ performs its non-finite output guard in the final write-back
+(or after the final stream-K fixup) instead of launching a separate
+full-output sanitizer. Finite results and the per-group reduction order are
+unchanged; the resident CUDA prefill harness checks them bit-for-bit against
+the former pack/MMQ/unpack path.
+
+Two additional CUDA fusions remain experimental until a device oracle passes
+on the target GPU. `DS4_CUDA_ENABLE_HC_NORM_MIX_FUSE=1` combines HC RMSNorm
+with the narrow F16 mixer when the selected standalone kernels have the same
+reduction order; with the normal one-token cuBLAS path, also set
+`DS4_CUDA_NO_F16_CUBLAS_ONE=1` to exercise it. The controls are
+`DS4_CUDA_DISABLE_HC_NORM_MIX_FUSE=1` and
+`DS4_CUDA_DISABLE_Q4_ATTN_OUT_HC_FUSE=1`. The Q4 attention-output B plus HC
+path is automatic when MMQ is disabled, where the existing one-dispatch Q8_K
+implementation is bit-compatible with its fallback. With the normal
+MMVQ/Q8_1 decode path, the GB10 graph-compatible call preserves both the
+canonical MMVQ projection and the ordinary HC expansion. The specialized
+row-packed epilogue is evaluated only by the oracle below and is never
+consumed by normal decoding. The older, truly single-dispatch Q8_K experiment
+is isolated behind
+`DS4_CUDA_Q4_ATTN_OUT_HC_Q8K_EXPERIMENT=1` and may differ numerically from
+MMVQ/Q8_1.
+
+For a fail-closed hardware comparison, set
+`DS4_CUDA_Q4_ATTN_OUT_HC_ORACLE=1`. It retains the canonical MMVQ/Q8_1 output,
+compares both the row-packed epilogue and the Q8_K compound bit-for-bit, and
+prints `epilogue_mismatches`, `q8k_mismatches`, and `skips` at exit. The
+oracle avoids readback while CUDA graph capture is active; run a separate
+non-captured diagnostic and require `calls>0`, `skips=0`, and
+`epilogue_mismatches=0` before promoting the row-packed path back into normal
+decoding. A zero-call summary is therefore an explicit failed coverage gate,
+not a silent pass.
+
+An experimental resident-CUDA path can run the existing aligned
+IQ2_XXS/Q2_K vector MoE kernels for two-to-five-draft routed batches,
+preserving the established fused-SoA path as an automatic fallback:
+
+```sh
+DS4_CUDA_DSPARK_TINY_ALIGNED_VEC=1 DS4_DSPARK_STATS=1 \
+./ds4 --cuda -m ds4flash.gguf \
+  --mtp-model gguf/DeepSeek-V4-Flash-DSpark-support-0731.gguf \
+  --dspark --temp 0 -p 'Write a Python quicksort function with comments.'
+```
+
+Keep the aligned tiny-batch path opt-in until the same-machine acceptance,
+decode-consistency, and throughput comparisons pass on CUDA hardware.
+
+An experimental exact two-token resident-CUDA verifier is available for a
+DGX Spark A/B test. It uses the ordinary decode kernels, commits a two-token
+full accept without rollback/replay, and replays only the first token on a
+partial accept. By default the switch runs both the proposer and verifier at
+width two, instead of evaluating the checkpoint's native five-row proposal
+when only two drafts can be consumed:
+
+```sh
+DS4_CUDA_DSPARK_EXACT2=1 DS4_DSPARK_STATS=1 \
+./ds4 --cuda -m ds4flash.gguf \
+  --mtp-model gguf/DeepSeek-V4-Flash-DSpark-support-0731.gguf \
+  --dspark --temp 0 -p 'Write a Python quicksort function with comments.'
+```
+
+The support model uses non-causal attention across the proposal block, so a
+two-row proposal is not guaranteed to be a prefix-identical version of its
+native five-row proposal. The target verifier still protects the emitted
+greedy continuation. For isolated A/B tests,
+`DS4_CUDA_DSPARK_PROPOSER_BLOCK_MAX=0` preserves the native proposer and an
+explicit value such as `2` caps it independently of exact-2.
+
+CUDA also has an opt-in tiled online-softmax kernel for this non-causal support
+attention. It shares each raw KV row across a group of attention heads and is
+selected only for the DSpark raw-ring/head geometry it supports; every other
+shape keeps the reference kernel. Enable it with
+`DS4_CUDA_ENABLE_DSPARK_NONCAUSAL_ONLINE=1`. The emergency control
+`DS4_CUDA_DISABLE_DSPARK_NONCAUSAL_ONLINE=1` wins when both variables are set.
+The online reduction order can change draft floating-point results even though
+the target verifier still protects greedy output. Use
+`DS4_DSPARK_VERIFY_NONCAUSAL=1` to print the first three comparisons against a
+host double-precision reference, and require the final target continuation to
+remain byte-identical in the performance A/B.
+
+Keep this path opt-in until the CUDA acceptance fixture is byte-identical and
+the same-machine statistics show lower `propose`, `verify` plus `replay` time.
+The stats line reports `prop_capped`, `prop_scheduled_rows`, `exact2_attempt`,
+`exact2_full`, `exact2_partial`, and `exact2_fallback`; a valid run must
+exercise exact-2 and leave its fallback counter at zero.
+
+A separate resident exact-N CUDA experiment extends the same canonical
+one-token tape to two through five draft rows. It leaves hidden rows and all
+target weights on one GPU, submits the per-row ordinary decode kernels in one
+stream, and reads back only the `N-1` acceptance ids plus final logits. A full
+match therefore commits its already-exact KV/compressor state without replay;
+a partial match restores the pre-cycle frontier once and replays the prefix
+already proven by exact-N, without running the legacy verifier a second time.
+Only a backend error retains the legacy verifier/replay fallback. Enable it
+independently with:
+
+```sh
+DS4_CUDA_DSPARK_EXACTN=1 DS4_DSPARK_STATS=1 \
+./ds4 --cuda -m ds4flash.gguf \
+  --mtp-model gguf/DeepSeek-V4-Flash-DSpark-support-0731.gguf \
+  --dspark --temp 0 -p 'Write a Python quicksort function with comments.'
+```
+
+The default verifier cap under this gate is five (or the available prefill
+workspace when smaller). The proposer uses at most one fewer workspace row,
+because its support stage also carries the current target row; the existing
+explicit proposer and verifier cap variables still take precedence.
+`DS4_CUDA_DISABLE_DSPARK_EXACTN=1` is the
+kill switch and restores the previous path without changing the enable
+variable. Track `cuda_exactn_attempt`, `cuda_exactn_full`,
+`cuda_exactn_fallback`, its partial/error split, and `cuda_exactn_rows`. Keep
+this experiment disabled by default until a real CUDA oracle and a long greedy
+A/B show byte-identical output, no fallback errors, and a throughput win.
+
+The layer tape can separately reuse its position-independent CUDA decode
+islands with `DS4_CUDA_DSPARK_EXACTN_GRAPHS=1`. Graph keys use the stable
+device address (including each batch-row offset), not the short-lived tensor
+view wrapper. Four cache entries per layer/island remain reserved for ordinary
+decode and five more are isolated for exact-N rows, so a width-five verifier
+cannot evict the normal decode keys. The first encounter warms lazy allocators,
+the second captures/instantiates, and only later encounters are pure replay;
+benchmark at least 128--256 generated tokens rather than judging a short
+capture-heavy run. `DS4_CUDA_DISABLE_DSPARK_EXACTN_GRAPHS=1` is the dedicated
+kill switch, while `DS4_CUDA_DECODE_GRAPHS=0` still disables all decode graphs.
+The stats fields `cuda_exactn_graph_attempt`, `..._use`, `..._warm`,
+`..._capture`, `..._replay`, `..._no_slot`, and `..._failure` expose warmup,
+reuse, capacity misses, and retired captures. This first rollout is for
+serialized, single-session DGX testing; the graph cache and cuBLAS capture
+state remain process-global. The fixture can require a clean post-warmup
+replay (including zero no-slot/failure events) with
+`DS4_DSPARK_FIXTURE_REQUIRE_CUDA_EXACTN_GRAPHS=1`.
+
+For a separate output-head A/B, set
+`DS4_CUDA_DSPARK_EXACTN_BATCH_HEAD=1`. The experiment keeps HC collapse and
+normalization on the canonical one-row kernels, then runs the Q8 vocabulary
+projection for all exact-N rows through the bit-exact decode-row kernel. It is
+automatically ineligible for non-Q8 output weights and falls back to the
+ordinary per-row heads on a dispatch failure. The emergency kill switch is
+`DS4_CUDA_DISABLE_DSPARK_EXACTN_BATCH_HEAD=1` and wins when both variables are
+present.
+
+The CUDA proposer tail has a second, independent experiment for the fixed
+confidence/Markov synchronization overhead:
+
+```sh
+DS4_CUDA_DSPARK_DEVICE_PROPOSER=1
+```
+
+When the final confidence projection and both Markov matrices are Q8_0, this
+keeps the previous token, confidence decisions, and all Markov argmax steps on
+the decode stream and reads one 64-byte result for the whole draft block.  It
+stops at the first rejected confidence row, preserves the Markov smaller-token
+tie break, and rechecks the returned confidence prefix with the established
+CPU sigmoid policy.  Unsupported layouts, an incomplete result, or a policy
+mismatch fall back to the per-row implementation.  The unconditional kill
+switch is `DS4_CUDA_DSPARK_NO_DEVICE_PROPOSER=1`; the older
+`DS4_DSPARK_NO_GPU_MARKOV` switch also keeps this path disabled.
+The initial gate is intentionally limited to resident, single-GPU, non-quality
+CUDA and reuses the already-computed first confidence value.
+
+This remains opt-in because the Q8 confidence accumulation moves from the host
+CPU to CUDA and must pass the DGX proposal/acceptance oracle before promotion.
+With `DS4_DSPARK_STATS=1`, require
+`cuda_device_proposer_attempt == cuda_device_proposer_use > 0`,
+`cuda_device_proposer_fallback=0`, and
+`cuda_device_proposer_policy_mismatch=0`.  The acceptance fixture can enforce
+those conditions with
+`DS4_DSPARK_FIXTURE_REQUIRE_CUDA_DEVICE_PROPOSER=1`.
+
+The stats line separates `cuda_exactn_ms` into setup, layer, head, and read
+components, and reports restore, legacy-error-fallback verification, and
+partial replay time independently. `cuda_exactn_partial_replay` and
+`cuda_exactn_verify_skip` should advance together on valid partial matches;
+the batch-head attempt/use/fallback counters make its dispatch unambiguous.
+
+Set `DS4_DSPARK_FIXTURE_REQUIRE_CUDA_EXACTN=1` on the candidate acceptance
+fixture to require at least one `cuda_exactn_attempt` and zero
+`cuda_exactn_error_fallback`. The aggregate `cuda_exactn_fallback` is reported
+but is not required to be zero: it also includes valid partial draft matches,
+which deliberately restore the frontier and use exact replay.
+
+The acceptance fixture can exercise the same SSD path on both the target-only
+baseline and the DSpark run. It also requires real proposals and accepted
+draft tokens, so an unavailable verifier cannot pass as a silent no-op:
+
+```sh
+DS4_DSPARK_FIXTURE_BACKEND=cuda \
+DS4_DSPARK_FIXTURE_SSD_STREAMING=1 \
+DS4_DSPARK_FIXTURE_SSD_STREAMING_CACHE_EXPERTS=32 \
+make dspark-acceptance
+```
+
+For ROCm, use `make rocm-dspark-acceptance` with the same model, support, and
+SSD fixture environment variables. The ROCm-specific target preserves the HIP
+object set and linker; the generic target selects CUDA objects on non-Apple
+hosts. `make rocm-dspark-verify-depth` provides the corresponding verifier
+invariant test.
+
 ## Speed
 
 The current q2 results use `ds4-bench` with the standard *Promessi sposi*
 input, 2048-token context steps, and 128 greedy generation tokens at every
 frontier. Each prefill number is for the next 2048-token chunk. The complete
 sweeps are in [m5_max.csv](speed-bench/m5_max.csv) and
-[gb10.csv](speed-bench/gb10.csv).
+[gb10.csv](speed-bench/gb10.csv). The GB10 optimization methodology and
+validation are documented in
+[ds4_gb10_q2_cuda_port_results.md](speed-bench/ds4_gb10_q2_cuda_port_results.md).
 
 | Machine | Backend | Context | Prefill | Generation |
 | --- | --- | ---: | ---: | ---: |
@@ -442,10 +1056,10 @@ sweeps are in [m5_max.csv](speed-bench/m5_max.csv) and
 | MacBook Pro M5 Max, 128 GB | Metal | 16384 | 572.53 t/s | 36.14 t/s |
 | MacBook Pro M5 Max, 128 GB | Metal | 32768 | 557.04 t/s | 34.36 t/s |
 | MacBook Pro M5 Max, 128 GB | Metal | 65536 | 398.50 t/s | 27.64 t/s |
-| DGX Spark GB10, 128 GB | CUDA | 2048 | 825.76 t/s | 18.05 t/s |
-| DGX Spark GB10, 128 GB | CUDA | 16384 | 872.44 t/s | 15.10 t/s |
-| DGX Spark GB10, 128 GB | CUDA | 32768 | 855.94 t/s | 14.43 t/s |
-| DGX Spark GB10, 128 GB | CUDA | 65536 | 822.98 t/s | 13.84 t/s |
+| DGX Spark GB10, 128 GB | CUDA | 2048 | 832.86 t/s | 20.58 t/s |
+| DGX Spark GB10, 128 GB | CUDA | 16384 | 883.81 t/s | 16.80 t/s |
+| DGX Spark GB10, 128 GB | CUDA | 32768 | 865.40 t/s | 15.99 t/s |
+| DGX Spark GB10, 128 GB | CUDA | 65536 | 833.44 t/s | 15.27 t/s |
 
 Older measurements for machines and model variants not rerun in this pass are
 kept for reference. They used the earlier CLI prompt procedure and are not
@@ -501,12 +1115,44 @@ context, and backend working-set limit leave less room. A plain number such as
 `--ssd-streaming-cache-experts 4000` requests 4000 dynamic expert slots without
 the two-layer reserve, but it can be reduced by the same final memory check.
 Non-routed weights, KV cache, graph scratch, and activations need additional
-memory. The automatic cache budget takes
+memory.
+
+Metal SSD+DSpark also has an experimental, support-aware pre-cap for A/B tests.
+Set `DS4_METAL_DSPARK_SAFE_EXPERT_COUNT=1` to convert a numeric count to bytes
+and cap it, when measurable, after accounting for the target's non-routed
+weights, the context/KV estimate, and a 2 GiB active reserve for the mmap-backed
+support model. It does not yet price the complete batch-prefill workspace or a
+separate routed-prefill transient reserve. Startup reports requested/effective
+slots and the support reserve. If this policy cannot measure safe room, it
+retains the explicit count with a warning; the normal final memory check remains
+authoritative. The experiment does not affect `NGB` budgets or CUDA/ROCm.
+Prefer an `NGB` budget for normal use. The automatic cache budget takes
 80% of the backend's recommended working set, subtracts non-routed weights, then
 applies the same routed-prefill headroom before sizing the dynamic cache. Leave
 the hot expert preload enabled for normal use; use `--ssd-streaming-cold` and
 `--ssd-streaming-preload-experts N` only for measurements.
 
+For Metal IQ2_XXS/Q2_K models, eligible SSD prefill chunks automatically use
+grouped address matmuls when the dynamic cache can retain the complete expert
+domain. This includes normal 128-token chunks; the automatic range is 32–760
+tokens on the 256-expert Flash model and requires, for example,
+`--ssd-streaming-cache-experts 256`. Once that material condition is met,
+selection is fail-closed by default; no environment prefix is required. Set
+`DS4_METAL_ENABLE_IQ2_XXS_SSD_PREFILL_MM=0` or
+`DS4_METAL_DISABLE_IQ2_XXS_SSD_PREFILL_MM=1` for the legacy sparse-matvec
+rollback. `DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM=0` keeps automatic
+selection but permits a fallback, while explicit `REQUIRE=1` also rejects an
+insufficient cache. Combining `REQUIRE=1` with `DISABLE=1` fails on an eligible
+grouped-MM candidate, while short tail chunks retain their normal fallback. The
+IQ2 live cache index remains automatic for its production shape, selected-load
+early commit remains off unless explicitly enabled, and grouped-MM statistics
+plus streaming timing summaries remain opt-in diagnostics. The grouped prefill
+loader also skips `F_RDADVISE` for chunks of at least 32 tokens because it
+immediately reads the same expert ranges with parallel `pread`; short chunks
+retain the hint. Set
+`DS4_METAL_ENABLE_STREAMING_PREFILL_EXPERT_READAHEAD=1` to restore the old
+hint-plus-read sequence for cold-storage A/B tests.
+
 ### Practical SSD streaming examples
 
 On 64GB MacBooks, start with the 2-bit Flash GGUF and a moderate expert cache:
diff --git a/STRIXHALO.md b/STRIXHALO.md
index 611239b7f5..f5f7b5a8f4 100644
--- a/STRIXHALO.md
+++ b/STRIXHALO.md
@@ -18,7 +18,7 @@ sudo apt-get update
 sudo apt-get install -y \
   hipcc rocminfo rocm-smi \
   libamdhip64-dev \
-  libhipblas-dev libhipblaslt-dev \
+  libhipblas-dev \
   librocblas-dev \
   librocwmma-dev \
   libhipcub-dev
diff --git a/cuda/mmq/ds4_ggml_stubs.cu b/cuda/mmq/ds4_ggml_stubs.cu
index 4e88c08f92..23224b6170 100644
--- a/cuda/mmq/ds4_ggml_stubs.cu
+++ b/cuda/mmq/ds4_ggml_stubs.cu
@@ -7,6 +7,7 @@
 // CUDA uses stream-ordered allocation. HIP retains stream-local allocations
 // because returning live MMQ scratch to its asynchronous pool is unreliable.
 
+#include "ds4_mmq.h"
 #include "common.cuh"   // pulls in ds4_ggml_stubs.h via redirect headers
 
 #if defined(GGML_USE_HIP)
@@ -23,6 +24,20 @@
 #include <mutex>
 #include <vector>
 
+/* Standalone MMQ tests do not link ds4_cuda.cu.  Full ds4 links its strong,
+ * stream-aware registry implementation over this fail-closed weak miss. */
+#if defined(__GNUC__)
+extern "C" __attribute__((weak)) int ds4_cuda_q8_fold_take_q81(
+        const void *src, uint64_t in_dim, cudaStream_t stream,
+        const void **q81) {
+    (void)src;
+    (void)in_dim;
+    (void)stream;
+    if (q81) *q81 = nullptr;
+    return 0;
+}
+#endif
+
 // ----------------------------------------------------------------------------
 // Device info singleton.
 //
@@ -133,10 +148,6 @@ extern "C" void ds4_pool_set_stream(cudaStream_t stream) {
     t_ds4_pool_stream = stream;
 }
 
-extern "C" cudaStream_t ds4_pool_get_stream(void) {
-    return t_ds4_pool_stream;
-}
-
 namespace {
 
 struct ds4_naive_pool : public ggml_cuda_pool {
diff --git a/cuda/mmq/ds4_mmq.cu b/cuda/mmq/ds4_mmq.cu
index 36c3104419..90838bf16f 100644
--- a/cuda/mmq/ds4_mmq.cu
+++ b/cuda/mmq/ds4_mmq.cu
@@ -25,11 +25,15 @@
 #include "quantize.cuh"
 #include "mmid.cuh"
 #include "ds4_mmq_d2r.cuh"
+#if !defined(GGML_USE_HIP)
+#include "ds4_mmq_q4_16warp.cuh"
+#endif
 
+#include <atomic>
 #include <cstdio>
 #include <cstdlib>
 #include <cstring>
-#include <cstring>
+#include <mutex>
 
 #if defined(__has_include)
 #if __has_include(<nvtx3/nvToolsExt.h>)
@@ -100,25 +104,38 @@ private:
 // Init
 // ----------------------------------------------------------------------------
 
-// Step 7 task #29: experimental persistent Q8_1 scratch buffer.
-//
-// Hypothesis: ggml_cuda_pool_alloc inside ds4_mmq_moe_vec_impl records a
-// cudaMallocAsync graph node into the captured layer graph.  At replay
-// time the alloc node returns a (potentially different) address, but the
-// matvec kernel's pointer argument was baked in at capture time.  Result:
-// the matvec reads stale/wrong memory and produces a different output
-// than eager execution, even with identical inputs.
-//
-// Mitigation under test: pre-allocate a persistent device buffer at
-// startup via plain cudaMalloc (NOT cudaMallocAsync, NOT inside any
-// capture).  When the env flag DS4_CUDA_MMQ_Q81_PERSISTENT=1 is set,
-// ds4_mmq_moe_vec_impl uses this persistent buffer instead of pool_alloc.
-//
-// Sized for V4 Flash decode shapes: gate Q8_1 ~8 KB, down Q8_1 ~14 KB.
-// 256 KB allocation gives generous headroom for short prefill batches.
+// Experimental persistent Q8_1 scratch.  The grouped raw prefill path below
+// aliases its input and down-Q8 staging in this arena: both ranges have the
+// same layout but disjoint lifetimes on the default stream.  The feature stays
+// opt-in because a process-global address is only safe for the single-owner
+// GB10 dispatch covered by q81_grouped_persistent_acquire().
 static void *g_q81_scratch_ptr   = nullptr;
 static size_t g_q81_scratch_bytes = 0;
-static bool   g_q81_scratch_enabled = false;
+// A failed resize retirement can leave both allocations live.  Keep the
+// unpublished replacement owned here so cleanup/reinit can retry its free.
+static void *g_q81_unpublished_replacement_ptr = nullptr;
+// Older vector wrappers still contain a generic persistent branch.  Keep that
+// branch disabled: unlike grouped fused_raw it has no default-stream lease or
+// capture exclusion and therefore cannot safely share the owned arena.
+static constexpr bool g_q81_scratch_enabled = false;
+static bool   g_q81_grouped_enabled = false;
+static bool   g_q81_scratch_poisoned = false;
+static int    g_q81_scratch_device = -1;
+static std::mutex g_q81_state_mutex; // Process-global state, not per-tensor.
+
+static uint64_t g_q81_grouped_candidates;
+static uint64_t g_q81_grouped_uses;
+static uint64_t g_q81_grouped_hits;
+static uint64_t g_q81_grouped_pool_fallbacks;
+static uint64_t g_q81_grouped_allocations;
+static uint64_t g_q81_grouped_resizes;
+static uint64_t g_q81_grouped_owner_rejects;
+static uint64_t g_q81_grouped_stream_rejects;
+static uint64_t g_q81_grouped_capture_rejects;
+static uint64_t g_q81_grouped_device_rejects;
+static uint64_t g_q81_grouped_size_rejects;
+static size_t   g_q81_grouped_high_water;
+
 static void  *g_aligned_q81_scratch_ptr = nullptr;
 static size_t g_aligned_q81_scratch_bytes = 0;
 static int    g_aligned_q81_scratch_device = -1;
@@ -140,6 +157,171 @@ struct mmq_pair_map_scratch {
 
 static mmq_pair_map_scratch g_mmq_pair_maps[GGML_CUDA_MAX_DEVICES] = {};
 
+// Backend init/teardown owns transitions, but dispatch admission reads this
+// flag without taking the Q8_1 arena mutex. Keep those reads race-free while
+// retaining the mutex below for arena ownership and cleanup serialization.
+static std::atomic<bool> g_gb10_optimizations{false};
+
+static bool gb10_optimizations_enabled() {
+    return g_gb10_optimizations.load(std::memory_order_relaxed);
+}
+
+static constexpr size_t DS4_MMQ_Q81_ARENA_MIN_BYTES = 4u * 1024u * 1024u;
+
+enum q81_arena_result {
+    Q81_ARENA_REJECTED = 0,
+    Q81_ARENA_HIT,
+    Q81_ARENA_ALLOCATED,
+    Q81_ARENA_RESIZED,
+};
+
+static bool q81_persistent_requested() {
+    const char *value = getenv("DS4_CUDA_MMQ_Q81_PERSISTENT");
+    if (!value || value[0] == '\0' || strcmp(value, "0") == 0 ||
+        strcmp(value, "off") == 0 || strcmp(value, "OFF") == 0 ||
+        strcmp(value, "false") == 0 || strcmp(value, "FALSE") == 0 ||
+        strcmp(value, "no") == 0 || strcmp(value, "NO") == 0) {
+        return false;
+    }
+    return strcmp(value, "1") == 0 ||
+           strcmp(value, "on") == 0 || strcmp(value, "ON") == 0 ||
+           strcmp(value, "true") == 0 || strcmp(value, "TRUE") == 0 ||
+           strcmp(value, "yes") == 0 || strcmp(value, "YES") == 0;
+}
+
+static bool q81_is_gb10_owner(int device) {
+    if (!gb10_optimizations_enabled() || device < 0 ||
+        device >= ggml_cuda_info().device_count) {
+        return false;
+    }
+    const auto &info = ggml_cuda_info().devices[device];
+    return info.integrated && info.cc == GGML_CUDA_CC_DGX_SPARK;
+}
+
+static q81_arena_result q81_arena_ensure_locked(int device, size_t required) {
+    if (g_q81_scratch_poisoned || required > SIZE_MAX - 255u ||
+        (g_q81_scratch_ptr && g_q81_scratch_device != device)) {
+        return Q81_ARENA_REJECTED;
+    }
+    if (g_q81_scratch_ptr && required <= g_q81_scratch_bytes) {
+        g_q81_grouped_enabled = true;
+        return Q81_ARENA_HIT;
+    }
+
+    const size_t aligned = (required + 255u) & ~(size_t)255u;
+    const size_t bytes = aligned > DS4_MMQ_Q81_ARENA_MIN_BYTES
+        ? aligned : DS4_MMQ_Q81_ARENA_MIN_BYTES;
+    void *replacement = nullptr;
+    cudaError_t err = cudaMalloc(&replacement, bytes);
+    if (err != cudaSuccess || !replacement) {
+        fprintf(stderr,
+                "ds4_mmq: cudaMalloc(persistent Q8_1 arena %zu B) failed: %s; "
+                "using stream pool\n",
+                bytes, cudaGetErrorString(err));
+        (void)cudaGetLastError();
+        return Q81_ARENA_REJECTED;
+    }
+
+    if (g_q81_scratch_ptr) {
+        /* The lease excludes another host submission and the persistent path
+         * only accepts the legacy default stream.  Drain the device before
+         * retiring the old address: allocation succeeds before any state is
+         * changed, so OOM leaves the previous arena usable.  A drain/free
+         * failure is different -- pointer liveness is ambiguous and the
+         * feature stays poisoned until explicit cleanup. */
+        err = cudaDeviceSynchronize();
+        if (err == cudaSuccess) {
+            err = cudaFree(g_q81_scratch_ptr);
+        }
+        if (err != cudaSuccess) {
+            fprintf(stderr,
+                    "ds4_mmq: persistent Q8_1 arena resize from %zu B to "
+                    "%zu B failed while retiring the old arena: %s; "
+                    "persistent reuse poisoned\n",
+                    g_q81_scratch_bytes, bytes, cudaGetErrorString(err));
+            (void)cudaGetLastError();
+            const cudaError_t replacement_free_err = cudaFree(replacement);
+            if (replacement_free_err != cudaSuccess) {
+                fprintf(stderr,
+                        "ds4_mmq: freeing unpublished Q8_1 replacement "
+                        "failed: %s\n",
+                        cudaGetErrorString(replacement_free_err));
+                (void)cudaGetLastError();
+                g_q81_unpublished_replacement_ptr = replacement;
+            }
+            g_q81_grouped_enabled = false;
+            g_q81_scratch_poisoned = true;
+            return Q81_ARENA_REJECTED;
+        }
+    }
+
+    const bool resized = g_q81_scratch_ptr != nullptr;
+    g_q81_scratch_ptr = replacement;
+    g_q81_scratch_bytes = bytes;
+    g_q81_scratch_device = device;
+    g_q81_grouped_enabled = true;
+    g_q81_grouped_allocations++;
+    if (resized) g_q81_grouped_resizes++;
+    fprintf(stderr,
+            "ds4_mmq: persistent Q8_1 arena %s (%zu B at %p, device %d)\n",
+            resized ? "resized" : "enabled", bytes,
+            g_q81_scratch_ptr, device);
+    return resized ? Q81_ARENA_RESIZED : Q81_ARENA_ALLOCATED;
+}
+
+// On success, lease remains locked until the complete host dispatch has been
+// submitted.  A following dispatch therefore cannot interleave its writes;
+// default-stream ordering protects the device-side lifetime after unlock.
+static char *q81_grouped_persistent_acquire(
+        int device, cudaStream_t stream, size_t required,
+        std::unique_lock<std::mutex> *lease) {
+    if (!lease || !q81_persistent_requested()) return nullptr;
+    lease->lock();
+    g_q81_grouped_candidates++;
+    if (required > g_q81_grouped_high_water) {
+        g_q81_grouped_high_water = required;
+    }
+    if (!q81_is_gb10_owner(device)) {
+        g_q81_grouped_device_rejects++;
+    } else if (stream != (cudaStream_t)0) {
+        g_q81_grouped_stream_rejects++;
+    } else {
+        int active_device = -1;
+        const cudaError_t device_err = cudaGetDevice(&active_device);
+        if (device_err != cudaSuccess || active_device != device ||
+            (g_q81_scratch_device >= 0 &&
+             g_q81_scratch_device != device)) {
+            (void)cudaGetLastError();
+            g_q81_grouped_owner_rejects++;
+        } else {
+            cudaStreamCaptureStatus capture = cudaStreamCaptureStatusNone;
+            const cudaError_t capture_err =
+                cudaStreamIsCapturing(stream, &capture);
+            if (capture_err != cudaSuccess ||
+                capture != cudaStreamCaptureStatusNone) {
+                (void)cudaGetLastError();
+                g_q81_grouped_capture_rejects++;
+            } else {
+                const q81_arena_result arena =
+                    q81_arena_ensure_locked(device, required);
+                if (arena != Q81_ARENA_REJECTED &&
+                    g_q81_grouped_enabled && !g_q81_scratch_poisoned &&
+                    required <= g_q81_scratch_bytes) {
+                    g_q81_grouped_uses++;
+                    if (arena == Q81_ARENA_HIT) g_q81_grouped_hits++;
+                    return (char *)g_q81_scratch_ptr;
+                }
+                if (g_q81_scratch_ptr && required > g_q81_scratch_bytes) {
+                    g_q81_grouped_size_rejects++;
+                }
+            }
+        }
+    }
+    g_q81_grouped_pool_fallbacks++;
+    lease->unlock();
+    return nullptr;
+}
+
 extern "C" void ds4_mmq_set_aligned_q81_scratch(void *ptr, size_t bytes) {
     g_aligned_q81_scratch_ptr = ptr;
     g_aligned_q81_scratch_bytes = ptr ? bytes : 0;
@@ -154,30 +336,204 @@ static void *ds4_mmq_aligned_q81_scratch(int device, size_t bytes) {
         ? g_aligned_q81_scratch_ptr : nullptr;
 }
 
-// Read by ds4_mmq_moe_vec_impl; non-zero means use the persistent buffer.
-// Set by ds4_mmq_init once based on env.  (Single-threaded GPU work; no
-// atomicity needed.)
-extern "C" int ds4_mmq_q81_persistent_enabled(void) {
-    return g_q81_scratch_enabled ? 1 : 0;
+/* Test-only preflight hook.  It deliberately traverses the production
+ * acquire path (including owner/default-stream/capture checks and resize
+ * retirement) without enqueueing a synthetic large MMQ fixture. */
+extern "C" int ds4_mmq_q81_persistent_preflight_for_test(
+        int device, size_t required) {
+    int previous = -1;
+    if (cudaGetDevice(&previous) != cudaSuccess ||
+        cudaSetDevice(device) != cudaSuccess) {
+        (void)cudaGetLastError();
+        return -1;
+    }
+    char *arena = nullptr;
+    {
+        std::unique_lock<std::mutex> lease(
+            g_q81_state_mutex, std::defer_lock);
+        arena = q81_grouped_persistent_acquire(
+            device, (cudaStream_t)0, required, &lease);
+    }
+    const cudaError_t restore_err = previous != device
+        ? cudaSetDevice(previous) : cudaSuccess;
+    if (restore_err != cudaSuccess) {
+        (void)cudaGetLastError();
+        return -2;
+    }
+    return arena ? 0 : -3;
+}
+
+static uint64_t g_q8_fold_oracle_byte_calls;
+static uint64_t g_q8_fold_oracle_byte_mismatches;
+static uint64_t g_q8_fold_oracle_output_calls;
+static uint64_t g_q8_fold_oracle_output_mismatches;
+static uint64_t g_q8_fold_oracle_raw_moe_calls;
+static uint64_t g_q8_fold_oracle_aligned_q8_calls;
+static uint64_t g_q8_fold_oracle_aligned_iq2_calls;
+static uint64_t g_q8_fold_oracle_skips;
+static int g_q8_fold_oracle_report_registered;
+
+static cudaError_t ds4_mmq_q8_fold_oracle_free(
+        void *ptr, const char *label, cudaError_t prior_err) {
+    if (!ptr) return prior_err;
+    const cudaError_t free_err = cudaFree(ptr);
+    if (free_err != cudaSuccess) {
+        fprintf(stderr,
+                "ds4: CUDA Q8_1 fold oracle cudaFree(%s) failed: %s\n",
+                label, cudaGetErrorString(free_err));
+        if (prior_err == cudaSuccess) return free_err;
+    }
+    return prior_err;
+}
+
+static void ds4_mmq_q8_fold_oracle_report(void) {
+    fprintf(stderr,
+            "ds4: CUDA Q8_1 fold oracle: byte_calls=%llu "
+            "byte_mismatches=%llu output_calls=%llu "
+            "output_mismatches=%llu raw_moe_calls=%llu "
+            "aligned_q8_calls=%llu aligned_iq2_calls=%llu skips=%llu "
+            "(canonical reference retained)\n",
+            (unsigned long long)g_q8_fold_oracle_byte_calls,
+            (unsigned long long)g_q8_fold_oracle_byte_mismatches,
+            (unsigned long long)g_q8_fold_oracle_output_calls,
+            (unsigned long long)g_q8_fold_oracle_output_mismatches,
+            (unsigned long long)g_q8_fold_oracle_raw_moe_calls,
+            (unsigned long long)g_q8_fold_oracle_aligned_q8_calls,
+            (unsigned long long)g_q8_fold_oracle_aligned_iq2_calls,
+            (unsigned long long)g_q8_fold_oracle_skips);
+}
+
+static bool ds4_mmq_q8_fold_oracle_enabled() {
+    const char *env = getenv("DS4_CUDA_Q8_FOLD_ORACLE");
+    const bool enabled = env && strcmp(env, "1") == 0;
+    if (enabled && !g_q8_fold_oracle_report_registered) {
+        g_q8_fold_oracle_report_registered = 1;
+        (void)atexit(ds4_mmq_q8_fold_oracle_report);
+    }
+    return enabled;
+}
+
+__global__ static void q8_fold_output_compare_kernel(
+        uint32_t *mismatch, const float *candidate,
+        const float *reference, uint64_t n) {
+    const uint64_t i = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x;
+    if (i < n && __float_as_uint(candidate[i]) !=
+                 __float_as_uint(reference[i])) {
+        atomicExch(mismatch, 1u);
+    }
 }
 
-extern "C" void *ds4_mmq_q81_scratch_ptr(void) {
-    return g_q81_scratch_ptr;
+/* Byte oracle for every consumer.  On mismatch the fresh canonical bytes
+ * overwrite the sidecar before it is consumed.  Any setup/capture failure
+ * rejects the fold entirely so the established prelude quantizes again. */
+static bool ds4_mmq_q8_fold_oracle_bytes(
+        const float *X_f32, int64_t K, int64_t ne10_padded,
+        char *folded, cudaStream_t stream) {
+    if (!ds4_mmq_q8_fold_oracle_enabled()) return true;
+    if (!X_f32 || !folded || K <= 0 || ne10_padded != K ||
+        (K % QK8_1) != 0) {
+        g_q8_fold_oracle_skips++;
+        return false;
+    }
+    cudaStreamCaptureStatus capture = cudaStreamCaptureStatusNone;
+    const cudaError_t capture_err = cudaStreamIsCapturing(stream, &capture);
+    if (capture_err != cudaSuccess) {
+        fprintf(stderr,
+                "ds4_mmq: Q8 fold oracle stream-capture query failed: %s\n",
+                cudaGetErrorString(capture_err));
+        (void)cudaGetLastError();
+        g_q8_fold_oracle_skips++;
+        return false;
+    }
+    if (capture != cudaStreamCaptureStatusNone) {
+        fprintf(stderr,
+                "ds4_mmq: Q8 fold oracle skipped during stream capture\n");
+        g_q8_fold_oracle_skips++;
+        return false;
+    }
+    const size_t bytes = (size_t)ne10_padded * sizeof(block_q8_1) / QK8_1;
+    if (bytes == 0u || bytes > 16384u) {
+        g_q8_fold_oracle_skips++;
+        return false;
+    }
+    char *fresh = nullptr;
+    char *host = (char *)malloc(bytes * 2u);
+    if (!host) {
+        fprintf(stderr,
+                "ds4_mmq: Q8 fold oracle host allocation (%zu B) failed\n",
+                bytes * 2u);
+        g_q8_fold_oracle_skips++;
+        return false;
+    }
+    const cudaError_t fresh_alloc_err = cudaMalloc((void **)&fresh, bytes);
+    if (fresh_alloc_err != cudaSuccess || !fresh) {
+        fprintf(stderr,
+                "ds4_mmq: Q8 fold oracle device allocation (%zu B) failed: "
+                "%s%s\n",
+                bytes, cudaGetErrorString(fresh_alloc_err),
+                fresh_alloc_err == cudaSuccess ? " (null pointer)" : "");
+        free(host);
+        (void)cudaGetLastError();
+        g_q8_fold_oracle_skips++;
+        return false;
+    }
+    quantize_row_q8_1_cuda(
+        X_f32, /*ids=*/nullptr, fresh, GGML_TYPE_Q8_0,
+        /*ne00=*/K, /*s11=*/K, /*s12=*/K, /*s13=*/K,
+        /*ne0=*/ne10_padded, /*ne1=*/1, /*ne2=*/1, /*ne3=*/1,
+        stream);
+    bool setup_ok = cudaGetLastError() == cudaSuccess &&
+                    cudaStreamSynchronize(stream) == cudaSuccess &&
+                    cudaMemcpy(host, folded, bytes,
+                               cudaMemcpyDeviceToHost) == cudaSuccess &&
+                    cudaMemcpy(host + bytes, fresh, bytes,
+                               cudaMemcpyDeviceToHost) == cudaSuccess;
+    if (!setup_ok) {
+        (void)cudaGetLastError();
+        cudaError_t cleanup_err = ds4_mmq_q8_fold_oracle_free(
+            fresh, "byte-fresh", cudaSuccess);
+        if (cleanup_err != cudaSuccess) (void)cudaGetLastError();
+        free(host);
+        g_q8_fold_oracle_skips++;
+        return false;
+    }
+    const bool match = memcmp(host, host + bytes, bytes) == 0;
+    g_q8_fold_oracle_byte_calls++;
+    if (!match) {
+        g_q8_fold_oracle_byte_mismatches++;
+        if (cudaMemcpyAsync(folded, fresh, bytes,
+                            cudaMemcpyDeviceToDevice, stream) != cudaSuccess ||
+            cudaStreamSynchronize(stream) != cudaSuccess) {
+            (void)cudaGetLastError();
+            cudaError_t cleanup_err = ds4_mmq_q8_fold_oracle_free(
+                fresh, "byte-fresh", cudaSuccess);
+            if (cleanup_err != cudaSuccess) (void)cudaGetLastError();
+            free(host);
+            g_q8_fold_oracle_skips++;
+            return false;
+        }
+    }
+    cudaError_t cleanup_err = ds4_mmq_q8_fold_oracle_free(
+        fresh, "byte-fresh", cudaSuccess);
+    free(host);
+    if (cleanup_err != cudaSuccess) {
+        (void)cudaGetLastError();
+        g_q8_fold_oracle_skips++;
+        return false;
+    }
+    return true;
 }
 
-// M2-Inc2a: registry of producer-emitted q8_1 activations (ds4_cuda.cu).
-// A hit returns canonical block_q8_1 codes for this exact activation
-// pointer (bit-exact vs quantize_row_q8_1_cuda), letting the caller skip
-// its quantize prelude.  Only valid for single-token unpadded rows
-// (ne10_padded == K); the registry itself guarantees freshness (slots are
-// reset by the producing entry every layer and pops are one-shot).
-extern "C" int ds4_cuda_q8_fold_take_q81(const void *src, uint64_t in_dim,
-                                         const void **q81);
 static char *ds4_mmq_folded_q81(const float *X_f32, int64_t K, int n_tokens,
-                                int64_t ne10_padded) {
+                                int64_t ne10_padded, cudaStream_t stream) {
     if (n_tokens != 1 || ne10_padded != K) return nullptr;
     const void *p = nullptr;
-    if (!ds4_cuda_q8_fold_take_q81((const void *)X_f32, (uint64_t)K, &p)) return nullptr;
+    if (!ds4_cuda_q8_fold_take_q81(
+            (const void *)X_f32, (uint64_t)K, stream, &p)) return nullptr;
+    if (!ds4_mmq_q8_fold_oracle_bytes(
+            X_f32, K, ne10_padded, (char *)(uintptr_t)p, stream)) {
+        return nullptr;
+    }
     static int logged = 0;
     if (!logged) {
         logged = 1;
@@ -308,10 +664,6 @@ static int64_t d2r_min_cols() {
     return cached;
 }
 
-extern "C" size_t ds4_mmq_q81_scratch_bytes(void) {
-    return g_q81_scratch_bytes;
-}
-
 extern "C" int ds4_mmq_init(int device) {
     if (device < 0) {
         fprintf(stderr, "ds4_mmq_init: invalid device %d\n", device);
@@ -349,29 +701,128 @@ extern "C" int ds4_mmq_init(int device) {
         maps.expert_bounds = base + 2u * MMQ_GFX1151_PAIR_MAP_ROWS;
     }
 
-    // Step 7 task #29: pre-allocate persistent Q8_1 scratch if enabled.
-    // Must happen here (before any layer-graph capture) so the cudaMalloc
-    // is not forbidden by capture-mode restrictions, and so the kernel
-    // pointer arg baked into the captured graph stays valid at replay.
-    if (getenv("DS4_CUDA_MMQ_Q81_PERSISTENT") && !g_q81_scratch_ptr) {
-        const size_t bytes = 256 * 1024;
-        cudaError_t err = cudaMalloc(&g_q81_scratch_ptr, bytes);
-        if (err != cudaSuccess) {
-            fprintf(stderr, "ds4_mmq_init: cudaMalloc(q81_scratch %zu B) failed: %s; "
-                            "falling back to pool_alloc\n",
-                    bytes, cudaGetErrorString(err));
-            g_q81_scratch_ptr = nullptr;
-            g_q81_scratch_enabled = false;
-        } else {
-            g_q81_scratch_bytes = bytes;
-            g_q81_scratch_enabled = true;
-            fprintf(stderr, "ds4_mmq_init: persistent Q8_1 scratch enabled (%zu B at %p)\n",
-                    bytes, g_q81_scratch_ptr);
+    // Allocation is intentionally lazy.  The first eligible grouped dispatch
+    // knows its exact maximum input/down staging requirement and resolves the
+    // arena during preflight, before it submits any device operation.
+    {
+        std::lock_guard<std::mutex> lock(g_q81_state_mutex);
+        const bool requested = q81_persistent_requested();
+        g_q81_grouped_enabled = requested && q81_is_gb10_owner(device) &&
+            g_q81_scratch_ptr && !g_q81_scratch_poisoned &&
+            g_q81_scratch_device == device;
+    }
+    return 0;
+}
+
+extern "C" int ds4_mmq_q81_persistent_cleanup(void) {
+    std::lock_guard<std::mutex> lock(g_q81_state_mutex);
+    g_q81_grouped_enabled = false;
+    if (!g_q81_scratch_ptr && !g_q81_unpublished_replacement_ptr) {
+        g_q81_scratch_bytes = 0;
+        g_q81_scratch_device = -1;
+        g_q81_scratch_poisoned = false;
+        return 0;
+    }
+
+    int previous = -1;
+    if (cudaGetDevice(&previous) != cudaSuccess ||
+        g_q81_scratch_device < 0 ||
+        cudaSetDevice(g_q81_scratch_device) != cudaSuccess) {
+        (void)cudaGetLastError();
+        g_q81_scratch_poisoned = true;
+        return -1;
+    }
+    cudaError_t sync_err = cudaDeviceSynchronize();
+    cudaError_t arena_free_err = cudaSuccess;
+    cudaError_t replacement_free_err = cudaSuccess;
+    if (sync_err == cudaSuccess) {
+        if (g_q81_scratch_ptr) {
+            arena_free_err = cudaFree(g_q81_scratch_ptr);
+            if (arena_free_err == cudaSuccess) {
+                g_q81_scratch_ptr = nullptr;
+                g_q81_scratch_bytes = 0;
+            } else {
+                (void)cudaGetLastError();
+            }
         }
+        if (g_q81_unpublished_replacement_ptr) {
+            replacement_free_err = cudaFree(
+                g_q81_unpublished_replacement_ptr);
+            if (replacement_free_err == cudaSuccess) {
+                g_q81_unpublished_replacement_ptr = nullptr;
+            } else {
+                (void)cudaGetLastError();
+            }
+        }
+    } else {
+        (void)cudaGetLastError();
+    }
+    const cudaError_t restore_err = previous != g_q81_scratch_device
+        ? cudaSetDevice(previous) : cudaSuccess;
+    if (sync_err != cudaSuccess || arena_free_err != cudaSuccess ||
+        replacement_free_err != cudaSuccess) {
+        fprintf(stderr,
+                "ds4_mmq: persistent Q8_1 arena cleanup failed: "
+                "sync=%s arena_free=%s replacement_free=%s\n",
+                cudaGetErrorString(sync_err),
+                cudaGetErrorString(arena_free_err),
+                cudaGetErrorString(replacement_free_err));
+        g_q81_scratch_poisoned = true;
+        return -1;
+    }
+    // Both owned allocations are now retired; only now may reinit clear the
+    // poison and admit a new lazy allocation.
+    g_q81_scratch_device = -1;
+    g_q81_scratch_poisoned = false;
+    if (restore_err != cudaSuccess) {
+        fprintf(stderr,
+                "ds4_mmq: persistent Q8_1 arena freed, but restoring CUDA "
+                "device %d failed: %s\n",
+                previous, cudaGetErrorString(restore_err));
+        (void)cudaGetLastError();
+        return -2;
     }
     return 0;
 }
 
+extern "C" void ds4_mmq_q81_persistent_counters(
+        uint64_t *candidates, uint64_t *uses, uint64_t *hits,
+        uint64_t *pool_fallbacks, uint64_t *allocations, uint64_t *resizes,
+        size_t *arena_bytes, size_t *high_water) {
+    std::lock_guard<std::mutex> lock(g_q81_state_mutex);
+    if (candidates) *candidates = g_q81_grouped_candidates;
+    if (uses) *uses = g_q81_grouped_uses;
+    if (hits) *hits = g_q81_grouped_hits;
+    if (pool_fallbacks) *pool_fallbacks = g_q81_grouped_pool_fallbacks;
+    if (allocations) *allocations = g_q81_grouped_allocations;
+    if (resizes) *resizes = g_q81_grouped_resizes;
+    if (arena_bytes) *arena_bytes = g_q81_scratch_bytes;
+    if (high_water) *high_water = g_q81_grouped_high_water;
+}
+
+extern "C" void ds4_mmq_q81_persistent_report(void) {
+    std::lock_guard<std::mutex> lock(g_q81_state_mutex);
+    fprintf(stderr,
+            "ds4: CUDA MMQ grouped Q8_1 persistent: candidates=%llu "
+            "uses=%llu hits=%llu pool_fallbacks=%llu allocations=%llu "
+            "resizes=%llu "
+            "arena=%zu high_water=%zu rejects(device/owner/stream/capture/size)="
+            "%llu/%llu/%llu/%llu/%llu poisoned=%d\n",
+            (unsigned long long)g_q81_grouped_candidates,
+            (unsigned long long)g_q81_grouped_uses,
+            (unsigned long long)g_q81_grouped_hits,
+            (unsigned long long)g_q81_grouped_pool_fallbacks,
+            (unsigned long long)g_q81_grouped_allocations,
+            (unsigned long long)g_q81_grouped_resizes,
+            g_q81_scratch_bytes, g_q81_grouped_high_water,
+            (unsigned long long)g_q81_grouped_device_rejects,
+            (unsigned long long)g_q81_grouped_owner_rejects,
+            (unsigned long long)g_q81_grouped_stream_rejects,
+            (unsigned long long)g_q81_grouped_capture_rejects,
+            (unsigned long long)g_q81_grouped_size_rejects,
+            g_q81_scratch_poisoned ? 1 : 0);
+}
+
 // ----------------------------------------------------------------------------
 // Gating: when should the caller choose mmq over dequant+cublas?
 //
@@ -513,6 +964,323 @@ bool ds4_mmq_k_tile_supported(const char *tag, int K, int cc) {
     return true;
 }
 
+#if !defined(GGML_USE_HIP)
+static bool ds4_q4_test_q8_1_layout(
+        int N, int K, size_t *payload_bytes, size_t *total_bytes) {
+    if (N <= 0 || K <= 0 || (K % QK_K) != 0) return false;
+    const int64_t padded_k = GGML_PAD((int64_t)K, MATRIX_ROW_PADDING);
+    const size_t blocks_per_column =
+        (size_t)padded_k / (4u * (size_t)QK8_1);
+    if ((size_t)N > SIZE_MAX / blocks_per_column) return false;
+    const size_t blocks = (size_t)N * blocks_per_column;
+    if (blocks > SIZE_MAX / sizeof(block_q8_1_mmq)) return false;
+    const size_t payload = blocks * sizeof(block_q8_1_mmq);
+    const size_t slack = 128u * sizeof(block_q8_1_mmq);
+    if (payload > SIZE_MAX - slack) return false;
+    if (payload_bytes) *payload_bytes = payload;
+    if (total_bytes) *total_bytes = payload + slack;
+    return true;
+}
+
+/* The candidate and its caller-owned fixup buffer model canonical m128n128.
+ * Confirm the real canonical picker chooses that tile: a width limit alone is
+ * insufficient because resource constraints or a ceil-division plateau can
+ * retain a narrower width and change both partitioning and scratch size. */
+static bool ds4_q4_test_reference_uses_m128n128(
+        int device, int cc, int N) {
+    if (device < 0 || device >= GGML_CUDA_MAX_DEVICES || N <= 0 ||
+        get_mmq_y_host(cc) != 128) {
+        return false;
+    }
+    const size_t smpbo = ggml_cuda_info().devices[device].smpbo;
+    const int warp_size = ggml_cuda_info().devices[device].warp_size;
+    const int nwarps = mmq_get_nwarps_host(cc, warp_size);
+    const int mmq_x_max = get_mmq_x_max_host(cc);
+    int mmq_x_best = 0;
+    int64_t ntiles_x_best = INT64_MAX;
+    for (int mmq_x = 8;
+         mmq_x <= mmq_x_max && ntiles_x_best > 1;
+         mmq_x += 8) {
+        const int granularity = mmq_get_granularity_host(mmq_x, cc);
+        if (mmq_x % granularity != 0 ||
+            mmq_get_nbytes_shared<GGML_TYPE_Q4_K>(
+                mmq_x, 128, cc, warp_size, nwarps) > smpbo) {
+            continue;
+        }
+        const int64_t ntiles_x =
+            ((int64_t)N + mmq_x - 1) / mmq_x;
+        if (ntiles_x < ntiles_x_best) {
+            mmq_x_best = mmq_x;
+            ntiles_x_best = ntiles_x;
+        }
+    }
+    return mmq_x_best == 128;
+}
+
+extern "C" int
+ds4_mmq_q4_K_dense_preq_reference_m128n128_for_test(int N) {
+    const int dev = ggml_cuda_get_device();
+    if (dev < 0 || dev >= GGML_CUDA_MAX_DEVICES) return 0;
+    return ds4_q4_test_reference_uses_m128n128(
+        dev, ggml_cuda_info().devices[dev].cc, N) ? 1 : 0;
+}
+
+extern "C" size_t ds4_mmq_q4_K_q8_1_scratch_bytes(int N, int K) {
+    size_t total = 0;
+    return ds4_q4_test_q8_1_layout(N, K, nullptr, &total) ? total : 0;
+}
+
+extern "C" int ds4_mmq_q4_K_quantize_q8_1_for_test(
+        const float *X_f32, void *q8_ds4, size_t q8_bytes,
+        int N, int K, cudaStream_t stream) {
+    size_t total = 0;
+    if (!X_f32 || !q8_ds4 ||
+        !ds4_q4_test_q8_1_layout(N, K, nullptr, &total) ||
+        q8_bytes < total) {
+        return -1;
+    }
+    cudaError_t err = cudaMemsetAsync(q8_ds4, 0, total, stream);
+    if (err != cudaSuccess) return -2;
+    quantize_mmq_q8_1_cuda(
+        X_f32, /*ids=*/nullptr, q8_ds4, GGML_TYPE_Q4_K,
+        /*ne00=*/K, /*s11=*/(int64_t)K, /*s12=*/0, /*s13=*/0,
+        /*ne0=*/GGML_PAD((int64_t)K, MATRIX_ROW_PADDING),
+        /*ne1=*/N, /*ne2=*/1, /*ne3=*/1, stream);
+    err = cudaGetLastError();
+    return err == cudaSuccess ? 0 : -3;
+}
+
+extern "C" int ds4_mmq_q4_K_dense_preq_reference_for_test(
+        const void *W_q4_K, const void *q8_ds4, size_t q8_bytes,
+        float *out_f32, int M, int N, int K, int use_stream_k,
+        void *stream_k_fixup, size_t stream_k_fixup_bytes,
+        cudaStream_t stream) {
+    size_t payload = 0, total = 0;
+    if (!W_q4_K || !q8_ds4 || !out_f32 || M <= 0 ||
+        !ds4_q4_test_q8_1_layout(N, K, &payload, &total) ||
+        q8_bytes < total) {
+        return -1;
+    }
+    const int dev = ggml_cuda_get_device();
+    if (dev < 0 || dev >= GGML_CUDA_MAX_DEVICES) return -1;
+    const int cc = ggml_cuda_info().devices[dev].cc;
+    if (!ds4_mmq_k_tile_supported<GGML_TYPE_Q4_K>(
+            "ds4_mmq_q4_K_dense_preq_reference_for_test", K, cc)) {
+        return -1;
+    }
+    ggml_backend_cuda_context *ctx = get_ctx_for_device(dev);
+    if (!ctx) return -1;
+    if ((stream_k_fixup == nullptr && stream_k_fixup_bytes != 0u) ||
+        (stream_k_fixup != nullptr &&
+         ((uintptr_t)stream_k_fixup % alignof(float)) != 0) ||
+        (stream_k_fixup_bytes % sizeof(float)) != 0u) {
+        return -1;
+    }
+    if (stream_k_fixup != nullptr) {
+        if (!use_stream_k) return -1;
+        if (!ds4_q4_test_reference_uses_m128n128(dev, cc, N)) {
+            return DS4_MMQ_NOT_APPLICABLE;
+        }
+        const size_t required =
+            ds4_mmq_q4_K_dense_16warp_streamk_scratch_bytes(
+                M, N, ggml_cuda_info().devices[dev].nsm);
+        if (required > stream_k_fixup_bytes) return -1;
+    }
+    ds4_pool_set_stream(stream);
+    const int64_t stride_row_x = (int64_t)K / QK_K;
+    const int64_t stride_y = (int64_t)(payload / sizeof(int));
+    const mmq_args args = {
+        /*x=*/(const char *)W_q4_K,
+        /*type_x=*/GGML_TYPE_Q4_K,
+        /*y=*/(const int *)q8_ds4,
+        /*ids_dst=*/nullptr,
+        /*expert_bounds=*/nullptr,
+        /*dst=*/out_f32,
+        /*ncols_x=*/(int64_t)K, /*nrows_x=*/(int64_t)M,
+        /*ncols_dst=*/(int64_t)N,
+        /*stride_row_x=*/stride_row_x, /*ncols_y=*/(int64_t)N,
+        /*nrows_dst=*/(int64_t)M,
+        /*nchannels_x=*/1, /*nchannels_y=*/1,
+        /*stride_channel_x=*/0, /*stride_channel_y=*/stride_y,
+        /*stride_channel_dst=*/0,
+        /*nsamples_x=*/1, /*nsamples_y=*/1,
+        /*stride_sample_x=*/0, /*stride_sample_y=*/stride_y,
+        /*stride_sample_dst=*/0,
+        /*use_stream_k=*/use_stream_k != 0,
+        /*ncols_max=*/(int64_t)N,
+        /*x_soa=*/nullptr,
+        /*soa_blocks=*/0,
+        /*stream_k_fixup=*/static_cast<float *>(stream_k_fixup),
+        /*stream_k_fixup_elements=*/stream_k_fixup_bytes / sizeof(float),
+    };
+    mul_mat_q_case<GGML_TYPE_Q4_K>(*ctx, args, stream);
+    const cudaError_t err = cudaGetLastError();
+    return err == cudaSuccess ? 0 : -2;
+}
+
+static int ds4_q4_16warp_prepare_once(int device);
+
+extern "C" int ds4_mmq_q4_K_dense_preq_16warp_for_test(
+        const void *W_q4_K, const void *q8_ds4, size_t q8_bytes,
+        float *out_f32, int M, int N, int K, cudaStream_t stream) {
+    size_t total = 0;
+    if (!W_q4_K || !q8_ds4 || !out_f32 ||
+        !ds4_q4_test_q8_1_layout(N, K, nullptr, &total) ||
+        q8_bytes < total) {
+        return -1;
+    }
+    // This Stream-K oracle hook deliberately accepts an N tail (the production
+    // selector is stricter until NVIDIA measurements justify broadening it),
+    // but it must reject shapes that cannot be represented by this kernel
+    // before enqueueing any work.
+    if (M < 128 || (M % 128) != 0 || N < 512 ||
+        K < 1024 || K > 8192 || (K % QK_K) != 0) {
+        return DS4_MMQ_NOT_APPLICABLE;
+    }
+    const int dev = ggml_cuda_get_device();
+    const int cc = ggml_cuda_info().devices[dev].cc;
+    if (!ds4_mmq_q4_K_dense_16warp_available(cc) ||
+        ds4_q4_16warp_prepare_once(dev) != 0) {
+        return DS4_MMQ_NOT_APPLICABLE;
+    }
+    ggml_backend_cuda_context *ctx = get_ctx_for_device(dev);
+    if (!ctx) return DS4_MMQ_NOT_APPLICABLE;
+    ds4_pool_set_stream(stream);
+    const int nsm = ggml_cuda_info().devices[dev].nsm;
+    const size_t fixup_bytes =
+        ds4_mmq_q4_K_dense_16warp_streamk_scratch_bytes(M, N, nsm);
+    ggml_cuda_pool_alloc<char> fixup(ctx->pool());
+    if (fixup_bytes != 0u) fixup.alloc(fixup_bytes);
+    return ds4_mmq_q4_K_dense_16warp_streamk_enqueue(
+        W_q4_K, q8_ds4, out_f32, fixup.get(), fixup_bytes,
+        M, N, K, nsm, stream);
+}
+
+enum {
+    DS4_Q4_16WARP_REQUEST = 1,
+    DS4_Q4_16WARP_REQUIRE = 2,
+    DS4_Q4_16WARP_DISABLE = 4,
+};
+
+static bool ds4_q4_16warp_env_enabled(const char *name) {
+    const char *value = getenv(name);
+    return value && value[0] && !(value[0] == '0' && value[1] == '\0');
+}
+
+static int ds4_q4_16warp_mode(void) {
+    static const int cached = [] {
+        int mode = 0;
+        if (ds4_q4_16warp_env_enabled("DS4_CUDA_Q4_MMQ_16WARP")) {
+            mode |= DS4_Q4_16WARP_REQUEST;
+        }
+        if (ds4_q4_16warp_env_enabled(
+                "DS4_CUDA_REQUIRE_Q4_MMQ_16WARP")) {
+            mode |= DS4_Q4_16WARP_REQUEST | DS4_Q4_16WARP_REQUIRE;
+        }
+        if (ds4_q4_16warp_env_enabled("DS4_CUDA_NO_Q4_MMQ_16WARP")) {
+            mode |= DS4_Q4_16WARP_DISABLE;
+        }
+        return mode;
+    }();
+    return cached;
+}
+
+static int ds4_q4_16warp_prepare_once(int device) {
+    static std::mutex mutex;
+    static std::atomic<int> state[GGML_CUDA_MAX_DEVICES];
+    if (device < 0 || device >= GGML_CUDA_MAX_DEVICES) return -1;
+    int cached = state[device].load(std::memory_order_acquire);
+    if (cached == 1) return 0;
+    if (cached < 0) return cached;
+    std::lock_guard<std::mutex> lock(mutex);
+    cached = state[device].load(std::memory_order_relaxed);
+    if (cached == 1) return 0;
+    if (cached < 0) return cached;
+    const int rc = ds4_mmq_q4_K_dense_16warp_prepare();
+    state[device].store(rc == 0 ? 1 : rc, std::memory_order_release);
+    return rc;
+}
+
+/* Keep the 16-warp experiment on geometries with enough independent output
+ * tiles to occupy the device well.  Below canonical's 90% whole-tile cutoff,
+ * the candidate mirrors canonical stream-K partitioning and fixup; this 80%
+ * gate is therefore only an admission/performance heuristic, not a numerical
+ * shortcut.  On GB10 the Q-A/KV N=4096 shapes score 88%. */
+static bool ds4_q4_16warp_grid_efficient(int M, int N, int nsm) {
+    if (M <= 0 || N <= 0 || nsm <= 0) return false;
+    const int64_t tiles_m = ((int64_t)M + 127) / 128;
+    const int64_t tiles_n = ((int64_t)N + 127) / 128;
+    if (tiles_m > INT64_MAX / tiles_n) return false;
+    const int64_t tiles = tiles_m * tiles_n;
+    const int64_t waves = (tiles + nsm - 1) / nsm;
+    if (waves <= 0 || (int64_t)nsm > INT64_MAX / waves) return false;
+    return (100 * tiles) / ((int64_t)nsm * waves) >= 80;
+}
+
+static bool ds4_q4_16warp_pair_leg_shape_supported(
+        int cc, int M, int N, int K) {
+    return ds4_mmq_q4_K_dense_16warp_available(cc) &&
+           M >= 512 && (M % 128) == 0 &&
+           N >= 512 && (N % 128) == 0 &&
+           K >= 1024 && K <= 4096 && (K % QK_K) == 0;
+}
+
+/* Resolve the experiment before allocation or enqueue. The standalone path
+ * uses the public M>=1024 gate; a dense-pair leg may go down to M=512 because
+ * Q-A/KV pairs contain a 512-row leg. Each candidate grid must retain at least
+ * 80% whole-tile SM-wave efficiency; scheduling itself follows canonical
+ * stream-K whenever canonical would split K. */
+static int ds4_q4_16warp_select(
+        const char *tag, int device, int cc, int M, int N, int K,
+        bool pair_leg, bool *selected) {
+    if (!selected) return DS4_MMQ_NOT_APPLICABLE;
+    *selected = false;
+    const int mode = ds4_q4_16warp_mode();
+    const bool disabled = (mode & DS4_Q4_16WARP_DISABLE) != 0;
+    const bool requested = (mode & DS4_Q4_16WARP_REQUEST) != 0;
+    const bool required = (mode & DS4_Q4_16WARP_REQUIRE) != 0;
+    if (!requested) return 0;
+
+    const bool shape_supported = pair_leg
+        ? ds4_q4_16warp_pair_leg_shape_supported(cc, M, N, K)
+        : ds4_mmq_q4_K_dense_16warp_supported(cc, M, N, K) != 0;
+    // The exact oracle models canonical m128n128 MMQ. Check the selector's
+    // actual result, not only its upper bound: resource limits and equal
+    // ceil-division plateaus can make it retain a narrower tile.
+    const bool canonical_x128 =
+        ds4_q4_test_reference_uses_m128n128(device, cc, N);
+    const bool grid_efficient = ds4_q4_16warp_grid_efficient(
+        M, N, ggml_cuda_info().devices[device].nsm);
+    if (required && (disabled || !shape_supported || !canonical_x128 ||
+                     !grid_efficient)) {
+        fprintf(stderr,
+                "%s: required Q4 16-warp path is ineligible "
+                "(scope=%s disabled=%d shape=%d x128=%d grid_eff=%d "
+                "M=%d N=%d K=%d)\n",
+                tag, pair_leg ? "pair-leg" : "dense",
+                disabled ? 1 : 0, shape_supported ? 1 : 0,
+                canonical_x128 ? 1 : 0, grid_efficient ? 1 : 0, M, N, K);
+        return DS4_MMQ_NOT_APPLICABLE;
+    }
+    if (disabled || !shape_supported || !canonical_x128 || !grid_efficient) {
+        return 0;
+    }
+
+    const int prep = ds4_q4_16warp_prepare_once(device);
+    if (prep == 0) {
+        *selected = true;
+        return 0;
+    }
+    if (required) {
+        fprintf(stderr, "%s: required Q4 16-warp preflight failed: %d\n",
+                tag, prep);
+        return DS4_MMQ_NOT_APPLICABLE;
+    }
+    (void)cudaGetLastError();
+    return 0;
+}
+#endif
+
 template <ggml_type type>
 int ds4_mmq_dense_impl(
         const char  * tag,
@@ -543,6 +1311,18 @@ int ds4_mmq_dense_impl(
     const int cc  = ggml_cuda_info().devices[dev].cc;
     if (!ds4_mmq_k_tile_supported<type>(tag, K, cc)) return -1;
 
+#if !defined(GGML_USE_HIP)
+    bool use_q4_16warp = false;
+    if constexpr (type == GGML_TYPE_Q4_K) {
+        const int select_rc = ds4_q4_16warp_select(
+            tag, dev, cc, M, N, K, /*pair_leg=*/false,
+            &use_q4_16warp);
+        if (select_rc != 0) {
+            return select_rc;
+        }
+    }
+#endif
+
     ggml_backend_cuda_context * ctx = get_ctx_for_device(dev);
     if (!ctx) {
         fprintf(stderr, "%s: failed to get cuda context for device %d\n", tag, dev);
@@ -639,6 +1419,29 @@ int ds4_mmq_dense_impl(
         (void)cudaMemsetAsync(out_f32, 0, (size_t)M * (size_t)N * sizeof(float), stream);
     }
 
+#if !defined(GGML_USE_HIP)
+    if constexpr (type == GGML_TYPE_Q4_K) {
+        if (use_q4_16warp) {
+            const int nsm = ggml_cuda_info().devices[dev].nsm;
+            const size_t fixup_bytes =
+                ds4_mmq_q4_K_dense_16warp_streamk_scratch_bytes(
+                    M, N, nsm);
+            ggml_cuda_pool_alloc<char> fixup(ctx->pool());
+            if (fixup_bytes != 0u) fixup.alloc(fixup_bytes);
+            const int rc = ds4_mmq_q4_K_dense_16warp_streamk_enqueue(
+                W, src1_q8_1, out_f32, fixup.get(), fixup_bytes,
+                M, N, K, nsm, stream);
+            if (rc != 0) {
+                fprintf(stderr,
+                        "%s: Q4 16-warp stream-K launch failed: %d\n",
+                        tag, rc);
+                return -3;
+            }
+            return 0;
+        }
+    }
+#endif
+
     const mmq_args args = {
         /*x=*/(const char *)W,
         /*type_x=*/type,
@@ -663,7 +1466,471 @@ int ds4_mmq_dense_impl(
         fprintf(stderr, "%s: mul_mat_q_case launch failed: %s\n", tag, cudaGetErrorString(err));
         return -3;
     }
-    ds4_mmq_sanitize_f32(out_f32, (uint64_t)M * (uint64_t)N, stream);
+    if constexpr (type != GGML_TYPE_Q4_K) {
+        ds4_mmq_sanitize_f32(out_f32, (uint64_t)M * (uint64_t)N, stream);
+    }
+    return 0;
+}
+
+/* Batched Q4_K pair for the prefill tier.  The two ordinary dense calls
+ * differ only in their weight/output rows; their Q8_1 MMQ activation is
+ * byte-identical.  Keep that activation alive across both established MMQ
+ * launches so Q-A and KV pay the quantize/tail-clear prelude once. */
+int ds4_mmq_q4_K_dense_pair_impl(
+        const void  * W0,
+        const void  * W1,
+        const float * X_f32,
+        float       * out0_f32,
+        float       * out1_f32,
+        int           M0,
+        int           M1,
+        int           N,
+        int           K,
+        cudaStream_t  stream) {
+    const char *tag = "ds4_mmq_q4_K_dense_pair";
+    if (!W0 || !W1 || !X_f32 || !out0_f32 || !out1_f32) {
+        fprintf(stderr, "%s: null pointer\n", tag);
+        return DS4_MMQ_NOT_APPLICABLE;
+    }
+    if (M0 <= 0 || M1 <= 0 || N <= 0 || K <= 0 || K % 256 != 0) {
+        fprintf(stderr, "%s: bad shape M0=%d M1=%d N=%d K=%d\n",
+                tag, M0, M1, N, K);
+        return DS4_MMQ_NOT_APPLICABLE;
+    }
+    if ((size_t)M0 > SIZE_MAX / (size_t)N / sizeof(float) ||
+        (size_t)M1 > SIZE_MAX / (size_t)N / sizeof(float)) {
+        fprintf(stderr, "%s: output size overflow\n", tag);
+        return DS4_MMQ_NOT_APPLICABLE;
+    }
+    const size_t out0_bytes = (size_t)M0 * (size_t)N * sizeof(float);
+    const size_t out1_bytes = (size_t)M1 * (size_t)N * sizeof(float);
+    const uintptr_t out0_addr = (uintptr_t)out0_f32;
+    const uintptr_t out1_addr = (uintptr_t)out1_f32;
+    const bool outputs_overlap = out0_addr <= out1_addr
+        ? (size_t)(out1_addr - out0_addr) < out0_bytes
+        : (size_t)(out0_addr - out1_addr) < out1_bytes;
+    if (outputs_overlap) {
+        fprintf(stderr, "%s: output ranges overlap\n", tag);
+        return DS4_MMQ_NOT_APPLICABLE;
+    }
+
+    const int dev = ggml_cuda_get_device();
+    const int cc  = ggml_cuda_info().devices[dev].cc;
+    if (!ds4_mmq_k_tile_supported<GGML_TYPE_Q4_K>(tag, K, cc)) {
+        return DS4_MMQ_NOT_APPLICABLE;
+    }
+
+#if !defined(GGML_USE_HIP)
+    bool use_q4_16warp0 = false;
+    bool use_q4_16warp1 = false;
+    int select_rc = ds4_q4_16warp_select(
+        tag, dev, cc, M0, N, K, /*pair_leg=*/true,
+        &use_q4_16warp0);
+    if (select_rc != 0) return select_rc;
+    select_rc = ds4_q4_16warp_select(
+        tag, dev, cc, M1, N, K, /*pair_leg=*/true,
+        &use_q4_16warp1);
+    if (select_rc != 0) return select_rc;
+#endif
+
+    ggml_backend_cuda_context *ctx = get_ctx_for_device(dev);
+    if (!ctx) {
+        fprintf(stderr, "%s: failed to get cuda context for device %d\n",
+                tag, dev);
+        return DS4_MMQ_NOT_APPLICABLE;
+    }
+    ds4_pool_set_stream(stream);
+
+    const int64_t ne10_padded = GGML_PAD((int64_t)K, MATRIX_ROW_PADDING);
+    const size_t blocks_per_col =
+        (size_t)ne10_padded / (4u * (size_t)QK8_1);
+    const size_t bytes_per_col =
+        blocks_per_col * sizeof(block_q8_1_mmq);
+    const size_t slack_blocks = (size_t)get_mmq_x_max_host(cc);
+    if ((size_t)N > SIZE_MAX / bytes_per_col ||
+        slack_blocks > SIZE_MAX / sizeof(block_q8_1_mmq)) {
+        fprintf(stderr, "%s: activation scratch size overflow\n", tag);
+        return DS4_MMQ_NOT_APPLICABLE;
+    }
+    const size_t payload_bytes = (size_t)N * bytes_per_col;
+    const size_t slack_bytes = slack_blocks * sizeof(block_q8_1_mmq);
+    if (payload_bytes > SIZE_MAX - slack_bytes) {
+        fprintf(stderr, "%s: activation scratch size overflow\n", tag);
+        return DS4_MMQ_NOT_APPLICABLE;
+    }
+    const size_t nbytes_q8_1 = payload_bytes + slack_bytes;
+
+    ggml_cuda_pool_alloc<char> src1_q8_1(ctx->pool(), nbytes_q8_1);
+    ybuf_memset(src1_q8_1.get(), nbytes_q8_1, stream);
+    quantize_mmq_q8_1_cuda(
+        X_f32, /*ids=*/nullptr, (void *)src1_q8_1.get(),
+        GGML_TYPE_Q4_K, /*ne00=*/K, /*s11=*/(int64_t)K,
+        /*s12=*/0, /*s13=*/0,
+        /*ne0=*/ne10_padded, /*ne1=*/(int64_t)N,
+        /*ne2=*/1, /*ne3=*/1, stream);
+    cudaError_t err = cudaGetLastError();
+    if (err != cudaSuccess) {
+        fprintf(stderr, "%s: quantize failed: %s\n",
+                tag, cudaGetErrorString(err));
+        return -2;
+    }
+
+    const int64_t stride_row_x = (int64_t)K / QK_K;
+    const int64_t stride_channel_y =
+        (int64_t)(payload_bytes / sizeof(int));
+    const bool use_stream_k =
+        (GGML_CUDA_CC_IS_NVIDIA(cc) &&
+         ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_VOLTA) ||
+        GGML_CUDA_CC_IS_CDNA(cc);
+
+#if !defined(GGML_USE_HIP)
+    const int q4_16warp_nsm = ggml_cuda_info().devices[dev].nsm;
+    const size_t q4_16warp_fixup0 = use_q4_16warp0
+        ? ds4_mmq_q4_K_dense_16warp_streamk_scratch_bytes(
+              M0, N, q4_16warp_nsm)
+        : 0u;
+    const size_t q4_16warp_fixup1 = use_q4_16warp1
+        ? ds4_mmq_q4_K_dense_16warp_streamk_scratch_bytes(
+              M1, N, q4_16warp_nsm)
+        : 0u;
+    const size_t q4_16warp_fixup_bytes =
+        q4_16warp_fixup0 > q4_16warp_fixup1
+            ? q4_16warp_fixup0 : q4_16warp_fixup1;
+    // Both legs are ordered on the same stream, so one allocation can be
+    // cleared and reused after the first leg's fixup has consumed it.
+    ggml_cuda_pool_alloc<char> q4_16warp_fixup(ctx->pool());
+    if (q4_16warp_fixup_bytes != 0u) {
+        q4_16warp_fixup.alloc(q4_16warp_fixup_bytes);
+    }
+#endif
+
+    if (out_memset_enabled()) {
+        cudaMemsetAsync(out0_f32, 0, out0_bytes, stream);
+    }
+    const mmq_args args0 = {
+        /*x=*/(const char *)W0,
+        /*type_x=*/GGML_TYPE_Q4_K,
+        /*y=*/(const int *)src1_q8_1.get(),
+        /*ids_dst=*/nullptr,
+        /*expert_bounds=*/nullptr,
+        /*dst=*/out0_f32,
+        /*ncols_x=*/(int64_t)K,
+        /*nrows_x=*/(int64_t)M0,
+        /*ncols_dst=*/(int64_t)N,
+        /*stride_row_x=*/stride_row_x,
+        /*ncols_y=*/(int64_t)N,
+        /*nrows_dst=*/(int64_t)M0,
+        /*nchannels_x=*/1,
+        /*nchannels_y=*/1,
+        /*stride_channel_x=*/0,
+        /*stride_channel_y=*/stride_channel_y,
+        /*stride_channel_dst=*/0,
+        /*nsamples_x=*/1,
+        /*nsamples_y=*/1,
+        /*stride_sample_x=*/0,
+        /*stride_sample_y=*/stride_channel_y,
+        /*stride_sample_dst=*/0,
+        /*use_stream_k=*/use_stream_k,
+        /*ncols_max=*/(int64_t)N,
+    };
+#if !defined(GGML_USE_HIP)
+    if (use_q4_16warp0) {
+        const int rc = ds4_mmq_q4_K_dense_16warp_streamk_enqueue(
+            W0, src1_q8_1.get(), out0_f32,
+            q4_16warp_fixup.get(), q4_16warp_fixup_bytes,
+            M0, N, K, q4_16warp_nsm, stream);
+        if (rc != 0) {
+            fprintf(stderr,
+                    "%s: first Q4 16-warp stream-K launch failed: %d\n",
+                    tag, rc);
+            return -3;
+        }
+    } else
+#endif
+    {
+        mul_mat_q_case<GGML_TYPE_Q4_K>(*ctx, args0, stream);
+        err = cudaGetLastError();
+        if (err != cudaSuccess) {
+            fprintf(stderr, "%s: first mul_mat_q_case launch failed: %s\n",
+                    tag, cudaGetErrorString(err));
+            return -3;
+        }
+    }
+
+    if (out_memset_enabled()) {
+        cudaMemsetAsync(out1_f32, 0, out1_bytes, stream);
+    }
+    const mmq_args args1 = {
+        /*x=*/(const char *)W1,
+        /*type_x=*/GGML_TYPE_Q4_K,
+        /*y=*/(const int *)src1_q8_1.get(),
+        /*ids_dst=*/nullptr,
+        /*expert_bounds=*/nullptr,
+        /*dst=*/out1_f32,
+        /*ncols_x=*/(int64_t)K,
+        /*nrows_x=*/(int64_t)M1,
+        /*ncols_dst=*/(int64_t)N,
+        /*stride_row_x=*/stride_row_x,
+        /*ncols_y=*/(int64_t)N,
+        /*nrows_dst=*/(int64_t)M1,
+        /*nchannels_x=*/1,
+        /*nchannels_y=*/1,
+        /*stride_channel_x=*/0,
+        /*stride_channel_y=*/stride_channel_y,
+        /*stride_channel_dst=*/0,
+        /*nsamples_x=*/1,
+        /*nsamples_y=*/1,
+        /*stride_sample_x=*/0,
+        /*stride_sample_y=*/stride_channel_y,
+        /*stride_sample_dst=*/0,
+        /*use_stream_k=*/use_stream_k,
+        /*ncols_max=*/(int64_t)N,
+    };
+#if !defined(GGML_USE_HIP)
+    if (use_q4_16warp1) {
+        const int rc = ds4_mmq_q4_K_dense_16warp_streamk_enqueue(
+            W1, src1_q8_1.get(), out1_f32,
+            q4_16warp_fixup.get(), q4_16warp_fixup_bytes,
+            M1, N, K, q4_16warp_nsm, stream);
+        if (rc != 0) {
+            fprintf(stderr,
+                    "%s: second Q4 16-warp stream-K launch failed: %d\n",
+                    tag, rc);
+            return -4;
+        }
+    } else
+#endif
+    {
+        mul_mat_q_case<GGML_TYPE_Q4_K>(*ctx, args1, stream);
+        err = cudaGetLastError();
+        if (err != cudaSuccess) {
+            fprintf(stderr, "%s: second mul_mat_q_case launch failed: %s\n",
+                    tag, cudaGetErrorString(err));
+            return -4;
+        }
+    }
+    return 0;
+}
+
+#if !defined(GGML_USE_HIP)
+static bool ds4_q4_grouped_q81_env_enabled(const char *name) {
+    const char *value = getenv(name);
+    return value && value[0] && !(value[0] == '0' && value[1] == '\0');
+}
+#endif
+
+/* Token-batched grouped Q4_K projection for attention output-A.  The source
+ * is token-major [N][G][K], while MMQ stores directly into token-major
+ * [N][G][M].  Quantizing the strided source as G channels removes the old
+ * pack/unpack copies and shares one scratch allocation/quantizer launch.
+ *
+ * single_grid=false retains the established one-MMQ-launch-per-group path.
+ * single_grid=true maps groups to grid.z in one launch, but isolates each
+ * z-slice's stream-k coordinate space.  Its grid.x, partial-K ownership and
+ * fixup order are therefore identical to the former per-group invocation. */
+int ds4_mmq_q4_K_grouped_dense_impl(
+        const void  *W,
+        const float *X,
+        float       *out,
+        int          M,
+        int          N,
+        int          K,
+        int          n_groups,
+        bool         single_grid,
+        cudaStream_t stream) {
+    const char *tag = single_grid
+        ? "ds4_mmq_q4_K_grouped_dense_single_grid"
+        : "ds4_mmq_q4_K_grouped_dense";
+    const int pre_enqueue_failure = single_grid
+        ? DS4_MMQ_NOT_APPLICABLE
+        : -1;
+    if (!W || !X || !out) {
+        fprintf(stderr, "%s: null pointer\n", tag);
+        return pre_enqueue_failure;
+    }
+    if (M <= 0 || N <= 0 || K <= 0 || n_groups <= 0 ||
+        K % QK_K != 0) {
+        fprintf(stderr, "%s: bad shape M=%d N=%d K=%d groups=%d\n",
+                tag, M, N, K, n_groups);
+        return pre_enqueue_failure;
+    }
+
+    const int dev = ggml_cuda_get_device();
+    const int cc = ggml_cuda_info().devices[dev].cc;
+#if !defined(GGML_USE_HIP)
+    const bool q81_disable =
+        getenv("DS4_CUDA_NO_Q4_GROUPED_ATTN_A_Q81") != nullptr ||
+        getenv("DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL") != nullptr ||
+        getenv("DS4_CUDA_NO_Q4_GROUPED_ATTN_A") != nullptr ||
+        getenv("DS4_CUDA_NO_Q4_GB10_FAST") != nullptr;
+    const bool q81_require = ds4_q4_grouped_q81_env_enabled(
+        "DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_Q81");
+    const bool q81_eligible =
+        gb10_optimizations_enabled() &&
+        cc == GGML_CUDA_CC_DGX_SPARK && M == 1024 && N > 8 &&
+        N <= INT32_MAX / (8*4096) && K == 4096 && n_groups == 8 &&
+        (((uintptr_t)X & 15u) == 0u);
+    if (q81_require && (q81_disable || !q81_eligible)) {
+        fprintf(stderr,
+                "%s: required grouped K4096/G8 Q8_1 quantizer is not "
+                "eligible\n",
+                tag);
+        return DS4_MMQ_NOT_APPLICABLE;
+    }
+    const bool use_specialized_q81 = q81_eligible && !q81_disable;
+#endif
+    ggml_backend_cuda_context *ctx = get_ctx_for_device(dev);
+    if (!ctx) {
+        fprintf(stderr, "%s: failed to get cuda context for device %d\n",
+                tag, dev);
+        return pre_enqueue_failure;
+    }
+    ds4_pool_set_stream(stream);
+
+    const int64_t ne10_padded = GGML_PAD((int64_t)K, MATRIX_ROW_PADDING);
+    const size_t blocks_per_col =
+        (size_t)ne10_padded / (4u * (size_t)QK8_1);
+    const size_t bytes_per_col =
+        blocks_per_col * sizeof(block_q8_1_mmq);
+    if ((size_t)N > SIZE_MAX / bytes_per_col) return pre_enqueue_failure;
+    const size_t channel_bytes = (size_t)N * bytes_per_col;
+    if ((size_t)n_groups > SIZE_MAX / channel_bytes) {
+        return pre_enqueue_failure;
+    }
+    const size_t payload_bytes = (size_t)n_groups * channel_bytes;
+    const size_t slack_blocks = (size_t)get_mmq_x_max_host(cc);
+    if (slack_blocks > SIZE_MAX / sizeof(block_q8_1_mmq)) {
+        return pre_enqueue_failure;
+    }
+    const size_t slack_bytes = slack_blocks * sizeof(block_q8_1_mmq);
+    if (payload_bytes > SIZE_MAX - slack_bytes) return pre_enqueue_failure;
+
+    const int64_t row_blocks = (int64_t)K / QK_K;
+    if ((size_t)M > SIZE_MAX / (size_t)row_blocks /
+                        sizeof(block_q4_K)) return pre_enqueue_failure;
+    const size_t group_weight_bytes =
+        (size_t)M * (size_t)row_blocks * sizeof(block_q4_K);
+    const int64_t group_weight_blocks = (int64_t)M * row_blocks;
+    const int64_t low_dim = (int64_t)M * n_groups;
+    if (low_dim > INT_MAX ||
+        (uint64_t)low_dim > UINT64_MAX / (uint64_t)N) {
+        return pre_enqueue_failure;
+    }
+    /* The grouped kernel ABI narrows strides and weight-block offsets to int.
+     * Reject before allocating or enqueueing so an optional caller can safely
+     * fall back to the established per-group launch loop. */
+    if (single_grid &&
+        (n_groups > 65535 || group_weight_blocks > INT_MAX ||
+         group_weight_blocks * (int64_t)n_groups > INT_MAX ||
+         channel_bytes / sizeof(int) > (size_t)INT_MAX)) {
+        return DS4_MMQ_NOT_APPLICABLE;
+    }
+
+    ggml_cuda_pool_alloc<char> y_q8_1(
+        ctx->pool(), payload_bytes + slack_bytes);
+    ybuf_memset(y_q8_1.get(), payload_bytes + slack_bytes, stream);
+#if !defined(GGML_USE_HIP)
+    if (use_specialized_q81) {
+        quantize_mmq_q8_1_q4_grouped_k4096_g8x2_cuda(
+            X, (void *)y_q8_1.get(), N, stream);
+    } else
+#endif
+    {
+        quantize_mmq_q8_1_cuda(
+            X, /*ids=*/nullptr, (void *)y_q8_1.get(), GGML_TYPE_Q4_K,
+            /*ne00=*/K,
+            /*s01=*/(int64_t)n_groups * K,
+            /*s02=*/(int64_t)K,
+            /*s03=*/(int64_t)n_groups * N * K,
+            /*ne0=*/ne10_padded, /*ne1=*/N,
+            /*ne2=*/n_groups, /*ne3=*/1, stream);
+    }
+    cudaError_t err = cudaGetLastError();
+    if (err != cudaSuccess) {
+        fprintf(stderr, "%s: quantize failed: %s\n",
+                tag, cudaGetErrorString(err));
+        return -2;
+    }
+
+    if (out_memset_enabled()) {
+        cudaMemsetAsync(out, 0,
+                        (size_t)N * (size_t)low_dim * sizeof(float), stream);
+    }
+    const bool use_stream_k =
+        (GGML_CUDA_CC_IS_NVIDIA(cc) &&
+         ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_VOLTA) ||
+        GGML_CUDA_CC_IS_CDNA(cc);
+    if (single_grid) {
+        const mmq_args args = {
+            /*x=*/(const char *)W,
+            /*type_x=*/GGML_TYPE_Q4_K,
+            /*y=*/(const int *)y_q8_1.get(),
+            /*ids_dst=*/nullptr,
+            /*expert_bounds=*/nullptr,
+            /*dst=*/out,
+            /*ncols_x=*/(int64_t)K,
+            /*nrows_x=*/(int64_t)M,
+            /*ncols_dst=*/(int64_t)N,
+            /*stride_row_x=*/row_blocks,
+            /*ncols_y=*/(int64_t)N,
+            /*nrows_dst=*/low_dim,
+            /*nchannels_x=*/(int64_t)n_groups,
+            /*nchannels_y=*/(int64_t)n_groups,
+            /*stride_channel_x=*/group_weight_blocks,
+            /*stride_channel_y=*/(int64_t)(channel_bytes / sizeof(int)),
+            /*stride_channel_dst=*/(int64_t)M,
+            /*nsamples_x=*/1,
+            /*nsamples_y=*/1,
+            /*stride_sample_x=*/0,
+            /*stride_sample_y=*/0,
+            /*stride_sample_dst=*/0,
+            /*use_stream_k=*/use_stream_k,
+            /*ncols_max=*/(int64_t)N,
+        };
+        mul_mat_q_case_grouped_channels<GGML_TYPE_Q4_K>(*ctx, args, stream);
+        err = cudaGetLastError();
+        if (err != cudaSuccess) {
+            fprintf(stderr, "%s: grouped launch failed: %s\n",
+                    tag, cudaGetErrorString(err));
+            return -3;
+        }
+        return 0;
+    }
+    for (int g = 0; g < n_groups; ++g) {
+        const mmq_args args = {
+            /*x=*/(const char *)W + (size_t)g * group_weight_bytes,
+            /*type_x=*/GGML_TYPE_Q4_K,
+            /*y=*/(const int *)(y_q8_1.get() + (size_t)g * channel_bytes),
+            /*ids_dst=*/nullptr,
+            /*expert_bounds=*/nullptr,
+            /*dst=*/out + (int64_t)g * M,
+            /*ncols_x=*/(int64_t)K,
+            /*nrows_x=*/(int64_t)M,
+            /*ncols_dst=*/(int64_t)N,
+            /*stride_row_x=*/row_blocks,
+            /*ncols_y=*/(int64_t)N,
+            /*nrows_dst=*/low_dim,
+            /*nchannels_x=*/1,
+            /*nchannels_y=*/1,
+            /*stride_channel_x=*/0,
+            /*stride_channel_y=*/(int64_t)(channel_bytes / sizeof(int)),
+            /*stride_channel_dst=*/0,
+            /*nsamples_x=*/1,
+            /*nsamples_y=*/1,
+            /*stride_sample_x=*/0,
+            /*stride_sample_y=*/(int64_t)(channel_bytes / sizeof(int)),
+            /*stride_sample_dst=*/0,
+            /*use_stream_k=*/use_stream_k,
+            /*ncols_max=*/(int64_t)N,
+        };
+        mul_mat_q_case<GGML_TYPE_Q4_K>(*ctx, args, stream);
+        err = cudaGetLastError();
+        if (err != cudaSuccess) {
+            fprintf(stderr, "%s: group %d launch failed: %s\n",
+                    tag, g, cudaGetErrorString(err));
+            return -3;
+        }
+    }
     return 0;
 }
 
@@ -895,6 +2162,62 @@ extern "C" int ds4_mmq_q4_K_dense(
     return ds4_mmq_dense_impl<GGML_TYPE_Q4_K>("ds4_mmq_q4_K_dense", W, X, out, M, N, K, stream);
 }
 
+extern "C" int ds4_mmq_q4_K_dense_pair(
+        const void * W0, const void * W1, const float * X,
+        float * out0, float * out1,
+        int M0, int M1, int N, int K, cudaStream_t stream) {
+    return ds4_mmq_q4_K_dense_pair_impl(
+        W0, W1, X, out0, out1, M0, M1, N, K, stream);
+}
+
+extern "C" int ds4_mmq_q4_K_grouped_dense(
+        const void *W, const float *X, float *out,
+        int M, int N, int K, int n_groups, cudaStream_t stream) {
+    return ds4_mmq_q4_K_grouped_dense_impl(
+        W, X, out, M, N, K, n_groups, false, stream);
+}
+
+extern "C" int ds4_mmq_q4_K_grouped_dense_single_grid(
+        const void *W, const float *X, float *out,
+        int M, int N, int K, int n_groups, cudaStream_t stream) {
+    return ds4_mmq_q4_K_grouped_dense_impl(
+        W, X, out, M, N, K, n_groups, true, stream);
+}
+
+#if !defined(GGML_USE_HIP)
+extern "C" size_t ds4_mmq_q4_K_grouped_q8_1_scratch_bytes_for_test(int N) {
+    if (N <= 0 || N > INT32_MAX / (8*4096)) return 0u;
+    constexpr size_t blocks_per_token = 8u * 4096u / (4u * QK8_1);
+    if ((size_t)N > SIZE_MAX / blocks_per_token /
+                        sizeof(block_q8_1_mmq)) {
+        return 0u;
+    }
+    return (size_t)N * blocks_per_token * sizeof(block_q8_1_mmq);
+}
+
+extern "C" int ds4_mmq_q4_K_grouped_quantize_q8_1_for_test(
+        const float *X, void *q8, size_t q8_bytes, int N,
+        int use_specialized, cudaStream_t stream) {
+    const size_t required =
+        ds4_mmq_q4_K_grouped_q8_1_scratch_bytes_for_test(N);
+    if (!X || !q8 || required == 0u || q8_bytes < required ||
+        (((uintptr_t)X & 15u) != 0u)) {
+        return -1;
+    }
+    if (use_specialized) {
+        quantize_mmq_q8_1_q4_grouped_k4096_g8x2_cuda(
+            X, q8, N, stream);
+    } else {
+        quantize_mmq_q8_1_cuda(
+            X, /*ids=*/nullptr, q8, GGML_TYPE_Q4_K,
+            /*ne00=*/4096, /*s01=*/8*4096, /*s02=*/4096,
+            /*s03=*/(int64_t)N*8*4096,
+            /*ne0=*/4096, /*ne1=*/N, /*ne2=*/8, /*ne3=*/1, stream);
+    }
+    return cudaGetLastError() == cudaSuccess ? 0 : -2;
+}
+#endif
+
 extern "C" int ds4_mmq_mxfp4_dense(
         const void * W, const float * X, float * out,
         int M, int N, int K, cudaStream_t stream) {
@@ -1163,7 +2486,7 @@ int ds4_mmq_moe_impl(
         fprintf(stderr, "%s: mul_mat_q_case (moe) launch failed: %s\n", tag, cudaGetErrorString(err));
         return -4;
     }
-    if (sanitize_out) {
+    if (sanitize_out && type != GGML_TYPE_Q4_K) {
         ds4_mmq_sanitize_f32(out_f32, (uint64_t)M * (uint64_t)ne_get_rows, stream);
     }
     return 0;
@@ -1336,6 +2659,16 @@ int ds4_mmq_moe_pair_impl(
     const int64_t s01          = (int64_t)K / blck;
     const int64_t s02          = (int64_t)M * s01;
 
+    // Past the shared-memory cap the launcher takes the bit-identical global
+    // variant unless the authoritative large-map kill switch rejects it.
+    if ((size_t)n_tokens * 4u > ggml_cuda_info().devices[dev].smpbo &&
+        !ds4_mmid_large_enabled()) {
+        fprintf(stderr,
+                "%s: n_tokens=%d exceeds mm_ids_helper shared-mem cap; "
+                "falling back\n", tag, n_tokens);
+        return -1;
+    }
+
     ggml_cuda_pool_alloc<int32_t> ids_src1_alloc;
     ggml_cuda_pool_alloc<int32_t> ids_dst_alloc;
     ggml_cuda_pool_alloc<int32_t> expert_bounds_alloc;
@@ -1408,29 +2741,63 @@ int ds4_mmq_moe_pair_impl(
         ids_src1 = (int32_t *)ids_src1_raw;
         ids_dst = (int32_t *)ids_dst_raw;
         expert_bounds = (int32_t *)expert_bounds_raw;
-    } else if (persistent_pair_maps) {
-        const auto & maps = g_mmq_pair_maps[dev];
-        ids_src1 = maps.ids_src1;
-        ids_dst = maps.ids_dst;
-        expert_bounds = maps.expert_bounds;
-    } else {
-        ids_src1 = ids_src1_alloc.alloc(ctx->pool(), ne_get_rows);
-        ids_dst = ids_dst_alloc.alloc(ctx->pool(), ne_get_rows);
-        expert_bounds = expert_bounds_alloc.alloc(ctx->pool(), n_experts + 1);
     }
 
-    const int si1  = n_expert_used;
-    const int sis1 = 1;
+    /* `fused_raw` is the only grouped path that owns neither aligned weights
+     * nor caller scratch.  Its input Q8 is dead after gate/up, before down Q8
+     * is produced, so one max-sized range can back both phases.  Resolve the
+     * opt-in arena (or allocate its one-block pool fallback) before the first
+     * expert-map enqueue; no mid-pipeline allocation failure can strand a
+     * partially submitted candidate. */
+    const bool grouped_raw_q81 = profile_fused_prefill &&
+        type == GGML_TYPE_IQ2_XXS && fused_down != nullptr &&
+        !direct_gateup_q8 && xa_soa == nullptr && xb_soa == nullptr &&
+        fused_down->W_soa == nullptr;
+    size_t grouped_down_q8_bytes = 0;
+    size_t grouped_q81_required = 0;
+    std::unique_lock<std::mutex> grouped_q81_lease(
+        g_q81_state_mutex, std::defer_lock);
+    ggml_cuda_pool_alloc<char> grouped_q81_pool;
+    char *grouped_q81_scratch = nullptr;
+    if (grouped_raw_q81 && q81_persistent_requested()) {
+        const int64_t down_padded = GGML_PAD((int64_t)M, MATRIX_ROW_PADDING);
+        const size_t tail =
+            (size_t)get_mmq_x_max_host(cc) * sizeof(block_q8_1_mmq);
+        if ((size_t)ne_get_rows > SIZE_MAX / (size_t)down_padded ||
+            (size_t)ne_get_rows * (size_t)down_padded >
+                (SIZE_MAX - tail) / sizeof(block_q8_1)) {
+            return -98;
+        }
+        grouped_down_q8_bytes =
+            (size_t)ne_get_rows * (size_t)down_padded *
+                sizeof(block_q8_1) / QK8_1 + tail;
+        grouped_q81_required = nbytes_src1_q8_1 > grouped_down_q8_bytes
+            ? nbytes_src1_q8_1 : grouped_down_q8_bytes;
+        grouped_q81_scratch = q81_grouped_persistent_acquire(
+            dev, stream, grouped_q81_required, &grouped_q81_lease);
+        if (!grouped_q81_scratch) {
+            grouped_q81_scratch = grouped_q81_pool.alloc(
+                ctx->pool(), grouped_q81_required);
+        }
+    }
 
-    // Same cap guard as ds4_mmq_moe_impl (see comment there): past the smem
-    // cap the launcher takes the bit-identical global variant (P5); only
-    // refuse with DS4_MMID_LARGE=0.
-    if ((size_t)n_tokens * 4u > ggml_cuda_info().devices[dev].smpbo && !ds4_mmid_large_enabled()) {
-        fprintf(stderr, "%s: n_tokens=%d exceeds mm_ids_helper shared-mem cap; falling back\n",
-                tag, n_tokens);
-        return -1;
+    if (!direct_gateup_q8) {
+        if (persistent_pair_maps) {
+            const auto & maps = g_mmq_pair_maps[dev];
+            ids_src1 = maps.ids_src1;
+            ids_dst = maps.ids_dst;
+            expert_bounds = maps.expert_bounds;
+        } else {
+            ids_src1 = ids_src1_alloc.alloc(ctx->pool(), ne_get_rows);
+            ids_dst = ids_dst_alloc.alloc(ctx->pool(), ne_get_rows);
+            expert_bounds = expert_bounds_alloc.alloc(
+                ctx->pool(), n_experts + 1);
+        }
     }
 
+    const int si1  = n_expert_used;
+    const int sis1 = 1;
+
     cudaError_t err = cudaSuccess;
     {
         ds4_mmq_nvtx_scope stage(
@@ -1478,7 +2845,9 @@ int ds4_mmq_moe_pair_impl(
     ggml_cuda_pool_alloc<char> src1_q8_1_alloc;
     char *src1_q8_1 = direct_gateup_q8
         ? (char *)fused_down->input_q8_scratch
-        : src1_q8_1_alloc.alloc(ctx->pool(), nbytes_src1_q8_1);
+        : (grouped_q81_scratch
+            ? grouped_q81_scratch
+            : src1_q8_1_alloc.alloc(ctx->pool(), nbytes_src1_q8_1));
 
     // S1.1a fix (same as the dense/moe paths): zero the over-allocated mmq Y buffer
     // so the kernel's unconditional masked-out tail-tile read (mmq.cuh:3528) returns
@@ -1760,8 +3129,11 @@ int ds4_mmq_moe_pair_impl(
             (size_t)ne_get_rows * (size_t)down_ne10_padded * sizeof(block_q8_1) / QK8_1;
         const size_t tail_q8_bytes =
             (size_t)get_mmq_x_max_host(cc) * sizeof(block_q8_1_mmq);
-        ggml_cuda_pool_alloc<char> down_q8_1(
-            ctx->pool(), logical_q8_bytes + tail_q8_bytes);
+        const size_t down_q8_bytes = logical_q8_bytes + tail_q8_bytes;
+        ggml_cuda_pool_alloc<char> down_q8_1_pool;
+        char *down_q8_1 = grouped_q81_scratch
+            ? grouped_q81_scratch
+            : down_q8_1_pool.alloc(ctx->pool(), down_q8_bytes);
 
         const uint64_t mid_values = (uint64_t)ne_get_rows * (uint64_t)M;
         {
@@ -1769,7 +3141,7 @@ int ds4_mmq_moe_pair_impl(
                     "ds4/prefill/moe/swiglu_down_quant",
                     ds4_mmq_nvtx_payload((uint32_t)ne_get_rows, (uint32_t)M),
                     nvtx_prefill);
-            ybuf_memset(down_q8_1.get(), logical_q8_bytes + tail_q8_bytes, stream);
+            ybuf_memset(down_q8_1, down_q8_bytes, stream);
             ds4_swiglu_weighted_f32<<<
                 (uint32_t)((mid_values + 255u) / 256u), 256, 0, stream>>>(
                     out_a, out_b, fused_down->router_weights,
@@ -1782,7 +3154,7 @@ int ds4_mmq_moe_pair_impl(
             }
 
             quantize_mmq_q8_1_cuda(
-                fused_down->mid_f32, ids_dst, (void *)down_q8_1.get(),
+                fused_down->mid_f32, ids_dst, (void *)down_q8_1,
                 GGML_TYPE_Q2_K, /*ne00=*/M, /*s01=*/M,
                 /*s02=*/(int64_t)M, /*s03=*/(int64_t)M * ne_get_rows,
                 /*ne0=*/down_ne10_padded, /*ne1=*/ne_get_rows,
@@ -1807,7 +3179,7 @@ int ds4_mmq_moe_pair_impl(
         const mmq_args down_args = {
             /*x=*/(const char *)fused_down->W,
             /*type_x=*/GGML_TYPE_Q2_K,
-            /*y=*/(const int *)down_q8_1.get(),
+            /*y=*/(const int *)down_q8_1,
             /*ids_dst=*/ids_dst,
             /*expert_bounds=*/expert_bounds,
             /*dst=*/fused_down->out,
@@ -1848,7 +3220,7 @@ int ds4_mmq_moe_pair_impl(
                 down_done = ds4_mmq_q2_K_moe_d2r_launch(
                         fused_down->W_soa,
                         fused_down->soa_blocks,
-                        down_q8_1.get(),
+                        down_q8_1,
                         ids_dst,
                         expert_bounds,
                         fused_down->out,
@@ -1876,7 +3248,7 @@ int ds4_mmq_moe_pair_impl(
             }
         }
     }
-    if (sanitize_out) {
+    if (sanitize_out && type != GGML_TYPE_Q4_K) {
         ds4_mmq_sanitize_f32(out_a, (uint64_t)M * (uint64_t)ne_get_rows, stream);
         ds4_mmq_sanitize_f32(out_b, (uint64_t)M * (uint64_t)ne_get_rows, stream);
     }
@@ -2110,6 +3482,100 @@ extern "C" int ds4_mmq_iq2_xxs_q2_K_moe_fused_direct_scratch_sizes(
     return 0;
 }
 
+/* Canonical-GGUF/raw counterpart of the materialized aligned-SoA pipeline.
+ * SSD streaming compacts the selected experts and remaps ids before this
+ * boundary, so n_experts describes the compact table rather than the model's
+ * global expert count.  Keep all preflight ahead of the single pair_impl
+ * invocation: after that point map/quantize/MMQ work may already be queued and
+ * a negative result must be propagated instead of being converted into the
+ * retryable NOT_APPLICABLE result. */
+extern "C" int ds4_mmq_iq2_xxs_q2_K_moe_fused_raw(
+        const void * W_gate, const void * W_up, const void * W_down,
+        const float * X, const int32_t * ids, const float * router_weights,
+        float * gate, float * up, float * mid_f32, float * down,
+        int expert_mid_dim, int expert_in_dim, int out_dim,
+        int n_tokens, int n_experts, int n_expert_used,
+        float clamp, cudaStream_t stream) {
+    if (!W_gate || !W_up || !W_down || !X || !ids || !router_weights ||
+        !gate || !up || !mid_f32 || !down ||
+        expert_mid_dim <= 0 || expert_in_dim <= 0 || out_dim <= 0 ||
+        n_tokens <= 0 || n_experts <= 0 || n_expert_used <= 0 ||
+        n_expert_used > n_experts || n_experts == INT_MAX ||
+        n_tokens >= (1 << 22) || n_expert_used >= (1 << 10) ||
+        expert_in_dim % 256 != 0 || expert_mid_dim % 256 != 0) {
+        return DS4_MMQ_NOT_APPLICABLE;
+    }
+
+    const size_t nt = (size_t)n_tokens;
+    const size_t nu = (size_t)n_expert_used;
+    const size_t mid = (size_t)expert_mid_dim;
+    const size_t in = (size_t)expert_in_dim;
+    const size_t out = (size_t)out_dim;
+    if (nt > SIZE_MAX / nu) return DS4_MMQ_NOT_APPLICABLE;
+    const size_t assignments = nt * nu;
+    if (assignments > SIZE_MAX / mid ||
+        assignments * mid > SIZE_MAX / sizeof(float) ||
+        assignments > SIZE_MAX / out ||
+        assignments * out > SIZE_MAX / sizeof(float) ||
+        assignments > SIZE_MAX / in ||
+        assignments * in > SIZE_MAX / 512u ||
+        assignments * mid > SIZE_MAX / 512u) {
+        return DS4_MMQ_NOT_APPLICABLE;
+    }
+
+    /* Bound the raw per-expert strides used by both MMQs before any pool or
+     * stream operation.  All dimensions enter as int, but their products do
+     * not necessarily fit int64_t. */
+    const int64_t iq2_k_blocks = expert_in_dim / 256;
+    const int64_t q2_k_blocks = expert_mid_dim / 256;
+    if ((int64_t)n_experts > INT64_MAX / expert_mid_dim ||
+        (int64_t)n_experts * expert_mid_dim > INT64_MAX / iq2_k_blocks ||
+        (int64_t)n_experts > INT64_MAX / out_dim ||
+        (int64_t)n_experts * out_dim > INT64_MAX / q2_k_blocks) {
+        return DS4_MMQ_NOT_APPLICABLE;
+    }
+
+    const int dev = ggml_cuda_get_device();
+    if (dev < 0 || dev >= GGML_CUDA_MAX_DEVICES) {
+        return DS4_MMQ_NOT_APPLICABLE;
+    }
+    const int cc = ggml_cuda_info().devices[dev].cc;
+    if (!ds4_mmq_k_tile_supported<GGML_TYPE_IQ2_XXS>(
+            "ds4_mmq_iq2_xxs_q2_K_moe_fused_raw", expert_in_dim, cc) ||
+        !get_ctx_for_device(dev) ||
+        ((size_t)n_tokens * 4u > ggml_cuda_info().devices[dev].smpbo &&
+         !ds4_mmid_large_enabled())) {
+        return DS4_MMQ_NOT_APPLICABLE;
+    }
+
+    const ds4_mmq_fused_down fused_down = {
+        W_down,
+        nullptr,
+        0,
+        router_weights,
+        mid_f32,
+        down,
+        out_dim,
+        clamp,
+        false,
+        nullptr,
+        0,
+        nullptr,
+        0,
+        nullptr,
+        0,
+        nullptr,
+        0,
+    };
+    return ds4_mmq_moe_pair_impl<GGML_TYPE_IQ2_XXS, true>(
+        "ds4_mmq_iq2_xxs_q2_K_moe_fused_raw",
+        W_gate, W_up, X, ids, gate, up,
+        expert_mid_dim, expert_in_dim, n_tokens, n_experts, n_expert_used,
+        stream,
+        nullptr, nullptr, 0,
+        /*sanitize_out=*/false, &fused_down);
+}
+
 /* Aligned-artifact production fast path: gate/up accumulators stay in
  * registers, weighted SwiGLU is quantized directly into down_q8_scratch by
  * the fused D2R kernel, and only the pair-major down output is materialized.
@@ -2135,11 +3601,30 @@ extern "C" int ds4_mmq_iq2_xxs_q2_K_moe_fused_direct_soa(
         !input_q8_scratch || input_q8_scratch_bytes == 0 ||
         !down_q8_scratch || down_q8_scratch_bytes == 0 ||
         !work_scratch || work_scratch_bytes == 0 || !down) {
-        return -1;
-    }
-    const size_t down_bytes =
-        (size_t)n_tokens * (size_t)n_expert_used *
-        (size_t)out_dim * sizeof(float);
+        return DS4_MMQ_NOT_APPLICABLE;
+    }
+    const size_t nt = (size_t)n_tokens;
+    const size_t nu = (size_t)n_expert_used;
+    const size_t od = (size_t)out_dim;
+    if (nt > SIZE_MAX / nu || nt * nu > SIZE_MAX / od ||
+        nt * nu * od > SIZE_MAX / sizeof(float)) {
+        return DS4_MMQ_NOT_APPLICABLE;
+    }
+    const size_t assignments = nt * nu;
+    const size_t expert_in = (size_t)expert_in_dim;
+    const size_t expert_mid = (size_t)expert_mid_dim;
+    if (assignments > SIZE_MAX / expert_in ||
+        assignments > SIZE_MAX / expert_mid) {
+        return DS4_MMQ_NOT_APPLICABLE;
+    }
+    /* The internal MMQ producer sizes multiply these logical element counts
+     * by block structs before dividing by their values-per-block. Keep ample
+     * headroom for that multiplication and its fixed tail allocation. */
+    if (assignments * expert_in > SIZE_MAX / 512u ||
+        assignments * expert_mid > SIZE_MAX / 512u) {
+        return DS4_MMQ_NOT_APPLICABLE;
+    }
+    const size_t down_bytes = assignments * od * sizeof(float);
     if (ds4_mmq_scratch_overlaps(
             input_q8_scratch, input_q8_scratch_bytes,
             down_q8_scratch, down_q8_scratch_bytes) ||
@@ -2155,12 +3640,20 @@ extern "C" int ds4_mmq_iq2_xxs_q2_K_moe_fused_direct_soa(
             down_q8_scratch, down_q8_scratch_bytes, down, down_bytes) ||
         ds4_mmq_scratch_overlaps(
             work_scratch, work_scratch_bytes, down, down_bytes)) {
-        return -1;
+        return DS4_MMQ_NOT_APPLICABLE;
+    }
+    const int64_t iq2_k_blocks = expert_in_dim / 256;
+    const int64_t q2_k_blocks = expert_mid_dim / 256;
+    if ((int64_t)n_experts > INT64_MAX / expert_mid_dim ||
+        (int64_t)n_experts * expert_mid_dim > INT64_MAX / iq2_k_blocks ||
+        (int64_t)n_experts > INT64_MAX / (out_dim / 2) ||
+        (int64_t)n_experts * (out_dim / 2) > INT64_MAX / q2_k_blocks) {
+        return DS4_MMQ_NOT_APPLICABLE;
     }
     const int64_t iq2_blocks =
-        (int64_t)n_experts * expert_mid_dim * (expert_in_dim / 256);
+        (int64_t)n_experts * expert_mid_dim * iq2_k_blocks;
     const int64_t q2_pairs =
-        (int64_t)n_experts * (out_dim / 2) * (expert_mid_dim / 256);
+        (int64_t)n_experts * (out_dim / 2) * q2_k_blocks;
     const ds4_mmq_fused_down fused_down = {
         W_down,
         (const char *)W_down,
@@ -2180,13 +3673,17 @@ extern "C" int ds4_mmq_iq2_xxs_q2_K_moe_fused_direct_soa(
         input_q8_ext,
         input_q8_ext_bytes,
     };
-    return ds4_mmq_moe_pair_impl<GGML_TYPE_IQ2_XXS, true>(
+    const int rc = ds4_mmq_moe_pair_impl<GGML_TYPE_IQ2_XXS, true>(
         "ds4_mmq_iq2_xxs_q2_K_moe_fused_direct_soa",
         W_gate, W_up, X, ids, nullptr, nullptr,
         expert_mid_dim, expert_in_dim, n_tokens, n_experts, n_expert_used,
         stream,
         (const char *)W_gate, (const char *)W_up, iq2_blocks,
         /*sanitize_out=*/false, &fused_down);
+    if (rc == -1 || (rc <= -91 && rc >= -97)) {
+        return DS4_MMQ_NOT_APPLICABLE;
+    }
+    return rc;
 }
 
 extern "C" int ds4_mmq_q4_K_moe_pair(
@@ -2291,11 +3788,10 @@ int ds4_mmq_moe_vec_impl(
     const int64_t ne10_padded = GGML_PAD((int64_t)K, MATRIX_ROW_PADDING);
     const size_t  nbytes_q8_1 = (size_t)n_tokens * ne10_padded *
                                 sizeof(block_q8_1) / QK8_1;
-    // Step 7 task #29: experimental persistent Q8_1 scratch.  Avoids
-    // pool_alloc (cudaMallocAsync) graph nodes whose pointer baked at
-    // capture time may not match the address resolved at replay.  When
-    // disabled (default) or when the persistent buffer is too small,
-    // fall back to the pool path.  See ds4_mmq_init for setup.
+    // Step 7 task #29: experimental persistent Q8_1 scratch. It avoids
+    // captured pool alloc/free nodes as a performance experiment. When
+    // disabled (default) or too small, the valid same-stream graph-memory
+    // pool path remains the fallback. See ds4_mmq_init for setup.
     ggml_cuda_pool_alloc<char> src1_q8_1_pool;
     char *src1_q8_1_ptr = nullptr;
     if (g_q81_scratch_enabled && g_q81_scratch_ptr &&
@@ -3047,6 +4543,162 @@ int ds4_mmq_moe_pair_vec_impl(
     return 0;
 }
 
+/* Diagnostic counters are host-dispatch counters.  CUDA graph replays do not
+ * re-enter this wrapper, so they are deliberately not presented as kernel
+ * execution counts.  They are still a fail-closed coverage signal: a GB10
+ * model run must observe at least one candidate and one use before this path
+ * can be promoted from opt-in to default. */
+static uint64_t g_q4_k1024_persistent_candidates;
+static uint64_t g_q4_k1024_persistent_uses;
+static uint64_t g_q4_k1024_persistent_fallbacks;
+static uint64_t g_q4_k1024_persistent_require_failures;
+static uint64_t g_q4_k1024_persistent_oracle_calls;
+static uint64_t g_q4_k1024_persistent_oracle_mismatches;
+static uint64_t g_q4_k1024_persistent_oracle_skips;
+static int g_q4_k1024_persistent_report_registered;
+static int g_q4_k1024_persistent_oracle_mismatch_reported;
+
+static bool q4_k1024_env_flag(const char *name) {
+    const char *value = getenv(name);
+    return value && value[0] && strcmp(value, "0") != 0;
+}
+
+static void q4_k1024_persistent_report(void) {
+    fprintf(stderr,
+            "ds4: CUDA Q4 K1024 persistent: "
+            "candidates=%llu uses=%llu fallbacks=%llu "
+            "require_failures=%llu oracle_calls=%llu "
+            "oracle_mismatches=%llu oracle_skips=%llu "
+            "(host dispatches; graph replays excluded, canonical oracle output retained)\n",
+            (unsigned long long)g_q4_k1024_persistent_candidates,
+            (unsigned long long)g_q4_k1024_persistent_uses,
+            (unsigned long long)g_q4_k1024_persistent_fallbacks,
+            (unsigned long long)g_q4_k1024_persistent_require_failures,
+            (unsigned long long)g_q4_k1024_persistent_oracle_calls,
+            (unsigned long long)g_q4_k1024_persistent_oracle_mismatches,
+            (unsigned long long)g_q4_k1024_persistent_oracle_skips);
+}
+
+static void q4_k1024_persistent_maybe_register_report(void) {
+    if (!g_q4_k1024_persistent_report_registered &&
+        (q4_k1024_env_flag("DS4_CUDA_Q4_K1024_PERSISTENT_STATS") ||
+         q4_k1024_env_flag("DS4_CUDA_Q4_K1024_PERSISTENT_ORACLE"))) {
+        g_q4_k1024_persistent_report_registered = 1;
+        (void)atexit(q4_k1024_persistent_report);
+    }
+}
+
+__global__ static void q4_K_k1024_bitwise_compare_kernel(
+        uint32_t    *mismatch,
+        const float *candidate,
+        const float *reference,
+        uint64_t     count) {
+    const uint64_t i = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x;
+    if (i < count &&
+        __float_as_uint(candidate[i]) != __float_as_uint(reference[i])) {
+        atomicExch(mismatch, 1u);
+    }
+}
+
+/* GB10 AProjQ4 Q-b decode specialization (M=32768, N=1, K=1024).
+ *
+ * The canonical MMVQ small-K launch uses four warps to evaluate four rows:
+ * warp 0 owns Q4_K superblocks 0/1, warp 1 owns 2/3, and warps 2/3
+ * contribute +0.0f.  Its reduction first adds the three peer-warp partials
+ * lane by lane, then applies warp_reduce_sum's XOR tree.  Two independent
+ * four-warp groups below preserve that assignment and arithmetic order while
+ * persistent CTAs walk eight-row tiles at a grid stride.  The immutable
+ * canonical Q8_1 activation is staged once per CTA; no Q8_K re-quantization
+ * or Q4_K weight repack is involved.
+ *
+ * Keep this kernel paired with the exact M/N/K admission in
+ * ds4_mmq_dense_vec_impl.  Generalizing the row-warp mapping would change
+ * floating-point association relative to MMVQ. */
+static __global__ __launch_bounds__(256, 4) void
+q4_K_dense_vec_k1024_persistent_kernel(
+        const block_q4_K * __restrict__ W,
+        const block_q8_1 * __restrict__ x8,
+        float            * __restrict__ out,
+        int                              M) {
+    constexpr int k_q4_blocks = 4;       /* 1024 / QK_K */
+    constexpr int k_q8_blocks = 32;      /* 1024 / QK8_1 */
+    constexpr int k_rows_per_group = 4;  /* canonical MMVQ small-K tile */
+    constexpr int k_groups = 2;
+
+    /* block_q8_1 is 36 bytes.  A uint32_t backing array both copies it
+     * efficiently and preserves the alignment required by vec_dot's int
+     * loads from qs. */
+    __shared__ __align__(16) uint32_t x8_words[
+        (k_q8_blocks * sizeof(block_q8_1)) / sizeof(uint32_t)];
+    __shared__ float partial[k_groups][3][k_rows_per_group][32];
+
+    const uint32_t *x8_src = (const uint32_t *)x8;
+    for (uint32_t i = threadIdx.x;
+         i < (uint32_t)(sizeof(x8_words) / sizeof(x8_words[0]));
+         i += blockDim.x) {
+        x8_words[i] = x8_src[i];
+    }
+    __syncthreads();
+
+    const block_q8_1 *x8_shared = (const block_q8_1 *)x8_words;
+    const uint32_t lane = threadIdx.x & 31u;
+    const uint32_t warp = threadIdx.x >> 5u;
+    const uint32_t group = warp >> 2u;
+    const uint32_t warp_in_group = warp & 3u;
+    const uint32_t group_tid = warp_in_group * 32u + lane;
+    const uint64_t row_tiles = ((uint64_t)(uint32_t)M + 7u) / 8u;
+
+    /* tile, row_tiles, and gridDim.x are block-uniform, and this loop has no
+     * divergent exit.  Every thread therefore reaches both barriers below on
+     * every iteration; unrolling is unrelated to their correctness. */
+    for (uint64_t tile = blockIdx.x; tile < row_tiles; tile += gridDim.x) {
+        const uint32_t row0 = (uint32_t)(tile * 8u) +
+                              group * k_rows_per_group;
+        float tmp[k_rows_per_group] = {0.0f};
+
+        /* This is the canonical N=1, K=1024 MMVQ small-K loop verbatim:
+         * qi/vdr = 16 and blocks_per_iter = 8 for Q4_K. */
+        const int kqs = VDR_Q4_K_Q8_1_MMVQ * (int)(group_tid % 16u);
+        for (int kbx = (int)(group_tid / 16u);
+             kbx < k_q4_blocks;
+             kbx += 8) {
+            const int kby = kbx * (QK_K / QK8_1);
+#pragma unroll
+            for (int i = 0; i < k_rows_per_group; ++i) {
+                tmp[i] += vec_dot_q4_K_q8_1(
+                    W, &x8_shared[kby],
+                    (int)((uint64_t)(row0 + (uint32_t)i) * k_q4_blocks) + kbx,
+                    kqs);
+            }
+        }
+
+        if (warp_in_group > 0u) {
+#pragma unroll
+            for (int i = 0; i < k_rows_per_group; ++i) {
+                partial[group][warp_in_group - 1u][i][lane] = tmp[i];
+            }
+        }
+        __syncthreads();
+
+        if (warp_in_group == 0u) {
+#pragma unroll
+            for (int i = 0; i < k_rows_per_group; ++i) {
+#pragma unroll
+                for (int peer = 0; peer < 3; ++peer) {
+                    tmp[i] += partial[group][peer][i][lane];
+                }
+                tmp[i] = warp_reduce_sum<32>(tmp[i]);
+            }
+            if (lane < k_rows_per_group) {
+                out[row0 + lane] = tmp[lane];
+            }
+        }
+        /* Both groups must finish consuming partial before the next
+         * grid-stride tile reuses it. */
+        __syncthreads();
+    }
+}
+
 template <ggml_type type>
 int ds4_mmq_dense_vec_impl(
         const char  * tag,
@@ -3056,6 +4708,7 @@ int ds4_mmq_dense_vec_impl(
         int           M,
         int           N,
         int           K,
+        int           q4_weight_device_resident,
         cudaStream_t  stream) {
 
     if (!W || !X_f32 || !out_f32) {
@@ -3083,21 +4736,302 @@ int ds4_mmq_dense_vec_impl(
         return -1;
     }
 
-    // Route the pool's cudaMallocAsync through the caller-supplied stream
-    // for Step 8 / CUDA Graph compatibility.  See ds4_mmq_moe_vec_impl.
+    /* Resolve the exact-shape admission before allocating pool storage,
+     * quantizing X, clearing output, or launching any kernel.  REQUIRE and
+     * the oracle are coverage gates: an ineligible candidate must therefore
+     * fail without leaving work queued on the caller's stream. */
+    bool q4_k1024_exact = false;
+    bool q4_k1024_eligible = false;
+    bool q4_k1024_oracle = false;
+    unsigned q4_k1024_grid = 0u;
+    if constexpr (type == GGML_TYPE_Q4_K) {
+        q4_k1024_persistent_maybe_register_report();
+        q4_k1024_exact = M == 32768 && N == 1 && K == 1024;
+        q4_k1024_oracle = q4_k1024_exact &&
+            q4_k1024_env_flag("DS4_CUDA_Q4_K1024_PERSISTENT_ORACLE");
+        const bool enable =
+            q4_k1024_env_flag("DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT");
+        const bool disable =
+            q4_k1024_env_flag("DS4_CUDA_NO_Q4_K1024_PERSISTENT") ||
+            q4_k1024_env_flag("DS4_CUDA_NO_Q4_GB10_FAST");
+        if (q4_k1024_exact) {
+            g_q4_k1024_persistent_candidates++;
+        }
+        if (q4_k1024_exact && gb10_optimizations_enabled() &&
+            (enable || q4_k1024_oracle) && !disable &&
+            q4_weight_device_resident > 0 &&
+            (((uintptr_t)W & 15u) == 0u)) {
+            const uint64_t row_tiles = ((uint64_t)(uint32_t)M + 7u) / 8u;
+            const int nsm = ggml_cuda_info().devices[dev].nsm;
+            const uint64_t resident_blocks =
+                nsm > 0 ? (uint64_t)(uint32_t)nsm * 4u : 0u;
+            const uint64_t grid64 = row_tiles < resident_blocks
+                ? row_tiles : resident_blocks;
+            if (grid64 > 0u && grid64 <= UINT32_MAX) {
+                q4_k1024_grid = (unsigned)grid64;
+                q4_k1024_eligible = true;
+            }
+        }
+        if (q4_k1024_exact && !q4_k1024_eligible) {
+            g_q4_k1024_persistent_fallbacks++;
+            const bool require =
+                q4_k1024_env_flag(
+                    "DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT");
+            if (require || q4_k1024_oracle) {
+                g_q4_k1024_persistent_require_failures++;
+                if (q4_k1024_oracle) {
+                    g_q4_k1024_persistent_oracle_skips++;
+                }
+                fprintf(stderr,
+                        "%s: required Q4_K K1024 persistent path unavailable "
+                        "before enqueue\n",
+                        tag);
+                return -4;
+            }
+        }
+        if (q4_k1024_eligible && q4_k1024_oracle) {
+            cudaStreamCaptureStatus capture = cudaStreamCaptureStatusNone;
+            const cudaError_t capture_err =
+                cudaStreamIsCapturing(stream, &capture);
+            if (capture_err != cudaSuccess ||
+                capture != cudaStreamCaptureStatusNone) {
+                (void)cudaGetLastError();
+                g_q4_k1024_persistent_fallbacks++;
+                g_q4_k1024_persistent_require_failures++;
+                g_q4_k1024_persistent_oracle_skips++;
+                fprintf(stderr,
+                        "%s: Q4_K K1024 persistent oracle refuses CUDA "
+                        "graph capture before enqueue; run the oracle with "
+                        "DS4_CUDA_DECODE_GRAPHS=0\n",
+                        tag);
+                return -5;
+            }
+        }
+    }
+
+    // Route the pool's cudaMallocAsync through the caller-supplied stream
+    // for Step 8 / CUDA Graph compatibility.  See ds4_mmq_moe_vec_impl.
+    ds4_pool_set_stream(stream);
+
+    /* Oracle-only storage.  Graph capture was rejected above, so the pool
+     * allocations and the host readback below cannot become graph nodes.
+     * The persistent candidate writes here; canonical MMVQ always owns the
+     * caller-visible output. */
+    ggml_cuda_pool_alloc<float> q4_k1024_candidate;
+    ggml_cuda_pool_alloc<uint32_t> q4_k1024_mismatch;
+    if (q4_k1024_eligible && q4_k1024_oracle) {
+        q4_k1024_candidate.alloc(ctx->pool(), (size_t)M);
+        q4_k1024_mismatch.alloc(ctx->pool(), 1u);
+    }
+
+    // Dense: no MoE, ids=null. Layout [K, N, 1, 1] for src1.
+    const int64_t ne10_padded = GGML_PAD((int64_t)K, MATRIX_ROW_PADDING);
+    const size_t  nbytes_q8_1 = (size_t)N * ne10_padded *
+                                sizeof(block_q8_1) / QK8_1;
+    ggml_cuda_pool_alloc<char> src1_q8_1;
+    char *x8 = nullptr;
+    if constexpr (type == GGML_TYPE_Q4_K) {
+        if (gb10_optimizations_enabled() &&
+            getenv("DS4_CUDA_NO_Q4_GB10_FAST") == nullptr &&
+            getenv("DS4_CUDA_NO_Q4_DENSE_SCRATCH") == nullptr) {
+            x8 = (char *)ds4_mmq_aligned_q81_scratch(dev, nbytes_q8_1);
+        }
+    }
+    if (!x8) {
+        src1_q8_1.alloc(ctx->pool(), nbytes_q8_1);
+        x8 = src1_q8_1.get();
+    }
+
+    // Dense src1 layout: K innermost, N next; ne11=N, ne12=1, ne13=1.
+    quantize_row_q8_1_cuda(
+        X_f32, /*ids=*/nullptr, (void *)x8,
+        type, /*ne00=*/K,
+        /*s11=*/(int64_t)K, /*s12=*/(int64_t)K * N, /*s13=*/(int64_t)K * N,
+        /*ne0=*/ne10_padded, /*ne1=*/N, /*ne2=*/1, /*ne3=*/1,
+        stream);
+
+    cudaError_t err = cudaGetLastError();
+    if (err != cudaSuccess) {
+        fprintf(stderr, "%s: quantize_row_q8_1_cuda failed: %s\n",
+                tag, cudaGetErrorString(err));
+        return -2;
+    }
+
+    // Dense (no ids): per upstream dispatch (mmvq.cu:1121-1127),
+    //   ncols_dst          = ne1  = N
+    //   nchannels_y        = ne12 = 1
+    //   nchannels_dst      = ne2  = 1
+    //   stride_col_y       = s11  = ne10_padded / QK8_1
+    //   stride_channel_y   = s12  = N * (ne10_padded / QK8_1)
+    const int64_t blck      = ggml_blck_size(type);
+    const int64_t s01_row   = (int64_t)K / blck;
+    const int64_t s11_y     = ne10_padded / QK8_1;
+    const int64_t s12_y     = (int64_t)N * s11_y;
+    const int64_t s1_dst    = (int64_t)M;
+
+    ggml_cuda_mm_fusion_args_device fusion = {};
+
+    (void)cudaMemsetAsync(out_f32, 0, (size_t)M * (size_t)N * sizeof(float), stream);
+
+    bool q4_k1024_persistent = false;
+    if constexpr (type == GGML_TYPE_Q4_K) {
+        if (q4_k1024_eligible) {
+            float *candidate_out = q4_k1024_oracle
+                ? q4_k1024_candidate.get() : out_f32;
+            q4_K_dense_vec_k1024_persistent_kernel<<<
+                q4_k1024_grid, 256, 0, stream>>>(
+                    (const block_q4_K *)W,
+                    (const block_q8_1 *)x8,
+                    candidate_out, M);
+            g_q4_k1024_persistent_uses++;
+            q4_k1024_persistent = !q4_k1024_oracle;
+        }
+    }
+
+    if (!q4_k1024_persistent) {
+        mul_mat_vec_q_switch_type(
+            /*vx=*/W, /*type_x=*/type,
+            /*vy=*/(const void *)x8,
+            /*ids=*/nullptr, /*fusion=*/fusion,
+            /*dst=*/out_f32,
+            /*ncols_x=*/K, /*nrows_x=*/M, /*ncols_dst=*/N,
+            /*stride_row_x=*/(int)s01_row,
+            /*stride_col_y=*/(int)s11_y,
+            /*stride_col_dst=*/(int)s1_dst,
+            /*nchannels_x=*/1,
+            /*nchannels_y=*/1,
+            /*nchannels_dst=*/1,
+            /*stride_channel_x=*/0,
+            /*stride_channel_y=*/(int)s12_y,
+            /*stride_channel_dst=*/0,
+            /*nsamples_x=*/1, /*nsamples_dst=*/1,
+            /*stride_sample_x=*/0, /*stride_sample_y=*/0, /*stride_sample_dst=*/0,
+            /*ids_stride=*/0, stream);
+    }
+
+    err = cudaGetLastError();
+    if (err != cudaSuccess) {
+        fprintf(stderr, "%s: mul_mat_vec_q_switch_type (dense) launch failed: %s\n",
+                tag, cudaGetErrorString(err));
+        return -3;
+    }
+    const uint64_t out_count = (uint64_t)M * (uint64_t)N;
+    ds4_mmq_sanitize_f32(out_f32, out_count, stream);
+    if (q4_k1024_oracle) {
+        ds4_mmq_sanitize_f32(q4_k1024_candidate.get(), out_count, stream);
+        if (cudaGetLastError() != cudaSuccess) {
+            fprintf(stderr, "%s: Q4_K K1024 oracle sanitize failed\n", tag);
+            g_q4_k1024_persistent_oracle_skips++;
+            return -6;
+        }
+        cudaError_t oracle_err = cudaMemsetAsync(
+            q4_k1024_mismatch.get(), 0, sizeof(uint32_t), stream);
+        if (oracle_err == cudaSuccess) {
+            q4_K_k1024_bitwise_compare_kernel<<<
+                (unsigned)((out_count + 255u) / 256u), 256, 0, stream>>>(
+                    q4_k1024_mismatch.get(), q4_k1024_candidate.get(),
+                    out_f32, out_count);
+            oracle_err = cudaGetLastError();
+        }
+        uint32_t mismatch_host = 0u;
+        if (oracle_err == cudaSuccess) {
+            oracle_err = cudaMemcpyAsync(
+                &mismatch_host, q4_k1024_mismatch.get(), sizeof(uint32_t),
+                cudaMemcpyDeviceToHost, stream);
+        }
+        if (oracle_err == cudaSuccess) {
+            oracle_err = cudaStreamSynchronize(stream);
+        }
+        if (oracle_err != cudaSuccess) {
+            fprintf(stderr,
+                    "%s: Q4_K K1024 persistent oracle failed: %s\n",
+                    tag, cudaGetErrorString(oracle_err));
+            (void)cudaGetLastError();
+            g_q4_k1024_persistent_oracle_skips++;
+            return -6;
+        }
+        g_q4_k1024_persistent_oracle_calls++;
+        if (mismatch_host != 0u) {
+            g_q4_k1024_persistent_oracle_mismatches++;
+            if (!g_q4_k1024_persistent_oracle_mismatch_reported) {
+                g_q4_k1024_persistent_oracle_mismatch_reported = 1;
+                fprintf(stderr,
+                        "%s: Q4_K K1024 persistent oracle found a bitwise "
+                        "mismatch; retained canonical MMVQ output\n",
+                        tag);
+            }
+        }
+    }
+    return 0;
+}
+
+template <ggml_type type>
+int ds4_mmq_dense_pair_vec_impl(
+        const char  * tag,
+        const void  * W0,
+        const void  * W1,
+        const float * X_f32,
+        float       * out0_f32,
+        float       * out1_f32,
+        int           M0,
+        int           M1,
+        int           N,
+        int           K,
+        cudaStream_t  stream) {
+
+    if (!W0 || !W1 || !X_f32 || !out0_f32 || !out1_f32) {
+        fprintf(stderr, "%s: null pointer\n", tag);
+        return DS4_MMQ_NOT_APPLICABLE;
+    }
+    if (M0 <= 0 || M1 <= 0 || N <= 0 || K <= 0) {
+        fprintf(stderr, "%s: bad shape M0=%d M1=%d N=%d K=%d\n",
+                tag, M0, M1, N, K);
+        return DS4_MMQ_NOT_APPLICABLE;
+    }
+    if (K % 256 != 0) {
+        fprintf(stderr, "%s: K=%d must be a multiple of 256\n", tag, K);
+        return DS4_MMQ_NOT_APPLICABLE;
+    }
+    if (N > MMVQ_MAX_BATCH_SIZE) {
+        fprintf(stderr, "%s: N=%d exceeds MMVQ_MAX_BATCH_SIZE=%d\n",
+                tag, N, MMVQ_MAX_BATCH_SIZE);
+        return DS4_MMQ_NOT_APPLICABLE;
+    }
+
+    const int dev = ggml_cuda_get_device();
+    ggml_backend_cuda_context * ctx = get_ctx_for_device(dev);
+    if (!ctx) {
+        fprintf(stderr, "%s: failed to get cuda context for device %d\n",
+                tag, dev);
+        return DS4_MMQ_NOT_APPLICABLE;
+    }
+
     ds4_pool_set_stream(stream);
 
-    // Dense: no MoE, ids=null. Layout [K, N, 1, 1] for src1.
+    /* Match ds4_mmq_dense_vec_impl's activation layout and quantizer exactly,
+     * but retain the Q8_1 row for both projections. */
     const int64_t ne10_padded = GGML_PAD((int64_t)K, MATRIX_ROW_PADDING);
-    const size_t  nbytes_q8_1 = (size_t)N * ne10_padded *
-                                sizeof(block_q8_1) / QK8_1;
-    ggml_cuda_pool_alloc<char> src1_q8_1(ctx->pool(), nbytes_q8_1);
+    const size_t nbytes_q8_1 = (size_t)N * ne10_padded *
+                               sizeof(block_q8_1) / QK8_1;
+    ggml_cuda_pool_alloc<char> src1_q8_1;
+    char *x8 = nullptr;
+    if constexpr (type == GGML_TYPE_Q4_K) {
+        if (gb10_optimizations_enabled() &&
+            getenv("DS4_CUDA_NO_Q4_GB10_FAST") == nullptr &&
+            getenv("DS4_CUDA_NO_Q4_DENSE_SCRATCH") == nullptr) {
+            x8 = (char *)ds4_mmq_aligned_q81_scratch(dev, nbytes_q8_1);
+        }
+    }
+    if (!x8) {
+        src1_q8_1.alloc(ctx->pool(), nbytes_q8_1);
+        x8 = src1_q8_1.get();
+    }
 
-    // Dense src1 layout: K innermost, N next; ne11=N, ne12=1, ne13=1.
     quantize_row_q8_1_cuda(
-        X_f32, /*ids=*/nullptr, (void *)src1_q8_1.get(),
+        X_f32, /*ids=*/nullptr, (void *)x8,
         type, /*ne00=*/K,
-        /*s11=*/(int64_t)K, /*s12=*/(int64_t)K * N, /*s13=*/(int64_t)K * N,
+        /*s11=*/(int64_t)K, /*s12=*/(int64_t)K * N,
+        /*s13=*/(int64_t)K * N,
         /*ne0=*/ne10_padded, /*ne1=*/N, /*ne2=*/1, /*ne3=*/1,
         stream);
 
@@ -3108,48 +5042,207 @@ int ds4_mmq_dense_vec_impl(
         return -2;
     }
 
-    // Dense (no ids): per upstream dispatch (mmvq.cu:1121-1127),
-    //   ncols_dst          = ne1  = N
-    //   nchannels_y        = ne12 = 1
-    //   nchannels_dst      = ne2  = 1
-    //   stride_col_y       = s11  = ne10_padded / QK8_1
-    //   stride_channel_y   = s12  = N * (ne10_padded / QK8_1)
-    const int64_t blck      = ggml_blck_size(type);
-    const int64_t s01_row   = (int64_t)K / blck;
-    const int64_t s11_y     = ne10_padded / QK8_1;
-    const int64_t s12_y     = (int64_t)N * s11_y;
-    const int64_t s1_dst    = (int64_t)M;
-
+    const int64_t blck = ggml_blck_size(type);
+    const int64_t s01_row = (int64_t)K / blck;
+    const int64_t s11_y = ne10_padded / QK8_1;
+    const int64_t s12_y = (int64_t)N * s11_y;
     ggml_cuda_mm_fusion_args_device fusion = {};
 
-    (void)cudaMemsetAsync(out_f32, 0, (size_t)M * (size_t)N * sizeof(float), stream);
+    /* Keep each leg's memset, canonical MMVQ dispatch, error check, and
+     * sanitizer in the same order as two dense_vec calls.  Only the activation
+     * quantization/allocation above is shared. */
+    cudaMemsetAsync(out0_f32, 0,
+                    (size_t)M0 * (size_t)N * sizeof(float), stream);
+    mul_mat_vec_q_switch_type(
+        /*vx=*/W0, /*type_x=*/type,
+        /*vy=*/(const void *)x8,
+        /*ids=*/nullptr, /*fusion=*/fusion,
+        /*dst=*/out0_f32,
+        /*ncols_x=*/K, /*nrows_x=*/M0, /*ncols_dst=*/N,
+        /*stride_row_x=*/(int)s01_row,
+        /*stride_col_y=*/(int)s11_y,
+        /*stride_col_dst=*/M0,
+        /*nchannels_x=*/1, /*nchannels_y=*/1, /*nchannels_dst=*/1,
+        /*stride_channel_x=*/0,
+        /*stride_channel_y=*/(int)s12_y,
+        /*stride_channel_dst=*/0,
+        /*nsamples_x=*/1, /*nsamples_dst=*/1,
+        /*stride_sample_x=*/0, /*stride_sample_y=*/0,
+        /*stride_sample_dst=*/0,
+        /*ids_stride=*/0, stream);
+    err = cudaGetLastError();
+    if (err != cudaSuccess) {
+        fprintf(stderr, "%s: first dense MMVQ launch failed: %s\n",
+                tag, cudaGetErrorString(err));
+        return -3;
+    }
+    ds4_mmq_sanitize_f32(out0_f32, (uint64_t)M0 * (uint64_t)N, stream);
 
+    cudaMemsetAsync(out1_f32, 0,
+                    (size_t)M1 * (size_t)N * sizeof(float), stream);
     mul_mat_vec_q_switch_type(
-        /*vx=*/W, /*type_x=*/type,
-        /*vy=*/(const void *)src1_q8_1.get(),
+        /*vx=*/W1, /*type_x=*/type,
+        /*vy=*/(const void *)x8,
         /*ids=*/nullptr, /*fusion=*/fusion,
-        /*dst=*/out_f32,
-        /*ncols_x=*/K, /*nrows_x=*/M, /*ncols_dst=*/N,
+        /*dst=*/out1_f32,
+        /*ncols_x=*/K, /*nrows_x=*/M1, /*ncols_dst=*/N,
         /*stride_row_x=*/(int)s01_row,
         /*stride_col_y=*/(int)s11_y,
-        /*stride_col_dst=*/(int)s1_dst,
-        /*nchannels_x=*/1,
-        /*nchannels_y=*/1,
-        /*nchannels_dst=*/1,
+        /*stride_col_dst=*/M1,
+        /*nchannels_x=*/1, /*nchannels_y=*/1, /*nchannels_dst=*/1,
         /*stride_channel_x=*/0,
         /*stride_channel_y=*/(int)s12_y,
         /*stride_channel_dst=*/0,
         /*nsamples_x=*/1, /*nsamples_dst=*/1,
-        /*stride_sample_x=*/0, /*stride_sample_y=*/0, /*stride_sample_dst=*/0,
+        /*stride_sample_x=*/0, /*stride_sample_y=*/0,
+        /*stride_sample_dst=*/0,
         /*ids_stride=*/0, stream);
+    err = cudaGetLastError();
+    if (err != cudaSuccess) {
+        fprintf(stderr, "%s: second dense MMVQ launch failed: %s\n",
+                tag, cudaGetErrorString(err));
+        return -4;
+    }
+    ds4_mmq_sanitize_f32(out1_f32, (uint64_t)M1 * (uint64_t)N, stream);
+    return 0;
+}
+
+__global__ static void ds4_mmq_group_ids_i32_kernel(
+        int32_t *ids, int n, int n_groups) {
+    const int i = (int)(blockIdx.x * blockDim.x + threadIdx.x);
+    if (i < n) ids[i] = i % n_groups;
+}
+
+/* Grouped AProjQ4 attention-A projection.  Flatten (token, group) into the
+ * MMVQ channel dimension (never the column dimension): ncols_dst stays one,
+ * so every pair uses exactly the same one-row Q4_K MMVQ specialization, K
+ * partition, peer-warp fold, and reduction tree as the canonical nested
+ * token/group loop.  The repeated ids select W[group], while channel_y and
+ * channel_dst retain the token-major flat index.  Only activation
+ * quantization and launch setup are shared. */
+static int ds4_mmq_q4_K_grouped_batch_vec_impl(
+        const void  *W,
+        const float *X,
+        float       *out,
+        int          M,
+        int          K,
+        int          n_tokens,
+        int          n_groups,
+        cudaStream_t stream) {
+    const char *tag = n_tokens == 1
+        ? "ds4_mmq_q4_K_grouped_vec"
+        : "ds4_mmq_q4_K_grouped_batch_vec";
+    if (!W || !X || !out) {
+        fprintf(stderr, "%s: null pointer\n", tag);
+        return -1;
+    }
+    if (!gb10_optimizations_enabled() ||
+        getenv("DS4_CUDA_NO_Q4_GB10_FAST") != nullptr ||
+        getenv("DS4_CUDA_NO_Q4_GROUPED_ATTN_A") != nullptr ||
+        M <= 0 || K <= 0 || n_tokens <= 0 || n_tokens > 8 ||
+        n_groups <= 0 || n_groups > 16 ||
+        K % 256 != 0) {
+        return DS4_MMQ_NOT_APPLICABLE;
+    }
+    if (n_tokens > 1) {
+        const char *enable =
+            getenv("DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_BATCH");
+        if (!enable || !enable[0] || strcmp(enable, "0") == 0 ||
+            getenv("DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH") != nullptr) {
+            return DS4_MMQ_NOT_APPLICABLE;
+        }
+    }
+
+    const int flat_channels = n_tokens * n_groups; /* <= 8 * 16 */
+
+    const int64_t row_blocks = (int64_t)K / ggml_blck_size(GGML_TYPE_Q4_K);
+    const int64_t weight_channel_stride = (int64_t)M * row_blocks;
+    if (row_blocks <= 0 || row_blocks > INT_MAX ||
+        weight_channel_stride > INT_MAX) {
+        return DS4_MMQ_NOT_APPLICABLE;
+    }
+
+    const int64_t ne10_padded = GGML_PAD((int64_t)K, MATRIX_ROW_PADDING);
+    const size_t q8_row_bytes = (size_t)ne10_padded *
+                                sizeof(block_q8_1) / QK8_1;
+    if ((size_t)flat_channels > SIZE_MAX / q8_row_bytes) {
+        return DS4_MMQ_NOT_APPLICABLE;
+    }
+    const size_t nbytes_q8_1 = (size_t)flat_channels * q8_row_bytes;
+    if (nbytes_q8_1 > SIZE_MAX - 15u) return DS4_MMQ_NOT_APPLICABLE;
+    const size_t ids_offset = (nbytes_q8_1 + 15u) & ~(size_t)15u;
+    const size_t ids_bytes = (size_t)flat_channels * sizeof(int32_t);
+    if (ids_offset > SIZE_MAX - ids_bytes) {
+        return DS4_MMQ_NOT_APPLICABLE;
+    }
+
+    const int dev = ggml_cuda_get_device();
+    char *x8 = (char *)ds4_mmq_aligned_q81_scratch(
+        dev, ids_offset + ids_bytes);
+    if (!x8) return DS4_MMQ_NOT_APPLICABLE;
+    int32_t *ids = (int32_t *)(x8 + ids_offset);
+    ds4_mmq_group_ids_i32_kernel<<<
+        (unsigned)(flat_channels + 31) / 32u, 32, 0, stream>>>(
+            ids, flat_channels, n_groups);
+    cudaError_t err = cudaGetLastError();
+    if (err != cudaSuccess) {
+        fprintf(stderr, "%s: group-id launch failed: %s\n",
+                tag, cudaGetErrorString(err));
+        return -2;
+    }
 
+    quantize_row_q8_1_cuda(
+        X, /*ids=*/nullptr, (void *)x8,
+        GGML_TYPE_Q4_K, /*ne00=*/K,
+        /*s11=*/(int64_t)K,
+        /*s12=*/(int64_t)K * flat_channels,
+        /*s13=*/(int64_t)K * flat_channels,
+        /*ne0=*/ne10_padded, /*ne1=*/flat_channels, /*ne2=*/1, /*ne3=*/1,
+        stream);
     err = cudaGetLastError();
     if (err != cudaSuccess) {
-        fprintf(stderr, "%s: mul_mat_vec_q_switch_type (dense) launch failed: %s\n",
+        fprintf(stderr, "%s: quantize_row_q8_1_cuda failed: %s\n",
                 tag, cudaGetErrorString(err));
         return -3;
     }
-    ds4_mmq_sanitize_f32(out_f32, (uint64_t)M * (uint64_t)N, stream);
+
+    const int64_t y_channel_stride = ne10_padded / QK8_1;
+    ggml_cuda_mm_fusion_args_device fusion = {};
+    err = cudaMemsetAsync(
+        out, 0, (size_t)flat_channels * (size_t)M * sizeof(float), stream);
+    if (err != cudaSuccess) {
+        fprintf(stderr, "%s: output clear failed: %s\n",
+                tag, cudaGetErrorString(err));
+        return -4;
+    }
+    mul_mat_vec_q_switch_type(
+        /*vx=*/W, /*type_x=*/GGML_TYPE_Q4_K,
+        /*vy=*/(const void *)x8,
+        /*ids=*/ids, /*fusion=*/fusion,
+        /*dst=*/out,
+        /*ncols_x=*/K, /*nrows_x=*/M, /*ncols_dst=*/1,
+        /*stride_row_x=*/(int)row_blocks,
+        /*stride_col_y=*/(int)y_channel_stride,
+        /*stride_col_dst=*/M,
+        /*nchannels_x=*/n_groups,
+        /*nchannels_y=*/flat_channels,
+        /*nchannels_dst=*/flat_channels,
+        /*stride_channel_x=*/(int)weight_channel_stride,
+        /*stride_channel_y=*/(int)y_channel_stride,
+        /*stride_channel_dst=*/M,
+        /*nsamples_x=*/1, /*nsamples_dst=*/1,
+        /*stride_sample_x=*/0, /*stride_sample_y=*/0,
+        /*stride_sample_dst=*/0,
+        /*ids_stride=*/1, stream);
+    err = cudaGetLastError();
+    if (err != cudaSuccess) {
+        fprintf(stderr, "%s: grouped MMVQ launch failed: %s\n",
+                tag, cudaGetErrorString(err));
+        return -5;
+    }
+    ds4_mmq_sanitize_f32(
+        out, (uint64_t)(uint32_t)flat_channels * (uint64_t)(uint32_t)M,
+        stream);
     return 0;
 }
 
@@ -3679,7 +5772,9 @@ int ds4_mmq_moe_gate_up_mid_vec_impl(
     ggml_cuda_pool_alloc<char> src1_q8_1_pool;
     // M2-Inc2a: the fused HC stage may have emitted this activation's q8_1
     // codes already (ffn_norm) -- take them and skip the quantize prelude.
-    char *src1_q8_1_ptr = ds4_mmq_folded_q81(X_f32, K, n_tokens, ne10_padded);
+    char *src1_q8_1_ptr = ds4_mmq_folded_q81(
+        X_f32, K, n_tokens, ne10_padded, stream);
+    const bool folded_hit = src1_q8_1_ptr != nullptr;
     cudaError_t err;
     if (!src1_q8_1_ptr) {
     if (g_q81_scratch_enabled && g_q81_scratch_ptr && g_q81_scratch_bytes >= nbytes_q8_1) {
@@ -3711,6 +5806,106 @@ int ds4_mmq_moe_gate_up_mid_vec_impl(
 
     const dim3 block_nums((M + 63) / 64, n_tokens * n_expert_used);
     const dim3 block_dims(256);
+    if (folded_hit && ds4_mmq_q8_fold_oracle_enabled()) {
+        const size_t q8_bytes =
+            (size_t)ne10_padded * sizeof(block_q8_1) / QK8_1;
+        const uint64_t mid_count =
+            (uint64_t)M * (uint64_t)n_tokens * (uint64_t)n_expert_used;
+        const size_t mid_bytes = (size_t)mid_count * sizeof(float);
+        block_q8_1 *fresh = nullptr;
+        float *reference = nullptr;
+        uint32_t *mismatch_device = nullptr;
+        const bool allocated =
+            cudaMalloc((void **)&fresh, q8_bytes) == cudaSuccess &&
+            cudaMalloc((void **)&reference, mid_bytes) == cudaSuccess &&
+            cudaMalloc((void **)&mismatch_device, sizeof(uint32_t)) == cudaSuccess &&
+            fresh && reference && mismatch_device;
+        if (!allocated) {
+            (void)cudaGetLastError();
+            cudaError_t cleanup_err = cudaSuccess;
+            cleanup_err = ds4_mmq_q8_fold_oracle_free(
+                fresh, "raw-moe-fresh", cleanup_err);
+            cleanup_err = ds4_mmq_q8_fold_oracle_free(
+                reference, "raw-moe-reference", cleanup_err);
+            cleanup_err = ds4_mmq_q8_fold_oracle_free(
+                mismatch_device, "raw-moe-mismatch", cleanup_err);
+            if (cleanup_err != cudaSuccess) (void)cudaGetLastError();
+            g_q8_fold_oracle_skips++;
+        } else {
+            cudaError_t oracle_err = cudaMemsetAsync(
+                mismatch_device, 0, sizeof(uint32_t), stream);
+            if (oracle_err == cudaSuccess) {
+                quantize_row_q8_1_cuda(
+                    X_f32, /*ids=*/nullptr, fresh, type,
+                    /*ne00=*/K, /*s11=*/K, /*s12=*/K, /*s13=*/K,
+                    /*ne0=*/ne10_padded, /*ne1=*/1, /*ne2=*/1, /*ne3=*/1,
+                    stream);
+                oracle_err = cudaGetLastError();
+            }
+            if (oracle_err == cudaSuccess) {
+                ds4_mmq_moe_gate_up_mid_q8_1_qwarp32_kernel<type><<<
+                    block_nums, block_dims, 0, stream>>>(
+                        W_gate, W_up,
+                        (const block_q8_1 *)src1_q8_1_ptr,
+                        ids, weights, mid_f32,
+                        (uint32_t)K, (uint32_t)M,
+                        (uint32_t)n_tokens, (uint32_t)n_experts,
+                        stride_row_x, stride_col_y, stride_channel_x, clamp);
+                oracle_err = cudaGetLastError();
+            }
+            if (oracle_err == cudaSuccess) {
+                ds4_mmq_moe_gate_up_mid_q8_1_qwarp32_kernel<type><<<
+                    block_nums, block_dims, 0, stream>>>(
+                        W_gate, W_up, fresh, ids, weights, reference,
+                        (uint32_t)K, (uint32_t)M,
+                        (uint32_t)n_tokens, (uint32_t)n_experts,
+                        stride_row_x, stride_col_y, stride_channel_x, clamp);
+                oracle_err = cudaGetLastError();
+            }
+            if (oracle_err == cudaSuccess) {
+                q8_fold_output_compare_kernel<<<
+                    (unsigned)((mid_count + 255u) / 256u), 256, 0, stream>>>(
+                        mismatch_device, mid_f32, reference, mid_count);
+                oracle_err = cudaGetLastError();
+            }
+            if (oracle_err == cudaSuccess) {
+                oracle_err = cudaMemcpyAsync(
+                    mid_f32, reference, mid_bytes,
+                    cudaMemcpyDeviceToDevice, stream);
+            }
+            uint32_t mismatch_host = 0u;
+            if (oracle_err == cudaSuccess) {
+                oracle_err = cudaMemcpyAsync(
+                    &mismatch_host, mismatch_device, sizeof(mismatch_host),
+                    cudaMemcpyDeviceToHost, stream);
+            }
+            if (oracle_err == cudaSuccess) {
+                oracle_err = cudaStreamSynchronize(stream);
+            }
+            oracle_err = ds4_mmq_q8_fold_oracle_free(
+                fresh, "raw-moe-fresh", oracle_err);
+            oracle_err = ds4_mmq_q8_fold_oracle_free(
+                reference, "raw-moe-reference", oracle_err);
+            oracle_err = ds4_mmq_q8_fold_oracle_free(
+                mismatch_device, "raw-moe-mismatch", oracle_err);
+            if (oracle_err != cudaSuccess) {
+                (void)cudaGetLastError();
+                g_q8_fold_oracle_skips++;
+                fprintf(stderr, "%s: fold consumer oracle failed\n", tag);
+                return -3;
+            }
+            g_q8_fold_oracle_output_calls++;
+            g_q8_fold_oracle_raw_moe_calls++;
+            if (mismatch_host != 0u) {
+                g_q8_fold_oracle_output_mismatches++;
+                fprintf(stderr,
+                        "ds4: CUDA Q8_1 fold oracle found a raw MoE "
+                        "consumer output mismatch; retained canonical "
+                        "output\n");
+            }
+            return 0;
+        }
+    }
     ds4_mmq_moe_gate_up_mid_q8_1_qwarp32_kernel<type><<<block_nums, block_dims, 0, stream>>>(
         W_gate, W_up, (const block_q8_1 *)src1_q8_1_ptr, ids, weights, mid_f32,
         (uint32_t)K, (uint32_t)M, (uint32_t)n_tokens, (uint32_t)n_experts,
@@ -3799,7 +5994,7 @@ static int ds4_mmq_q4_K_dense_pair_vec_impl(
     const size_t qbytes =
         (size_t)padded * sizeof(block_q8_1) / QK8_1;
     ggml_cuda_pool_alloc<char> q8_pool;
-    char *x8 = ds4_mmq_folded_q81(X, K, 1, padded);
+    char *x8 = ds4_mmq_folded_q81(X, K, 1, padded, stream);
     if (!x8) {
         if (void *scratch = ds4_mmq_aligned_q81_scratch(dev, qbytes)) {
             x8 = (char *)scratch;
@@ -3836,6 +6031,27 @@ static int ds4_mmq_q4_K_dense_pair_vec_impl(
 
 } // anonymous namespace
 
+extern "C" void ds4_mmq_q4_K_k1024_persistent_counters(
+        uint64_t *candidates,
+        uint64_t *uses,
+        uint64_t *fallbacks,
+        uint64_t *require_failures,
+        uint64_t *oracle_calls,
+        uint64_t *oracle_mismatches,
+        uint64_t *oracle_skips) {
+    if (candidates) *candidates = g_q4_k1024_persistent_candidates;
+    if (uses) *uses = g_q4_k1024_persistent_uses;
+    if (fallbacks) *fallbacks = g_q4_k1024_persistent_fallbacks;
+    if (require_failures) {
+        *require_failures = g_q4_k1024_persistent_require_failures;
+    }
+    if (oracle_calls) *oracle_calls = g_q4_k1024_persistent_oracle_calls;
+    if (oracle_mismatches) {
+        *oracle_mismatches = g_q4_k1024_persistent_oracle_mismatches;
+    }
+    if (oracle_skips) *oracle_skips = g_q4_k1024_persistent_oracle_skips;
+}
+
 extern "C" int ds4_mmq_q8_0_moe_vec(
         const void * W, const float * X, const int32_t * ids, float * out,
         int M, int K, int n_tokens, int n_experts, int n_expert_used,
@@ -3932,6 +6148,20 @@ extern "C" int ds4_mmq_iq2_xxs_aligned_derepack(
     return 0;
 }
 
+extern "C" void ds4_mmq_set_gb10_optimizations(int enabled) {
+    {
+        // Serialize the transition with any persistent Q8_1 host lease.  The
+        // atomic also covers GB10 admission reads outside this arena lock.
+        std::lock_guard<std::mutex> lock(g_q81_state_mutex);
+        g_gb10_optimizations.store(enabled != 0, std::memory_order_relaxed);
+    }
+    if (!enabled) {
+        // Backend teardown/reinit already funnels through this setter.  Keep
+        // MMQ's owned arena lifecycle local to this translation unit.
+        (void)ds4_mmq_q81_persistent_cleanup();
+    }
+}
+
 // ---------------------------------------------------------------------------
 // Aligned-SoA Q8_0 dense decode matvec (megakernel program M1-Inc3).
 //
@@ -3984,6 +6214,96 @@ __global__ void q8_0_aligned_dense_vec_kernel(
     if (lane == 0) out[row] = acc;
 }
 
+/* K=1024 decode specialization. Eight persistent row warps per CTA hoist the
+ * 32 Q8_1 activation blocks into registers and walk output rows at a grid
+ * stride. Each lane still owns the same single block term and the warp tree is
+ * unchanged, so output bits match q8_0_aligned_dense_vec_kernel. */
+__global__ __launch_bounds__(256, 6) void q8_0_aligned_dense_vec_k1024_persistent_kernel(
+        float             *out,
+        const int4        *qs,
+        const __half      *dq,
+        const block_q8_1  *x8,
+        int                M)
+{
+    const int lane = threadIdx.x & 31;
+    const int warp = threadIdx.x >> 5;
+    const int *u = (const int *)x8[lane].qs;
+    const int u0 = u[0];
+    const int u1 = u[1];
+    const int u2 = u[2];
+    const int u3 = u[3];
+    const int u4 = u[4];
+    const int u5 = u[5];
+    const int u6 = u[6];
+    const int u7 = u[7];
+    const float dx = __low2float(x8[lane].ds);
+    const int64_t row0 = (int64_t)blockIdx.x * 8 + warp;
+    const int64_t row_stride = (int64_t)gridDim.x * 8;
+
+    for (int64_t row = row0; row < (int64_t)M; row += row_stride) {
+        const long long block = (long long)row * 32 + lane;
+        const int4 w0 = qs[block * 2 + 0];
+        const int4 w1 = qs[block * 2 + 1];
+        int s0 = ggml_cuda_dp4a(w0.x, u0, 0);
+        s0 = ggml_cuda_dp4a(w0.y, u1, s0);
+        int s1 = ggml_cuda_dp4a(w0.z, u2, 0);
+        s1 = ggml_cuda_dp4a(w0.w, u3, s1);
+        int s2 = ggml_cuda_dp4a(w1.x, u4, 0);
+        s2 = ggml_cuda_dp4a(w1.y, u5, s2);
+        int s3 = ggml_cuda_dp4a(w1.z, u6, 0);
+        s3 = ggml_cuda_dp4a(w1.w, u7, s3);
+        const int sumi = (s0 + s1) + (s2 + s3);
+        float acc = 0.0f;
+        acc += __half2float(dq[block]) * dx * (float)sumi;
+#pragma unroll
+        for (int off = 16; off > 0; off >>= 1)
+            acc += __shfl_down_sync(0xffffffffu, acc, off);
+        if (lane == 0) out[row] = acc;
+    }
+}
+
+/* Persistent-CTA form for the K=4096 vocabulary projection. It preserves
+ * the original lane/block assignment, per-lane term order, and warp tree;
+ * grouping eight row warps removes the one-warp CTA occupancy ceiling. */
+__global__ __launch_bounds__(256, 6) void q8_0_aligned_dense_vec_persistent_kernel(
+        float             *out,
+        const int4        *qs,
+        const __half      *dq,
+        const block_q8_1  *x8,
+        int                M,
+        int                nb)
+{
+    const int lane = threadIdx.x & 31;
+    const int warp = threadIdx.x >> 5;
+    const int64_t row0 = (int64_t)blockIdx.x * 8 + warp;
+    const int64_t row_stride = (int64_t)gridDim.x * 8;
+    for (int64_t row = row0; row < (int64_t)M; row += row_stride) {
+        const long long rbase = (long long)row * nb;
+        float acc = 0.0f;
+        for (int b0 = 0; b0 < nb; b0 += 32) {
+            const int b = b0 + lane;
+            const int4 w0 = qs[(rbase + b) * 2 + 0];
+            const int4 w1 = qs[(rbase + b) * 2 + 1];
+            const int *u = (const int *)x8[b].qs;
+            int sumi = 0;
+            sumi = ggml_cuda_dp4a(w0.x, u[0], sumi);
+            sumi = ggml_cuda_dp4a(w0.y, u[1], sumi);
+            sumi = ggml_cuda_dp4a(w0.z, u[2], sumi);
+            sumi = ggml_cuda_dp4a(w0.w, u[3], sumi);
+            sumi = ggml_cuda_dp4a(w1.x, u[4], sumi);
+            sumi = ggml_cuda_dp4a(w1.y, u[5], sumi);
+            sumi = ggml_cuda_dp4a(w1.z, u[6], sumi);
+            sumi = ggml_cuda_dp4a(w1.w, u[7], sumi);
+            acc += __half2float(dq[rbase + b]) *
+                   __low2float(x8[b].ds) * (float)sumi;
+        }
+#pragma unroll
+        for (int off = 16; off > 0; off >>= 1)
+            acc += __shfl_down_sync(0xffffffffu, acc, off);
+        if (lane == 0) out[row] = acc;
+    }
+}
+
 // Verify-width variant (v0.4 dense chase, proto_q8_aligned_nc): same aligned
 // weight stream read ONCE per row, NC output columns accumulated per lane
 // against col-strided q8_1 activations (which L1/L2-broadcast across rows).
@@ -4038,6 +6358,170 @@ __global__ void q8_0_aligned_dense_vec_nc_kernel(
     }
 }
 
+static int ds4_q8_aligned_warps_per_block(int cc);
+
+static cudaError_t q8_0_aligned_dense_vec_launch(
+        float *out, const int4 *qs, const __half *dq,
+        const block_q8_1 *x8, int M, int N, int K,
+        cudaStream_t stream) {
+    switch (N) {
+    case 1:
+        if (gb10_optimizations_enabled() &&
+            getenv("DS4_CUDA_NO_Q8_ALIGNED_PERSISTENT") == NULL &&
+            (K == 1024 || K == 4096) && M >= 32768) {
+            const uint64_t row_blocks = ((uint64_t)(unsigned)M + 7u) / 8u;
+            const unsigned persistent_blocks =
+                row_blocks < 288u ? (unsigned)row_blocks : 288u;
+            if (K == 1024) {
+                q8_0_aligned_dense_vec_k1024_persistent_kernel<<<
+                    persistent_blocks, 256, 0, stream>>>(
+                        out, qs, dq, x8, M);
+            } else {
+                q8_0_aligned_dense_vec_persistent_kernel<<<
+                    persistent_blocks, 256, 0, stream>>>(
+                        out, qs, dq, x8, M, K / 32);
+            }
+        } else {
+            switch (ds4_q8_aligned_warps_per_block(
+                        ggml_cuda_info().devices[ggml_cuda_get_device()].cc)) {
+            case 16:
+                q8_0_aligned_dense_vec_kernel<16>
+                    <<<((unsigned)M + 15u) / 16u, 512, 0, stream>>>(
+                        out, qs, dq, x8, M, K / 32);
+                break;
+            case 8:
+                q8_0_aligned_dense_vec_kernel<8>
+                    <<<((unsigned)M + 7u) / 8u, 256, 0, stream>>>(
+                        out, qs, dq, x8, M, K / 32);
+                break;
+            case 4:
+                q8_0_aligned_dense_vec_kernel<4>
+                    <<<((unsigned)M + 3u) / 4u, 128, 0, stream>>>(
+                        out, qs, dq, x8, M, K / 32);
+                break;
+            case 2:
+                q8_0_aligned_dense_vec_kernel<2>
+                    <<<((unsigned)M + 1u) / 2u, 64, 0, stream>>>(
+                        out, qs, dq, x8, M, K / 32);
+                break;
+            default:
+                q8_0_aligned_dense_vec_kernel<1>
+                    <<<(unsigned)M, 32, 0, stream>>>(
+                        out, qs, dq, x8, M, K / 32);
+                break;
+            }
+        }
+        break;
+    case 2: q8_0_aligned_dense_vec_nc_kernel<2><<<(unsigned)M, 32, 0, stream>>>(out, qs, dq, x8, M, K / 32); break;
+    case 3: q8_0_aligned_dense_vec_nc_kernel<3><<<(unsigned)M, 32, 0, stream>>>(out, qs, dq, x8, M, K / 32); break;
+    case 4: q8_0_aligned_dense_vec_nc_kernel<4><<<(unsigned)M, 32, 0, stream>>>(out, qs, dq, x8, M, K / 32); break;
+    case 5: q8_0_aligned_dense_vec_nc_kernel<5><<<(unsigned)M, 32, 0, stream>>>(out, qs, dq, x8, M, K / 32); break;
+    case 6: q8_0_aligned_dense_vec_nc_kernel<6><<<(unsigned)M, 32, 0, stream>>>(out, qs, dq, x8, M, K / 32); break;
+    case 7: q8_0_aligned_dense_vec_nc_kernel<7><<<(unsigned)M, 32, 0, stream>>>(out, qs, dq, x8, M, K / 32); break;
+    case 8: q8_0_aligned_dense_vec_nc_kernel<8><<<(unsigned)M, 32, 0, stream>>>(out, qs, dq, x8, M, K / 32); break;
+    default: return cudaErrorInvalidValue;
+    }
+    return cudaGetLastError();
+}
+
+/* Full consumer oracle for the folded single-column Q8_0 aligned entry.
+ * It regenerates canonical Q8_1, runs the exact same consumer twice, compares
+ * output bits, and always leaves the freshly quantized reference output in
+ * the caller buffer.  Return 1 when handled, 0 when diagnostics could not be
+ * set up before enqueue, and -1 after a CUDA failure. */
+static int q8_fold_q8_aligned_output_oracle(
+        const float *X_f32, const block_q8_1 *folded,
+        float *out, const int4 *qs, const __half *dq,
+        int M, int K, cudaStream_t stream) {
+    if (!ds4_mmq_q8_fold_oracle_enabled() || !folded || M <= 0 || K <= 0) {
+        return 0;
+    }
+    cudaStreamCaptureStatus capture = cudaStreamCaptureStatusNone;
+    if (cudaStreamIsCapturing(stream, &capture) != cudaSuccess ||
+        capture != cudaStreamCaptureStatusNone) {
+        (void)cudaGetLastError();
+        g_q8_fold_oracle_skips++;
+        return 0;
+    }
+    const size_t q8_bytes = (size_t)K * sizeof(block_q8_1) / QK8_1;
+    const size_t out_bytes = (size_t)M * sizeof(float);
+    block_q8_1 *fresh = nullptr;
+    float *reference = nullptr;
+    uint32_t *mismatch_device = nullptr;
+    if (cudaMalloc((void **)&fresh, q8_bytes) != cudaSuccess ||
+        cudaMalloc((void **)&reference, out_bytes) != cudaSuccess ||
+        cudaMalloc((void **)&mismatch_device, sizeof(uint32_t)) != cudaSuccess ||
+        !fresh || !reference || !mismatch_device) {
+        (void)cudaGetLastError();
+        cudaError_t cleanup_err = cudaSuccess;
+        cleanup_err = ds4_mmq_q8_fold_oracle_free(
+            fresh, "aligned-q8-fresh", cleanup_err);
+        cleanup_err = ds4_mmq_q8_fold_oracle_free(
+            reference, "aligned-q8-reference", cleanup_err);
+        cleanup_err = ds4_mmq_q8_fold_oracle_free(
+            mismatch_device, "aligned-q8-mismatch", cleanup_err);
+        if (cleanup_err != cudaSuccess) (void)cudaGetLastError();
+        g_q8_fold_oracle_skips++;
+        return 0;
+    }
+
+    cudaError_t err = cudaMemsetAsync(
+        mismatch_device, 0, sizeof(uint32_t), stream);
+    if (err == cudaSuccess) {
+        quantize_row_q8_1_cuda(
+            X_f32, /*ids=*/nullptr, fresh, GGML_TYPE_Q8_0,
+            /*ne00=*/K, /*s11=*/K, /*s12=*/K, /*s13=*/K,
+            /*ne0=*/K, /*ne1=*/1, /*ne2=*/1, /*ne3=*/1, stream);
+        err = cudaGetLastError();
+    }
+    if (err == cudaSuccess) {
+        err = q8_0_aligned_dense_vec_launch(
+            out, qs, dq, folded, M, 1, K, stream);
+    }
+    if (err == cudaSuccess) {
+        err = q8_0_aligned_dense_vec_launch(
+            reference, qs, dq, fresh, M, 1, K, stream);
+    }
+    if (err == cudaSuccess) {
+        q8_fold_output_compare_kernel<<<
+            (unsigned)(((uint64_t)M + 255u) / 256u), 256, 0, stream>>>(
+                mismatch_device, out, reference, (uint64_t)M);
+        err = cudaGetLastError();
+    }
+    if (err == cudaSuccess) {
+        err = cudaMemcpyAsync(out, reference, out_bytes,
+                              cudaMemcpyDeviceToDevice, stream);
+    }
+    uint32_t mismatch_host = 0u;
+    if (err == cudaSuccess) {
+        err = cudaMemcpyAsync(&mismatch_host, mismatch_device,
+                              sizeof(mismatch_host),
+                              cudaMemcpyDeviceToHost, stream);
+    }
+    if (err == cudaSuccess) err = cudaStreamSynchronize(stream);
+
+    err = ds4_mmq_q8_fold_oracle_free(
+        fresh, "aligned-q8-fresh", err);
+    err = ds4_mmq_q8_fold_oracle_free(
+        reference, "aligned-q8-reference", err);
+    err = ds4_mmq_q8_fold_oracle_free(
+        mismatch_device, "aligned-q8-mismatch", err);
+    if (err != cudaSuccess) {
+        (void)cudaGetLastError();
+        g_q8_fold_oracle_skips++;
+        return -1;
+    }
+    g_q8_fold_oracle_output_calls++;
+    g_q8_fold_oracle_aligned_q8_calls++;
+    if (mismatch_host != 0u) {
+        g_q8_fold_oracle_output_mismatches++;
+        fprintf(stderr,
+                "ds4: CUDA Q8_1 fold oracle found a Q8 aligned consumer "
+                "output mismatch; retained canonical output\n");
+    }
+    return 1;
+}
+
 extern "C" uint64_t ds4_mmq_q8_0_aligned_bytes(int M, int K) {
     if (M <= 0 || K <= 0 || K % 1024 != 0) return 0;
     const uint64_t nblk = (uint64_t)M * (uint64_t)(K / 32);
@@ -4122,7 +6606,7 @@ extern "C" int ds4_mmq_q8_0_aligned_dense_vec_pair(
     const size_t qbytes =
         (size_t)padded * sizeof(block_q8_1) / QK8_1;
     ggml_cuda_pool_alloc<char> q8_pool;
-    char *x8 = ds4_mmq_folded_q81(X_f32, K, 1, padded);
+    char *x8 = ds4_mmq_folded_q81(X_f32, K, 1, padded, stream);
     if (!x8) {
         if (g_q81_scratch_enabled && g_q81_scratch_ptr &&
             g_q81_scratch_bytes >= qbytes) {
@@ -4202,26 +6686,37 @@ extern "C" int ds4_mmq_q8_0_aligned_dense_vec(
     // M2-Inc2a: producer-emitted q8_1 codes (qr_norm from the qkv-rms
     // kernel) -- take them and skip the quantize prelude.  Single-column
     // producers only; verify widths always quantize.
-    char *x8 = N == 1 ? ds4_mmq_folded_q81(X_f32, K, 1, ne10_padded) : NULL;
+    char *x8 = N == 1
+        ? ds4_mmq_folded_q81(X_f32, K, 1, ne10_padded, stream)
+        : NULL;
+    const bool folded_hit = x8 != NULL;
     cudaError_t err;
     if (!x8) {
-    if (g_q81_scratch_enabled && g_q81_scratch_ptr && g_q81_scratch_bytes >= nbytes_q8_1) {
-        x8 = (char *)g_q81_scratch_ptr;
-    } else {
-        q8_pool.alloc(ctx->pool(), nbytes_q8_1);
-        x8 = q8_pool.get();
-    }
-    quantize_row_q8_1_cuda(
-        X_f32, /*ids=*/nullptr, (void *)x8,
-        GGML_TYPE_Q8_0, /*ne00=*/K,
-        /*s11=*/(int64_t)K, /*s12=*/(int64_t)K * N, /*s13=*/(int64_t)K * N,
-        /*ne0=*/ne10_padded, /*ne1=*/N, /*ne2=*/1, /*ne3=*/1,
-        stream);
-    err = cudaGetLastError();
-    if (err != cudaSuccess) {
-        fprintf(stderr, "%s: quantize_row_q8_1_cuda failed: %s\n", tag, cudaGetErrorString(err));
-        return -2;
-    }
+        if (getenv("DS4_CUDA_NO_Q8_ALIGNED_DENSE_SCRATCH") == NULL) {
+            x8 = (char *)ds4_mmq_aligned_q81_scratch(dev, nbytes_q8_1);
+        }
+        if (!x8 && g_q81_scratch_enabled && g_q81_scratch_ptr &&
+            g_q81_scratch_bytes >= nbytes_q8_1) {
+            x8 = (char *)g_q81_scratch_ptr;
+        }
+        if (!x8) {
+            q8_pool.alloc(ctx->pool(), nbytes_q8_1);
+            x8 = q8_pool.get();
+        }
+        quantize_row_q8_1_cuda(
+            X_f32, /*ids=*/nullptr, (void *)x8,
+            GGML_TYPE_Q8_0, /*ne00=*/K,
+            /*s11=*/(int64_t)K, /*s12=*/(int64_t)K * N,
+            /*s13=*/(int64_t)K * N,
+            /*ne0=*/ne10_padded, /*ne1=*/N, /*ne2=*/1, /*ne3=*/1,
+            stream);
+        err = cudaGetLastError();
+        if (err != cudaSuccess) {
+            fprintf(stderr,
+                    "%s: quantize_row_q8_1_cuda failed: %s\n",
+                    tag, cudaGetErrorString(err));
+            return -2;
+        }
     }
 
     const uint64_t nblk = (uint64_t)M * (uint64_t)(K / 32);
@@ -4229,46 +6724,17 @@ extern "C" int ds4_mmq_q8_0_aligned_dense_vec(
     const int4   *qsp = (const int4 *)((const char *)W_aligned + dq_bytes);
     const __half *dqp = (const __half *)W_aligned;
     const block_q8_1 *x8p = (const block_q8_1 *)x8;
-    switch (N) {
-    case 1:
-        switch (ds4_q8_aligned_warps_per_block(
-                    ggml_cuda_info().devices[dev].cc)) {
-        case 16:
-            q8_0_aligned_dense_vec_kernel<16>
-                <<<((unsigned)M + 15u) / 16u, 512, 0, stream>>>(
-                    out_f32, qsp, dqp, x8p, M, K / 32);
-            break;
-        case 8:
-            q8_0_aligned_dense_vec_kernel<8>
-                <<<((unsigned)M + 7u) / 8u, 256, 0, stream>>>(
-                    out_f32, qsp, dqp, x8p, M, K / 32);
-            break;
-        case 4:
-            q8_0_aligned_dense_vec_kernel<4>
-                <<<((unsigned)M + 3u) / 4u, 128, 0, stream>>>(
-                    out_f32, qsp, dqp, x8p, M, K / 32);
-            break;
-        case 2:
-            q8_0_aligned_dense_vec_kernel<2>
-                <<<((unsigned)M + 1u) / 2u, 64, 0, stream>>>(
-                    out_f32, qsp, dqp, x8p, M, K / 32);
-            break;
-        default:
-            q8_0_aligned_dense_vec_kernel<1>
-                <<<(unsigned)M, 32, 0, stream>>>(
-                    out_f32, qsp, dqp, x8p, M, K / 32);
-            break;
+    if (folded_hit && N == 1 && ds4_mmq_q8_fold_oracle_enabled()) {
+        const int oracle_rc = q8_fold_q8_aligned_output_oracle(
+            X_f32, x8p, out_f32, qsp, dqp, M, K, stream);
+        if (oracle_rc > 0) return 0;
+        if (oracle_rc < 0) {
+            fprintf(stderr, "%s: fold consumer oracle failed\n", tag);
+            return -3;
         }
-        break;
-    case 2: q8_0_aligned_dense_vec_nc_kernel<2><<<(unsigned)M, 32, 0, stream>>>(out_f32, qsp, dqp, x8p, M, K / 32); break;
-    case 3: q8_0_aligned_dense_vec_nc_kernel<3><<<(unsigned)M, 32, 0, stream>>>(out_f32, qsp, dqp, x8p, M, K / 32); break;
-    case 4: q8_0_aligned_dense_vec_nc_kernel<4><<<(unsigned)M, 32, 0, stream>>>(out_f32, qsp, dqp, x8p, M, K / 32); break;
-    case 5: q8_0_aligned_dense_vec_nc_kernel<5><<<(unsigned)M, 32, 0, stream>>>(out_f32, qsp, dqp, x8p, M, K / 32); break;
-    case 6: q8_0_aligned_dense_vec_nc_kernel<6><<<(unsigned)M, 32, 0, stream>>>(out_f32, qsp, dqp, x8p, M, K / 32); break;
-    case 7: q8_0_aligned_dense_vec_nc_kernel<7><<<(unsigned)M, 32, 0, stream>>>(out_f32, qsp, dqp, x8p, M, K / 32); break;
-    case 8: q8_0_aligned_dense_vec_nc_kernel<8><<<(unsigned)M, 32, 0, stream>>>(out_f32, qsp, dqp, x8p, M, K / 32); break;
     }
-    err = cudaGetLastError();
+    err = q8_0_aligned_dense_vec_launch(
+        out_f32, qsp, dqp, x8p, M, N, K, stream);
     if (err != cudaSuccess) {
         fprintf(stderr, "%s: kernel launch failed: %s\n", tag, cudaGetErrorString(err));
         return -3;
@@ -4579,7 +7045,9 @@ extern "C" uint64_t ds4_mmq_iq2_xxs_aligned_bytes(int M, int K, int n_experts) {
 // pool otherwise) or nullptr on failure; *pool must outlive the launches.
 static char *iq2_aligned_quantize_xn(
         const char *tag, const float *X_f32, int K, int n_tokens,
-        ggml_cuda_pool_alloc<char> *pool, cudaStream_t stream) {
+        ggml_cuda_pool_alloc<char> *pool, cudaStream_t stream,
+        bool *was_folded) {
+    if (was_folded) *was_folded = false;
     const int dev = ggml_cuda_get_device();
     ggml_backend_cuda_context * ctx = get_ctx_for_device(dev);
     if (!ctx) {
@@ -4591,8 +7059,10 @@ static char *iq2_aligned_quantize_xn(
     const size_t  nbytes_q8_1 = (size_t)n_tokens * ne10_padded * sizeof(block_q8_1) / QK8_1;
     // M2-Inc2a: producer-emitted q8_1 codes (ffn_norm from the fused HC
     // stage) -- take them and skip the quantize prelude.
-    char *folded = ds4_mmq_folded_q81(X_f32, K, n_tokens, ne10_padded);
+    char *folded = ds4_mmq_folded_q81(
+        X_f32, K, n_tokens, ne10_padded, stream);
     if (folded) {
+        if (was_folded) *was_folded = true;
         // C3-Inc4 fold twin selftest (DS4_Q8_FOLD_SELFTEST=<call budget>,
         // eager legs only -- syncs the stream): the taken sidecar must be
         // byte-identical to the fresh quantize this prelude would have run.
@@ -4670,7 +7140,8 @@ extern "C" int ds4_mmq_iq2_xxs_aligned_moe_pair_vec(
         return -1;
     }
     ggml_cuda_pool_alloc<char> q8_pool;
-    char *x8 = iq2_aligned_quantize_xn(tag, X_f32, K, n_tokens, &q8_pool, stream);
+    char *x8 = iq2_aligned_quantize_xn(
+        tag, X_f32, K, n_tokens, &q8_pool, stream, nullptr);
     if (!x8) return -2;
 
     const uint64_t nblk = (uint64_t)n_experts * (uint64_t)M * (uint64_t)(K / 256);
@@ -4707,7 +7178,9 @@ extern "C" int ds4_mmq_iq2_xxs_aligned_moe_gate_up_mid_vec(
         return -1;
     }
     ggml_cuda_pool_alloc<char> q8_pool;
-    char *x8 = iq2_aligned_quantize_xn(tag, X_f32, K, n_tokens, &q8_pool, stream);
+    bool folded_hit = false;
+    char *x8 = iq2_aligned_quantize_xn(
+        tag, X_f32, K, n_tokens, &q8_pool, stream, &folded_hit);
     if (!x8) return -2;
 
     const uint64_t nblk = (uint64_t)n_experts * (uint64_t)M * (uint64_t)(K / 256);
@@ -4717,6 +7190,99 @@ extern "C" int ds4_mmq_iq2_xxs_aligned_moe_gate_up_mid_vec(
     const __half *dq_g = (const __half *)W_gate_aligned;
     const uint2  *qs_u = (const uint2 *)((const char *)W_up_aligned + dq_bytes);
     const __half *dq_u = (const __half *)W_up_aligned;
+    if (folded_hit && n_tokens == 1 &&
+        ds4_mmq_q8_fold_oracle_enabled()) {
+        const size_t q8_bytes = (size_t)K * sizeof(block_q8_1) / QK8_1;
+        const uint64_t mid_count =
+            (uint64_t)M * (uint64_t)n_expert_used;
+        const size_t mid_bytes = (size_t)mid_count * sizeof(float);
+        block_q8_1 *fresh = nullptr;
+        float *reference = nullptr;
+        uint32_t *mismatch_device = nullptr;
+        const bool allocated =
+            cudaMalloc((void **)&fresh, q8_bytes) == cudaSuccess &&
+            cudaMalloc((void **)&reference, mid_bytes) == cudaSuccess &&
+            cudaMalloc((void **)&mismatch_device, sizeof(uint32_t)) == cudaSuccess &&
+            fresh && reference && mismatch_device;
+        if (!allocated) {
+            (void)cudaGetLastError();
+            cudaError_t cleanup_err = cudaSuccess;
+            cleanup_err = ds4_mmq_q8_fold_oracle_free(
+                fresh, "aligned-iq2-fresh", cleanup_err);
+            cleanup_err = ds4_mmq_q8_fold_oracle_free(
+                reference, "aligned-iq2-reference", cleanup_err);
+            cleanup_err = ds4_mmq_q8_fold_oracle_free(
+                mismatch_device, "aligned-iq2-mismatch", cleanup_err);
+            if (cleanup_err != cudaSuccess) (void)cudaGetLastError();
+            g_q8_fold_oracle_skips++;
+        } else {
+            cudaError_t oracle_err = cudaMemsetAsync(
+                mismatch_device, 0, sizeof(uint32_t), stream);
+            if (oracle_err == cudaSuccess) {
+                quantize_row_q8_1_cuda(
+                    X_f32, /*ids=*/nullptr, fresh, GGML_TYPE_IQ2_XXS,
+                    /*ne00=*/K, /*s11=*/K, /*s12=*/K, /*s13=*/K,
+                    /*ne0=*/K, /*ne1=*/1, /*ne2=*/1, /*ne3=*/1, stream);
+                oracle_err = cudaGetLastError();
+            }
+            if (oracle_err == cudaSuccess) {
+                iq2_xxs_aligned_moe_gate_up_mid_kernel<<<grid, 32, 0, stream>>>(
+                    mid_f32, qs_g, dq_g, qs_u, dq_u,
+                    (const block_q8_1 *)x8, ids, weights,
+                    M, K / 256, K / 32, n_expert_used, clamp);
+                oracle_err = cudaGetLastError();
+            }
+            if (oracle_err == cudaSuccess) {
+                iq2_xxs_aligned_moe_gate_up_mid_kernel<<<grid, 32, 0, stream>>>(
+                    reference, qs_g, dq_g, qs_u, dq_u, fresh,
+                    ids, weights, M, K / 256, K / 32,
+                    n_expert_used, clamp);
+                oracle_err = cudaGetLastError();
+            }
+            if (oracle_err == cudaSuccess) {
+                q8_fold_output_compare_kernel<<<
+                    (unsigned)((mid_count + 255u) / 256u), 256, 0, stream>>>(
+                        mismatch_device, mid_f32, reference, mid_count);
+                oracle_err = cudaGetLastError();
+            }
+            if (oracle_err == cudaSuccess) {
+                oracle_err = cudaMemcpyAsync(
+                    mid_f32, reference, mid_bytes,
+                    cudaMemcpyDeviceToDevice, stream);
+            }
+            uint32_t mismatch_host = 0u;
+            if (oracle_err == cudaSuccess) {
+                oracle_err = cudaMemcpyAsync(
+                    &mismatch_host, mismatch_device, sizeof(mismatch_host),
+                    cudaMemcpyDeviceToHost, stream);
+            }
+            if (oracle_err == cudaSuccess) {
+                oracle_err = cudaStreamSynchronize(stream);
+            }
+            oracle_err = ds4_mmq_q8_fold_oracle_free(
+                fresh, "aligned-iq2-fresh", oracle_err);
+            oracle_err = ds4_mmq_q8_fold_oracle_free(
+                reference, "aligned-iq2-reference", oracle_err);
+            oracle_err = ds4_mmq_q8_fold_oracle_free(
+                mismatch_device, "aligned-iq2-mismatch", oracle_err);
+            if (oracle_err != cudaSuccess) {
+                (void)cudaGetLastError();
+                g_q8_fold_oracle_skips++;
+                fprintf(stderr, "%s: fold consumer oracle failed\n", tag);
+                return -3;
+            }
+            g_q8_fold_oracle_output_calls++;
+            g_q8_fold_oracle_aligned_iq2_calls++;
+            if (mismatch_host != 0u) {
+                g_q8_fold_oracle_output_mismatches++;
+                fprintf(stderr,
+                        "ds4: CUDA Q8_1 fold oracle found an IQ2 MoE "
+                        "consumer output mismatch; retained canonical "
+                        "output\n");
+            }
+            return 0;
+        }
+    }
     /* v0.4 V6: verify widths dedup expert overlap (see the dedup kernel's
      * header comment).  n_tokens==1 has no cross-token overlap and keeps
      * the per-slot kernel; widths beyond the verify envelope likewise.
@@ -4925,14 +7491,80 @@ extern "C" int ds4_mmq_q8_0_dense_vec(
         const void * W, const float * X, float * out,
         int M, int N, int K, cudaStream_t stream) {
     return ds4_mmq_dense_vec_impl<GGML_TYPE_Q8_0>(
-        "ds4_mmq_q8_0_dense_vec", W, X, out, M, N, K, stream);
+        "ds4_mmq_q8_0_dense_vec", W, X, out, M, N, K,
+        /*q4_weight_device_resident=*/0, stream);
+}
+
+static int ds4_mmq_pointer_is_device_resident(
+        const void *ptr, int expected_device) {
+    if (!ptr || expected_device < 0) return 0;
+    cudaPointerAttributes attr = {};
+    const cudaError_t err = cudaPointerGetAttributes(&attr, ptr);
+    if (err != cudaSuccess) {
+        (void)cudaGetLastError();
+        return 0;
+    }
+#if defined(GGML_USE_HIP) && HIP_VERSION >= 60000000
+    return attr.type == cudaMemoryTypeDevice &&
+           attr.device == expected_device;
+#elif defined(GGML_USE_HIP)
+    return attr.memoryType == cudaMemoryTypeDevice &&
+           attr.device == expected_device;
+#elif CUDART_VERSION >= 10000
+    return attr.type == cudaMemoryTypeDevice &&
+           attr.device == expected_device;
+#else
+    return attr.memoryType == cudaMemoryTypeDevice &&
+           attr.device == expected_device;
+#endif
+}
+
+extern "C" int ds4_mmq_q4_K_dense_vec(
+        const void * W, const float * X, float * out,
+        int M, int N, int K, cudaStream_t stream) {
+    int weight_device_resident = 0;
+    /* Keep pointer introspection out of generic Q4 MMVQ traffic. It is only
+     * needed when the exact-shape persistent candidate could be considered;
+     * the full runtime uses the explicit provenance API below instead. */
+    if (M == 32768 && N == 1 && K == 1024) {
+        weight_device_resident = ds4_mmq_pointer_is_device_resident(
+            W, ggml_cuda_get_device());
+    }
+    return ds4_mmq_dense_vec_impl<GGML_TYPE_Q4_K>(
+        "ds4_mmq_q4_K_dense_vec", W, X, out, M, N, K,
+        weight_device_resident, stream);
+}
+
+extern "C" int ds4_mmq_q4_K_dense_vec_with_weight_residency(
+        const void * W, const float * X, float * out,
+        int M, int N, int K, int weight_device_resident,
+        cudaStream_t stream) {
+    return ds4_mmq_dense_vec_impl<GGML_TYPE_Q4_K>(
+        "ds4_mmq_q4_K_dense_vec_with_weight_residency",
+        W, X, out, M, N, K, weight_device_resident > 0, stream);
+}
+
+extern "C" int ds4_mmq_q4_K_grouped_vec(
+        const void *W, const float *X, float *out,
+        int M, int K, int n_groups, cudaStream_t stream) {
+    return ds4_mmq_q4_K_grouped_batch_vec_impl(
+        W, X, out, M, K, 1, n_groups, stream);
+}
+
+extern "C" int ds4_mmq_q4_K_grouped_batch_vec(
+        const void *W, const float *X, float *out,
+        int M, int K, int n_tokens, int n_groups, cudaStream_t stream) {
+    return ds4_mmq_q4_K_grouped_batch_vec_impl(
+        W, X, out, M, K, n_tokens, n_groups, stream);
 }
 
 extern "C" int ds4_mmq_q4_K_dense_pair_vec(
-        const void *W0, const void *W1, const float *X,
-        float *out0, float *out1, int M, int K, cudaStream_t stream) {
-    return ds4_mmq_q4_K_dense_pair_vec_impl(
-        W0, W1, X, out0, out1, M, K, stream);
+        const void * W0, const void * W1, const float * X,
+        float * out0, float * out1,
+        int M0, int M1, int N, int K, cudaStream_t stream) {
+    return ds4_mmq_dense_pair_vec_impl<GGML_TYPE_Q4_K>(
+        "ds4_mmq_q4_K_dense_pair_vec", W0, W1, X, out0, out1,
+        M0, M1, N, K, stream);
 }
 
 // Explicit instantiations. One per quant type the public API exposes.
diff --git a/cuda/mmq/ds4_mmq.h b/cuda/mmq/ds4_mmq.h
index a77a1c66cb..425ce62ef7 100644
--- a/cuda/mmq/ds4_mmq.h
+++ b/cuda/mmq/ds4_mmq.h
@@ -34,6 +34,35 @@ extern "C" {
 int ds4_mmq_init(int device);
 void ds4_mmq_set_aligned_q81_scratch(void *ptr, size_t bytes);
 
+// Producer-fold registry bridge implemented by the full CUDA runtime.
+// A hit returns the canonical Q8_1 sidecar for this exact activation pointer
+// and stream. It is valid only for single-token unpadded rows; registry slots
+// are refreshed by each producer layer and consumed once. The standalone MMQ
+// library provides a weak, fail-closed miss so its tests do not need to link
+// ds4_cuda.cu; full ds4 overrides it with the stream-aware implementation.
+int ds4_cuda_q8_fold_take_q81(
+    const void   *src,
+    uint64_t      in_dim,
+    cudaStream_t  stream,
+    const void  **q81);
+
+// Opt-in grouped-MMQ Q8_1 arena controlled by
+// DS4_CUDA_MMQ_Q81_PERSISTENT. Unset and =0 keep the stream-pool path.
+// cleanup drains the owner device before freeing and is safe across reinit;
+// report/counters expose host-dispatch coverage (graph replays excluded).
+int  ds4_mmq_q81_persistent_cleanup(void);
+int  ds4_mmq_q81_persistent_preflight_for_test(int device, size_t required);
+void ds4_mmq_q81_persistent_report(void);
+void ds4_mmq_q81_persistent_counters(
+    uint64_t *candidates,
+    uint64_t *uses,
+    uint64_t *hits,
+    uint64_t *pool_fallbacks,
+    uint64_t *allocations,
+    uint64_t *resizes,
+    size_t   *arena_bytes,
+    size_t   *high_water);
+
 // Query whether ds4_mmq is willing to handle a given matmul. Returns
 //   1 if mmq is faster than dequant+cublas for this shape on this device,
 //   0 otherwise (caller should fall back to its existing dequant+cublas path).
@@ -159,6 +188,122 @@ int ds4_mmq_q4_K_dense(
     int           K,
     cudaStream_t  stream);
 
+#if !defined(GGML_USE_HIP)
+// CUDA benchmark/test boundary for separating activation quantization from
+// the Q4_K MMQ kernel.  The scratch layout is canonical block_q8_1_mmq DS4
+// ([K/128][N]) plus a zeroed 128-column tail.  These helpers never transfer
+// model weights and never synchronize the stream.
+size_t ds4_mmq_q4_K_q8_1_scratch_bytes(int N, int K);
+
+int ds4_mmq_q4_K_quantize_q8_1_for_test(
+    const float * X_f32,
+    void        * q8_ds4,
+    size_t        q8_bytes,
+    int           N,
+    int           K,
+    cudaStream_t  stream);
+
+// Return non-zero only when the current device's canonical Q4_K MMQ picker
+// selects m128n128 for this activation-column count. Kernel A/B harnesses use
+// this to reject tail geometries whose Stream-K tile partition is different.
+int ds4_mmq_q4_K_dense_preq_reference_m128n128_for_test(int N);
+
+// Enqueue-only A/B arms over a caller-owned, already-quantized activation.
+// The reference may disable stream-K for a complete-K control; the 16-warp
+// candidate helper follows the production canonical stream-K/fixup policy.
+// Passing caller-owned reference fixup storage keeps pool alloc/free nodes out
+// of CUDA-event kernel benchmarks; null/zero retains the production pool.
+// Zero is success; nonzero is rejection/failure.
+int ds4_mmq_q4_K_dense_preq_reference_for_test(
+    const void  * W_q4_K,
+    const void  * q8_ds4,
+    size_t        q8_bytes,
+    float       * out_f32,
+    int           M,
+    int           N,
+    int           K,
+    int           use_stream_k,
+    void        * stream_k_fixup,
+    size_t        stream_k_fixup_bytes,
+    cudaStream_t  stream);
+
+int ds4_mmq_q4_K_dense_preq_16warp_for_test(
+    const void  * W_q4_K,
+    const void  * q8_ds4,
+    size_t        q8_bytes,
+    float       * out_f32,
+    int           M,
+    int           N,
+    int           K,
+    cudaStream_t  stream);
+
+// Byte-parity boundary for the fixed [N][8][4096] grouped attention-A Q8_1
+// producer.  use_specialized=0 launches the canonical strided quantizer;
+// nonzero launches the K4096/G8x2 candidate.  Both write the same canonical
+// group-major block_q8_1_mmq DS4 payload and never synchronize the stream.
+size_t ds4_mmq_q4_K_grouped_q8_1_scratch_bytes_for_test(int N);
+
+int ds4_mmq_q4_K_grouped_quantize_q8_1_for_test(
+    const float * X_f32,
+    void        * q8_ds4,
+    size_t        q8_bytes,
+    int           N,
+    int           use_specialized,
+    cudaStream_t  stream);
+#endif
+
+// Two dense Q4_K MMQ projections that share one token-tiled Q8_1
+// activation buffer.  This is the prefill sibling of
+// ds4_mmq_q4_K_dense_pair_vec: N is not limited to the MMVQ batch ceiling,
+// M0 and M1 may differ, and each leg preserves ds4_mmq_q4_K_dense's
+// reduction and output layout. The two output ranges must be disjoint.
+// On CUDA, the opt-in 16-warp experiment selects each leg independently
+// (down to M=512) when its output-tile grid retains at least 80% SM-wave
+// efficiency; below canonical's whole-tile cutoff it mirrors canonical
+// stream-K partitioning and fixup. REQUIRE rejects the pair before allocation
+// unless both legs can use the candidate.
+// Shape/capability rejection returns DS4_MMQ_NOT_APPLICABLE before enqueue;
+// negative values report an attempted-path launch failure.
+int ds4_mmq_q4_K_dense_pair(
+    const void  * W0_q4_K,
+    const void  * W1_q4_K,
+    const float * X_f32,
+    float       * out0_f32,
+    float       * out1_f32,
+    int           M0,
+    int           M1,
+    int           N,
+    int           K,
+    cudaStream_t  stream);
+
+// Prefill attention output-A with W=[groups][M][K],
+// X=[N][groups][K], and out=[N][groups][M].  It quantizes the strided source
+// in one launch and writes each group directly to the final token-major
+// layout, while preserving the established per-group MMQ reduction tree.
+int ds4_mmq_q4_K_grouped_dense(
+    const void  * W_q4_K,
+    const float * X_f32,
+    float       * out_f32,
+    int           M,
+    int           N,
+    int           K,
+    int           n_groups,
+    cudaStream_t  stream);
+
+// Opt-in sibling of ds4_mmq_q4_K_grouped_dense that submits all groups in one
+// grid, with group selected by grid.z.  Stream-k coordinates and fixup storage
+// are isolated per grid.z slice, preserving the reduction tree and output bits
+// of the established one-launch-per-group implementation.
+int ds4_mmq_q4_K_grouped_dense_single_grid(
+    const void  * W_q4_K,
+    const float * X_f32,
+    float       * out_f32,
+    int           M,
+    int           N,
+    int           K,
+    int           n_groups,
+    cudaStream_t  stream);
+
 int ds4_mmq_mxfp4_dense(
     const void  * W_mxfp4,
     const float * X_f32,
@@ -344,6 +489,42 @@ int ds4_mmq_iq2_xxs_q2_K_moe_fused_soa(
     float           clamp,
     cudaStream_t    stream);
 
+/* Optional fused entries return this before enqueueing work when their
+ * capability/shape/scratch preflight cannot engage.  Callers may safely retry
+ * a materialized fallback only for this result; zero is success and negative
+ * values may follow partial enqueue. */
+#define DS4_MMQ_NOT_APPLICABLE 1
+
+/* Raw-layout twin of ds4_mmq_iq2_xxs_q2_K_moe_fused_soa.  This is the
+ * grouped SSD entry: callers may pass a compact expert table together with
+ * ids remapped into [0, n_experts).  Gate/up/down use canonical GGUF block
+ * layouts while the routing map, activation quantize, and expert bounds are
+ * built only once for the complete fused pipeline.  As with router top-k,
+ * ids for one token must be unique.
+ *
+ * Capability/shape failures return DS4_MMQ_NOT_APPLICABLE before enqueue.
+ * Once work has been submitted, failures are negative and must not be
+ * retried on the same stream as a materialized fallback. */
+int ds4_mmq_iq2_xxs_q2_K_moe_fused_raw(
+    const void    * W_gate_raw,
+    const void    * W_up_raw,
+    const void    * W_down_raw,
+    const float   * X_f32,
+    const int32_t * ids,
+    const float   * router_weights,
+    float         * gate_f32,
+    float         * up_f32,
+    float         * mid_f32,
+    float         * down_f32,
+    int             expert_mid_dim,
+    int             expert_in_dim,
+    int             out_dim,
+    int             n_tokens,
+    int             n_experts,
+    int             n_expert_used,
+    float           clamp,
+    cudaStream_t    stream);
+
 /* Aligned-artifact production fast path: gate/up stay in registers, weighted
  * SwiGLU is quantized directly into down_q8_scratch, and only the pair-major
  * down output is materialized.  Caller-owned scratch keeps this hot path free
@@ -578,6 +759,11 @@ int ds4_mmq_q2_K_aligned_derepack(
 // return non-zero so the caller can fall back to ds4_mmq_q8_0_dense_vec.
 uint64_t ds4_mmq_q8_0_aligned_bytes(int M, int K);
 
+// Enable decode shapes validated on integrated sm_121 (GB10). The CUDA
+// backend sets this after device discovery; other devices retain the generic
+// aligned kernel. DS4_CUDA_NO_Q8_ALIGNED_PERSISTENT is the runtime rollback.
+void ds4_mmq_set_gb10_optimizations(int enabled);
+
 int ds4_mmq_q8_0_aligned_dense_vec(
     const void  * W_aligned,
     const float * X_f32,
@@ -872,13 +1058,110 @@ int ds4_mmq_q8_0_dense_vec(
     int           K,
     cudaStream_t  stream);
 
+int ds4_mmq_q4_K_dense_vec(
+    const void  * W_q4_K,
+    const float * X_f32,
+    float       * out_f32,
+    int           M,
+    int           N,
+    int           K,
+    cudaStream_t  stream);
+
+// Full-runtime form of ds4_mmq_q4_K_dense_vec. The CUDA model resolver
+// supplies weight_device_resident from allocation/cache provenance so the
+// exact K1024 persistent candidate can reject mapped-host/HMM weights without
+// issuing cudaPointerGetAttributes on every decode dispatch. Only a positive
+// hint admits that candidate; zero/negative provenance remains fail-closed and
+// canonical MMVQ stays available. The legacy entry
+// above performs its own pointer-attribute query for standalone callers and
+// benchmarks that allocate W with cudaMalloc.
+int ds4_mmq_q4_K_dense_vec_with_weight_residency(
+    const void  * W_q4_K,
+    const float * X_f32,
+    float       * out_f32,
+    int           M,
+    int           N,
+    int           K,
+    int           weight_device_resident,
+    cudaStream_t  stream);
+
+// Exact grouped one-row Q4_K MMVQ for AProjQ4 attention-A on a single GB10.
+// W is [n_groups][M][K], X is [n_groups][K], and out is
+// [n_groups][M].  Each group retains the canonical dense_vec reduction tree;
+// only Q8_1 quantization and launch setup are shared.  Returns
+// DS4_MMQ_NOT_APPLICABLE before enqueue when its GB10/scratch/shape gates do
+// not hold.
+int ds4_mmq_q4_K_grouped_vec(
+    const void  * W_q4_K,
+    const float * X_f32,
+    float       * out_f32,
+    int           M,
+    int           K,
+    int           n_groups,
+    cudaStream_t  stream);
+
+// Token-aware form of the exact grouped Q4_K entry above.  X and out are
+// token-major: [n_tokens][n_groups][K] and
+// [n_tokens][n_groups][M].  Internally (token, group) is flattened into the
+// MMVQ channel dimension while ncols_dst remains one.  This is deliberate:
+// every pair therefore retains the canonical one-row Q8_1 quantization,
+// Q4_K K partition, peer-warp fold and reduction tree.  The GB10 path accepts
+// at most eight tokens and is opt-in with
+// DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_BATCH=1.  Either
+// DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH=1 or the existing grouped/global kill
+// switches disables it.  DS4_MMQ_NOT_APPLICABLE is returned before enqueue
+// whenever a gate or scratch-capacity check fails.
+// The graph-level diagnostic
+// DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_BATCH=1 turns such ineligibility into a
+// visible failure when this attention-output path is reached.
+int ds4_mmq_q4_K_grouped_batch_vec(
+    const void  * W_q4_K,
+    const float * X_f32,
+    float       * out_f32,
+    int           M,
+    int           K,
+    int           n_tokens,
+    int           n_groups,
+    cudaStream_t  stream);
+
+// On a single GB10, the exact AProjQ4 Q-b shape (M=32768, N=1, K=1024)
+// can opt into a persistent-CTA form with
+// DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT=1.  The rollback switch
+// DS4_CUDA_NO_Q4_K1024_PERSISTENT=1 is authoritative when both are set.
+// DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT=1 makes an unavailable exact-shape
+// dispatch fail before any CUDA work is enqueued instead of silently running
+// canonical MMVQ. DS4_CUDA_Q4_K1024_PERSISTENT_ORACLE=1 forces a candidate,
+// compares it bit-for-bit with canonical MMVQ, and retains canonical output;
+// run it with DS4_CUDA_DECODE_GRAPHS=0. Set
+// DS4_CUDA_Q4_K1024_PERSISTENT_STATS=1 for the atexit counter summary.
+// DS4_CUDA_NO_Q4_GB10_FAST=1 is the umbrella rollback for this and the
+// GB10 Q4 activation scratch. The candidate additionally requires W to be in
+// CUDA device allocation/cache storage; mapped host and managed/HMM pointers
+// retain canonical MMVQ. Other shapes and devices retain canonical MMVQ.
+void ds4_mmq_q4_K_k1024_persistent_counters(
+    uint64_t *candidates,
+    uint64_t *uses,
+    uint64_t *fallbacks,
+    uint64_t *require_failures,
+    uint64_t *oracle_calls,
+    uint64_t *oracle_mismatches,
+    uint64_t *oracle_skips);
+
+// Two independent dense Q4_K projections that share the canonical Q8_1
+// activation quantization.  Each output is dispatched through the same MMVQ
+// entry as ds4_mmq_q4_K_dense_vec, so its reduction and output bits are
+// unchanged; M0 and M1 may differ (the DS4 Q-A/KV decode shape does).
+// Shape/capability rejection returns DS4_MMQ_NOT_APPLICABLE before enqueue;
+// negative values report an attempted-path launch failure.
 int ds4_mmq_q4_K_dense_pair_vec(
     const void  * W0_q4_K,
     const void  * W1_q4_K,
     const float * X_f32,
     float       * out0_f32,
     float       * out1_f32,
-    int           M,
+    int           M0,
+    int           M1,
+    int           N,
     int           K,
     cudaStream_t  stream);
 
diff --git a/cuda/mmq/ds4_mmq_q4_16warp.cu b/cuda/mmq/ds4_mmq_q4_16warp.cu
new file mode 100644
index 0000000000..f1b22b4239
--- /dev/null
+++ b/cuda/mmq/ds4_mmq_q4_16warp.cu
@@ -0,0 +1,685 @@
+// SPDX-License-Identifier: MIT
+// Dense Q4_K x canonical-MMQ-Q8_1, m128n128, 16-warp experiment.
+//
+// The canonical Turing/Ampere MMQ kernel assigns two 16-row MMA minitiles to
+// each of eight warps at N=128.  That leaves 64 F32 accumulators per thread
+// and can spill on shallow-K, wide-M prefill projections.  This kernel keeps
+// the canonical shared representation and arithmetic but splits each 128-row
+// tile over four N-warps for each 32-row band.  Each warp therefore owns
+// 32 rows x 32 columns and carries 32 accumulators.  Keeping both 16-row A
+// fragments in one warp also preserves the canonical reuse of each B load.
+//
+// Numerical contract:
+//   * canonical Q4_K nibble/scales/min unpack;
+//   * canonical Q8_1 DS4 (half scale + half sum) activation blocks;
+//   * identical ascending sequence of eight K32 folds per Q4_K block;
+//   * the two canonical F32 accumulation statements are kept verbatim;
+//   * direct tiling and canonical Stream-K/fixup reduction trees are both
+//     available; the caller chooses explicitly at the enqueue boundary.
+
+#include "ds4_mmq_q4_16warp.cuh"
+
+#include "common.cuh"
+#include "mmq.cuh"
+
+#include <cstddef>
+#include <cstdint>
+
+namespace {
+namespace q4w16 {
+
+constexpr int kMTile       = 128;
+constexpr int kNTile       = 128;
+constexpr int kRowGroups   = 4;
+constexpr int kColWarps    = 4;
+constexpr int kWarps       = kRowGroups * kColWarps;
+constexpr int kThreads     = 32 * kWarps;
+constexpr int kRowFrag     = 2;
+constexpr int kNFrag       = kNTile / 8;
+constexpr int kNFragPerWarp = kNFrag / kColWarps;
+constexpr int kMetadataWarps = kMTile / 16;
+constexpr int kWeightStride = MMQ_MMA_TILE_X_K_Q8_1;
+constexpr int kYStrideInts = sizeof(block_q8_1_mmq) / sizeof(int);
+constexpr int kYChunks16   = sizeof(block_q8_1_mmq) / 16;
+constexpr size_t kTileElements = (size_t)kMTile * (size_t)kNTile;
+
+constexpr size_t kWeightTileBytes =
+    (size_t)kMTile * (size_t)kWeightStride * sizeof(int);
+constexpr size_t kYTileBytes =
+    (size_t)kNTile * sizeof(block_q8_1_mmq);
+constexpr size_t kSharedBytes = kWeightTileBytes + kYTileBytes;
+constexpr int kYTileVectors = (int)(kYTileBytes / sizeof(int4));
+
+static_assert(kWarps == 16, "Q4 16-warp decomposition changed");
+static_assert(kThreads == 512, "Q4 16-warp CTA must have 512 threads");
+static_assert(kRowGroups * kRowFrag * 16 == kMTile,
+              "Q4 split-N row coverage changed");
+static_assert(kNFragPerWarp == 4, "Q4 split-N fragment count changed");
+static_assert(kWeightStride == 76, "canonical Q4_K MMA row stride changed");
+static_assert(kYStrideInts == 36, "canonical Q8_1 DS4 stride changed");
+static_assert(kYChunks16 == 9, "canonical Q8_1 DS4 block size changed");
+static_assert(kYTileBytes % sizeof(int4) == 0,
+              "Q8_1 tile must support vectorized copies");
+static_assert(MMQ_ITER_K % QK_K == 0 && MMQ_ITER_K / QK_K == 1,
+              "Stream-K scheduler must restore canonical K alignment");
+static_assert(kSharedBytes == 57344, "Q4 16-warp shared-memory model changed");
+static_assert(kSharedBytes <= 99ull * 1024ull,
+              "Q4 16-warp kernel exceeds the intended opt-in shared limit");
+
+__device__ __forceinline__ int lane_id() {
+    return (int)threadIdx.x;
+}
+
+__device__ __forceinline__ int warp_id() {
+    return (int)threadIdx.y;
+}
+
+__device__ __forceinline__ int linear_tid() {
+    return (warp_id() << 5) | lane_id();
+}
+
+template <bool need_check>
+__device__ __forceinline__ void load_weight_tile(
+        const block_q4_K * __restrict__ W,
+        int * __restrict__ tile,
+        int cta_row0,
+        int M,
+        int blocks_per_row,
+        int kb) {
+    int *x_qs = tile;
+    half2 *x_dm = reinterpret_cast<half2 *>(x_qs + 2 * MMQ_TILE_NE_K);
+    const int lane = lane_id();
+    const int warp = warp_id();
+
+    // Canonical load_tiles_q4_K nibble expansion.  With 16 warps each warp
+    // visits eight rows; all 128 rows are covered exactly once.
+#pragma unroll
+    for (int row = warp; row < kMTile; row += kWarps) {
+        const int global_row = need_check && cta_row0 + row >= M
+            ? M - 1 : cta_row0 + row;
+        const block_q4_K *b =
+            W + (uint64_t)global_row * (uint64_t)blocks_per_row + kb;
+        const int qs0 = get_int_b4(b->qs, lane);
+        x_qs[row * kWeightStride + 16 * (lane / 8) + lane % 8 + 0] =
+            (qs0 >> 0) & 0x0F0F0F0F;
+        x_qs[row * kWeightStride + 16 * (lane / 8) + lane % 8 + 8] =
+            (qs0 >> 4) & 0x0F0F0F0F;
+    }
+
+    // The canonical loader uses 16 rows/warp and two lanes/row for metadata.
+    // Only eight warps participate, so the extra split-N warps do not
+    // duplicate any metadata row.
+    if (warp < kMetadataWarps) {
+        const int row = warp * 16 + lane / 2;
+        const int ksc = lane & 1;
+        const int global_row = need_check && cta_row0 + row >= M
+            ? M - 1 : cta_row0 + row;
+        const block_q4_K *b =
+            W + (uint64_t)global_row * (uint64_t)blocks_per_row + kb;
+        const int *scales = reinterpret_cast<const int *>(b->scales);
+        const int sc32 = unpack_scales_q45_K(scales, ksc + 0);
+        const int m32  = unpack_scales_q45_K(scales, ksc + 2);
+        const uint8_t *sc8 = reinterpret_cast<const uint8_t *>(&sc32);
+        const uint8_t *m8  = reinterpret_cast<const uint8_t *>(&m32);
+        const half2 dm = b->dm * make_half2(1.0f, -1.0f);
+#pragma unroll
+        for (int l = 0; l < (int)sizeof(int); ++l) {
+            x_dm[row * kWeightStride + (int)sizeof(int) * ksc + l] =
+                dm * make_half2(sc8[l], m8[l]);
+        }
+    }
+}
+
+__device__ __forceinline__ void load_y_tile(
+        const block_q8_1_mmq * __restrict__ q8,
+        block_q8_1_mmq * __restrict__ tile,
+        int N,
+        int col0,
+        int k128) {
+    const int tid = linear_tid();
+
+    // The production selector admits complete N128 tiles.  Copy those as one
+    // contiguous vector range, matching canonical MMQ's flat cooperative
+    // load.  The former column-major mapping made every warp issue 144-byte-
+    // strided global loads; flattening turns each warp's accesses into adjacent
+    // 16-byte vectors while preserving the shared representation byte-for-byte.
+    if (col0 <= N - kNTile) {
+        const int4 * __restrict__ src = reinterpret_cast<const int4 *>(
+            q8 + (uint64_t)k128 * (uint64_t)N + (uint64_t)col0);
+        int4 * __restrict__ dst = reinterpret_cast<int4 *>(tile);
+#pragma unroll
+        for (int vector = tid; vector < kYTileVectors;
+             vector += kThreads) {
+            dst[vector] = src[vector];
+        }
+        return;
+    }
+
+    // Keep the guarded per-column copy for the N tail accepted by the direct
+    // oracle hook.  Production never takes this path.
+    constexpr int threads_per_col = kThreads / kNTile;
+    static_assert(threads_per_col == 4,
+                  "Q8_1 DS4 copy mapping changed");
+    const int col = tid & (kNTile - 1);
+#pragma unroll
+    for (int chunk = tid >> 7; chunk < kYChunks16;
+         chunk += threads_per_col) {
+        int4 value = make_int4(0, 0, 0, 0);
+        if (col0 + col < N) {
+            const char *src = reinterpret_cast<const char *>(
+                q8 + (uint64_t)k128 * (uint64_t)N + (uint64_t)(col0 + col));
+            value = *reinterpret_cast<const int4 *>(src + chunk * 16);
+        }
+        char *dst = reinterpret_cast<char *>(tile + col);
+        *reinterpret_cast<int4 *>(dst + chunk * 16) = value;
+    }
+}
+
+template <typename TileA, typename TileB, typename TileC>
+__device__ __forceinline__ void fold_y_half(
+        float (&acc)[kNFragPerWarp][kRowFrag][TileC::ne],
+        const int * __restrict__ x_tile,
+        const block_q8_1_mmq * __restrict__ y_tile,
+        int x_group0) {
+    static_assert(TileC::ne == 4,
+                  "expected m16n8 s32 accumulator fragment");
+    const half2 *x_dm = reinterpret_cast<const half2 *>(
+        x_tile + 2 * MMQ_TILE_NE_K);
+    const int warp = warp_id();
+    const int row0 = (warp / kColWarps) * (kRowFrag * 16);
+    const int nf0 = (warp % kColWarps) * kNFragPerWarp;
+    const int c0 = TileC::get_j(0);
+    const int c1 = TileC::get_j(1);
+    const int r0 = TileC::get_i(0);
+    const int r1 = TileC::get_i(2);
+
+    // K32-phased A loads keep only the two fragments needed for this 32-row
+    // band live, instead of canonical MMQ's eight K32 phases at once.  Each
+    // B fragment is reused by both A fragments exactly as in canonical MMQ.
+    // For every output element folds remain in canonical group order 0..7.
+#pragma unroll
+    for (int local_group = 0; local_group < 4; ++local_group) {
+        const int x_group = x_group0 + local_group;
+        TileA A[kRowFrag];
+        float2 dmA[kRowFrag][2];
+#pragma unroll
+        for (int nr = 0; nr < kRowFrag; ++nr) {
+            const int frag_row0 = row0 + nr * 16;
+            ggml_cuda_mma::load_ldmatrix(
+                A[nr],
+                x_tile + frag_row0 * kWeightStride + x_group * QI8_1,
+                kWeightStride);
+            dmA[nr][0] = __half22float2(
+                x_dm[(frag_row0 + r0) * kWeightStride + x_group]);
+            dmA[nr][1] = __half22float2(
+                x_dm[(frag_row0 + r1) * kWeightStride + x_group]);
+        }
+
+#pragma unroll
+        for (int nf = 0; nf < kNFragPerWarp; ++nf) {
+            const int col_base = (nf0 + nf) * 8;
+            TileB B;
+            const int *b_qs = reinterpret_cast<const int *>(
+                &y_tile[col_base].qs[local_group * QK8_1]);
+            // Canonical NVIDIA MMQ deliberately uses load_generic for B.
+            ggml_cuda_mma::load_generic(B, b_qs, kYStrideInts);
+
+            const float2 dsB[2] = {
+                __half22float2(y_tile[col_base + c0].ds4[local_group]),
+                __half22float2(y_tile[col_base + c1].ds4[local_group]),
+            };
+
+            // These are the canonical vec_dot_q8_1_q8_1_mma accumulation
+            // statements.  Do not fuse the min correction into the dot fold
+            // or change their order: parity depends on this reduction tree.
+#pragma unroll
+            for (int nr = 0; nr < kRowFrag; ++nr) {
+                TileC C;
+                ggml_cuda_mma::mma(C, A[nr], B);
+#pragma unroll
+                for (int l = 0; l < TileC::ne; ++l) {
+                    acc[nf][nr][l] +=
+                        dmA[nr][l / 2].x * dsB[l % 2].x * C.x[l];
+                    acc[nf][nr][l] +=
+                        dmA[nr][l / 2].y * dsB[l % 2].y;
+                }
+            }
+        }
+    }
+}
+
+template <bool to_fixup, bool need_check>
+__device__ __forceinline__ void process_tile_range(
+        const block_q4_K * __restrict__ W,
+        const block_q8_1_mmq * __restrict__ q8,
+        float * __restrict__ out,
+        float * __restrict__ tmp_fixup,
+        int M,
+        int N,
+        int K,
+        int it,
+        int jt,
+        int kb_start,
+        int kb_stop,
+        int * __restrict__ x_tile,
+        block_q8_1_mmq * __restrict__ y_tile) {
+#if defined(TURING_MMA_AVAILABLE)
+    using tile_A = ggml_cuda_mma::tile<16, 8, int>;
+    using tile_B = ggml_cuda_mma::tile<8, 8, int>;
+    using tile_C = ggml_cuda_mma::tile<16, 8, int>;
+
+    const int cta_row0 = it * kMTile;
+    const int col0 = jt * kNTile;
+    const int blocks_per_row = K / QK_K;
+    float acc[kNFragPerWarp][kRowFrag][tile_C::ne] = {};
+
+    for (int kb = kb_start; kb < kb_stop; ++kb) {
+        load_weight_tile<need_check>(
+            W, x_tile, cta_row0, M, blocks_per_row, kb);
+        load_y_tile(q8, y_tile, N, col0, 2 * kb + 0);
+        __syncthreads();
+
+        fold_y_half<tile_A, tile_B, tile_C>(acc, x_tile, y_tile, 0);
+        __syncthreads();
+
+        load_y_tile(q8, y_tile, N, col0, 2 * kb + 1);
+        __syncthreads();
+
+        fold_y_half<tile_A, tile_B, tile_C>(acc, x_tile, y_tile, 4);
+        // Protect both shared tiles before the following K256 iteration.
+        __syncthreads();
+    }
+
+    const int warp = warp_id();
+    const int out_row0 =
+        cta_row0 + (warp / kColWarps) * (kRowFrag * 16);
+    const int out_col0 =
+        col0 + (warp % kColWarps) * (kNFragPerWarp * 8);
+#pragma unroll
+    for (int nf = 0; nf < kNFragPerWarp; ++nf) {
+#pragma unroll
+        for (int nr = 0; nr < kRowFrag; ++nr) {
+#pragma unroll
+            for (int l = 0; l < tile_C::ne; ++l) {
+                const int row = out_row0 + nr * 16 + tile_C::get_i(l);
+                const int col = out_col0 + nf * 8 + tile_C::get_j(l);
+                if constexpr (to_fixup) {
+                    // Canonical MMQ always materializes a complete 128x128
+                    // final-partial tile in block-private storage.  Tail rows
+                    // and columns are masked only when the fixup publishes it.
+                    tmp_fixup[(size_t)blockIdx.x * kTileElements +
+                              (size_t)(col - col0) * kMTile +
+                              (size_t)(row - cta_row0)] = acc[nf][nr][l];
+                } else if (row < M && col < N) {
+                    float value = acc[nf][nr][l];
+                    // A leading Stream-K partial is deliberately left
+                    // unsanitized.  Canonical fixup sanitizes only after the
+                    // complete reduction tree has been reconstructed.
+                    if (kb_start == 0 && kb_stop == blocks_per_row &&
+                        !isfinite(value)) {
+                        value = 0.0f;
+                    }
+                    out[(uint64_t)col * (uint64_t)M + (uint64_t)row] =
+                        value;
+                }
+            }
+        }
+    }
+#else
+    GGML_UNUSED_VARS(W, q8, out, tmp_fixup, M, N, K, it, jt,
+                     kb_start, kb_stop);
+    GGML_UNUSED_VARS(x_tile, y_tile);
+    NO_DEVICE_CODE;
+#endif
+}
+
+__global__ __launch_bounds__(kThreads, 1)
+void dense_q4_16warp_kernel(
+        const block_q4_K * __restrict__ W,
+        const block_q8_1_mmq * __restrict__ q8,
+        float * __restrict__ out,
+        int M,
+        int N,
+        int K) {
+#if defined(TURING_MMA_AVAILABLE)
+    extern __shared__ __align__(16) unsigned char dynamic_smem[];
+    int *x_tile = reinterpret_cast<int *>(dynamic_smem);
+    block_q8_1_mmq *y_tile = reinterpret_cast<block_q8_1_mmq *>(
+        dynamic_smem + kWeightTileBytes);
+    const int it = (int)blockIdx.x;
+    const int jt = (int)blockIdx.y;
+    process_tile_range<false, false>(
+        W, q8, out, nullptr, M, N, K, it, jt,
+        0, K / QK_K, x_tile, y_tile);
+#else
+    GGML_UNUSED_VARS(W, q8, out, M, N, K);
+    NO_DEVICE_CODE;
+#endif
+}
+
+// The integer partition and flattened tile order intentionally mirror
+// mul_mat_q<GGML_TYPE_Q4_K, 128, ...>.  One CTA may finish a leading split
+// tile, own zero or more complete tiles, and publish one trailing prefix for
+// canonical mul_mat_q_stream_k_fixup.
+template <bool need_check>
+__global__ __launch_bounds__(kThreads, 1)
+void dense_q4_16warp_streamk_kernel(
+        const block_q4_K * __restrict__ W,
+        const block_q8_1_mmq * __restrict__ q8,
+        float * __restrict__ out,
+        float * __restrict__ tmp_fixup,
+        int M,
+        int N,
+        int K) {
+#if defined(TURING_MMA_AVAILABLE)
+    extern __shared__ __align__(16) unsigned char dynamic_smem[];
+    int *x_tile = reinterpret_cast<int *>(dynamic_smem);
+    block_q8_1_mmq *y_tile = reinterpret_cast<block_q8_1_mmq *>(
+        dynamic_smem + kWeightTileBytes);
+
+    // Convert before adding the tile bias so syntactically valid INT_MAX
+    // dimensions cannot overflow signed arithmetic in device code.
+    const int nty = (int)(((unsigned)M + (unsigned)kMTile - 1u) /
+                          (unsigned)kMTile);
+    const int ntx = (int)(((unsigned)N + (unsigned)kNTile - 1u) /
+                          (unsigned)kNTile);
+    const int blocks_per_row = K / QK_K;
+    const int64_t total = (int64_t)nty * ntx * blocks_per_row;
+
+    int kbc = (int)((int64_t)blockIdx.x * total / gridDim.x);
+    const int kbc_stop =
+        (int)((int64_t)(blockIdx.x + 1) * total / gridDim.x);
+
+    int kb_start = kbc % blocks_per_row;
+    int kb_stop = min(blocks_per_row, kb_start + kbc_stop - kbc);
+    while (kbc < kbc_stop && kb_stop == blocks_per_row) {
+        const int tile = kbc / blocks_per_row;
+        const int jt = tile % ntx;
+        const int it = tile / ntx;
+        process_tile_range<false, need_check>(
+            W, q8, out, tmp_fixup, M, N, K, it, jt,
+            kb_start, kb_stop, x_tile, y_tile);
+
+        kbc += blocks_per_row;
+        kbc -= kbc % blocks_per_row;
+        kb_start = 0;
+        kb_stop = min(blocks_per_row, kbc_stop - kbc);
+    }
+
+    if (kbc >= kbc_stop) {
+        return;
+    }
+
+    const int tile = kbc / blocks_per_row;
+    const int jt = tile % ntx;
+    const int it = tile / ntx;
+    process_tile_range<true, need_check>(
+        W, q8, out, tmp_fixup, M, N, K, it, jt,
+        kb_start, kb_stop, x_tile, y_tile);
+#else
+    GGML_UNUSED_VARS(W, q8, out, tmp_fixup, M, N, K);
+    NO_DEVICE_CODE;
+#endif
+}
+
+struct streamk_schedule {
+    unsigned nty;
+    unsigned ntx;
+    unsigned ntiles;
+    unsigned grid_x;
+    bool fixup_needed;
+    size_t scratch_bytes;
+};
+
+static bool make_streamk_schedule(
+        int M, int N, int nsm, streamk_schedule *schedule) {
+    if (M <= 0 || N <= 0 || nsm <= 0 || schedule == nullptr) {
+        return false;
+    }
+
+    const uint64_t nty64 =
+        ((uint64_t)(unsigned)M + (uint64_t)kMTile - 1u) / kMTile;
+    const uint64_t ntx64 =
+        ((uint64_t)(unsigned)N + (uint64_t)kNTile - 1u) / kNTile;
+    if (nty64 == 0 || ntx64 == 0 || nty64 > UINT32_MAX ||
+        ntx64 > UINT32_MAX || nty64 > UINT32_MAX / ntx64) {
+        return false;
+    }
+
+    const uint64_t ntiles64 = nty64 * ntx64;
+    const uint64_t nsm64 = (uint64_t)(unsigned)nsm;
+    const uint64_t nwaves = (ntiles64 + nsm64 - 1u) / nsm64;
+    if (nwaves == 0 || nsm64 > UINT64_MAX / nwaves) {
+        return false;
+    }
+    const uint64_t wave_slots = nsm64 * nwaves;
+    const uint64_t efficiency = 100u * ntiles64 / wave_slots;
+    const uint64_t grid64 = efficiency >= 90u ? ntiles64 : nsm64;
+    if (grid64 == 0 || grid64 > UINT32_MAX) {
+        return false;
+    }
+
+    const bool fixup_needed = ntiles64 % grid64 != 0;
+    size_t bytes = 0;
+    if (fixup_needed) {
+        if (grid64 > SIZE_MAX / kTileElements / sizeof(float)) {
+            return false;
+        }
+        bytes = (size_t)grid64 * kTileElements * sizeof(float);
+    }
+
+    schedule->nty = (unsigned)nty64;
+    schedule->ntx = (unsigned)ntx64;
+    schedule->ntiles = (unsigned)ntiles64;
+    schedule->grid_x = (unsigned)grid64;
+    schedule->fixup_needed = fixup_needed;
+    schedule->scratch_bytes = bytes;
+    return true;
+}
+
+} // namespace q4w16
+} // anonymous namespace
+
+extern "C" int ds4_mmq_q4_K_dense_16warp_available(int cc) {
+    return GGML_CUDA_CC_IS_NVIDIA(cc) &&
+           ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_AMPERE;
+}
+
+extern "C" int ds4_mmq_q4_K_dense_16warp_supported(
+        int cc, int M, int N, int K) {
+    if (!ds4_mmq_q4_K_dense_16warp_available(cc)) {
+        return 0;
+    }
+    return M >= 1024 && (M % q4w16::kMTile) == 0 &&
+           N >= 512 && (N % q4w16::kNTile) == 0 &&
+           K >= 1024 && K <= 8192 && (K % QK_K) == 0;
+}
+
+extern "C" int ds4_mmq_q4_K_dense_16warp_prepare(void) {
+    using namespace q4w16;
+    int device = -1;
+    cudaError_t err = cudaGetDevice(&device);
+    if (err != cudaSuccess) {
+        return -1;
+    }
+    cudaDeviceProp prop;
+    err = cudaGetDeviceProperties(&prop, device);
+    if (err != cudaSuccess || prop.major < 8 ||
+        prop.maxThreadsPerBlock < kThreads) {
+        return -2;
+    }
+#if CUDART_VERSION >= 9000
+    if ((size_t)prop.sharedMemPerBlockOptin < kSharedBytes) {
+        return -2;
+    }
+#else
+    if ((size_t)prop.sharedMemPerBlock < kSharedBytes) {
+        return -2;
+    }
+#endif
+    err = cudaFuncSetAttribute(
+        dense_q4_16warp_kernel,
+        cudaFuncAttributeMaxDynamicSharedMemorySize,
+        (int)kSharedBytes);
+    if (err != cudaSuccess) {
+        return -3;
+    }
+    err = cudaFuncSetAttribute(
+        dense_q4_16warp_streamk_kernel<false>,
+        cudaFuncAttributeMaxDynamicSharedMemorySize,
+        (int)kSharedBytes);
+    if (err != cudaSuccess) {
+        return -3;
+    }
+    err = cudaFuncSetAttribute(
+        dense_q4_16warp_streamk_kernel<true>,
+        cudaFuncAttributeMaxDynamicSharedMemorySize,
+        (int)kSharedBytes);
+    return err == cudaSuccess ? 0 : -3;
+}
+
+extern "C" int ds4_mmq_q4_K_dense_16warp_enqueue(
+        const void *W,
+        const void *q8_ds4,
+        float *out,
+        int M,
+        int N,
+        int K,
+        cudaStream_t stream) {
+    using namespace q4w16;
+    if (!W || !q8_ds4 || !out || M <= 0 || N <= 0 || K <= 0 ||
+        (M % kMTile) != 0 || (K % QK_K) != 0) {
+        return -1;
+    }
+
+    // Convert before adding the tile bias: N is a positive signed int, but
+    // N + 127 would otherwise overflow for a (syntactically valid) INT_MAX
+    // direct-enqueue request.
+    const unsigned grid_y =
+        ((unsigned)N + (unsigned)kNTile - 1u) / (unsigned)kNTile;
+    // CUDA guarantees only 65535 blocks on y/z. Production shapes are far
+    // below this, but reject oversized raw-enqueue requests before launch.
+    if (grid_y > 65535u) {
+        return -1;
+    }
+    const dim3 grid((unsigned)M / (unsigned)kMTile, grid_y, 1);
+    const dim3 block(32, kWarps, 1);
+    dense_q4_16warp_kernel<<<grid, block, kSharedBytes, stream>>>(
+        static_cast<const block_q4_K *>(W),
+        static_cast<const block_q8_1_mmq *>(q8_ds4),
+        out, M, N, K);
+    const cudaError_t err = cudaGetLastError();
+    return err == cudaSuccess ? 0 : -4;
+}
+
+extern "C" size_t ds4_mmq_q4_K_dense_16warp_streamk_scratch_bytes(
+        int M, int N, int nsm) {
+    q4w16::streamk_schedule schedule;
+    return q4w16::make_streamk_schedule(M, N, nsm, &schedule)
+        ? schedule.scratch_bytes : 0;
+}
+
+extern "C" int ds4_mmq_q4_K_dense_16warp_streamk_enqueue(
+        const void *W,
+        const void *q8_ds4,
+        float *out,
+        void *scratch,
+        size_t scratch_bytes,
+        int M,
+        int N,
+        int K,
+        int nsm,
+        cudaStream_t stream) {
+    using namespace q4w16;
+    if (!W || !q8_ds4 || !out || M <= 0 || N <= 0 || K <= 0 ||
+        nsm <= 0 || (K % QK_K) != 0) {
+        return -1;
+    }
+
+    streamk_schedule schedule;
+    if (!make_streamk_schedule(M, N, nsm, &schedule)) {
+        return -1;
+    }
+
+    const uint64_t blocks_per_row = (uint64_t)(unsigned)K / QK_K;
+    const uint64_t total = (uint64_t)schedule.ntiles * blocks_per_row;
+    // Match the canonical launcher's invariant: the device scheduler stores
+    // flattened K-block coordinates in signed int variables.
+    if (blocks_per_row == 0 || total >= (1ull << 30)) {
+        return -1;
+    }
+
+    // Canonical Stream-K degenerates to one complete-K CTA per output tile
+    // at high whole-tile efficiency.  Preserve that policy while using the
+    // cheaper 2-D direct launch for the aligned production shapes: it avoids
+    // per-CTA flattened-index divisions and needs neither scratch nor fixup.
+    if (!schedule.fixup_needed && (M % kMTile) == 0) {
+        return ds4_mmq_q4_K_dense_16warp_enqueue(
+            W, q8_ds4, out, M, N, K, stream);
+    }
+
+    if (schedule.fixup_needed) {
+        if (!scratch || scratch_bytes < schedule.scratch_bytes ||
+            ((uintptr_t)scratch % alignof(float)) != 0) {
+            return -1;
+        }
+        const cudaError_t memset_err = cudaMemsetAsync(
+            scratch, 0, schedule.scratch_bytes, stream);
+        if (memset_err != cudaSuccess) {
+            return -2;
+        }
+    }
+
+    const dim3 grid(schedule.grid_x, 1, 1);
+    const dim3 block(32, kWarps, 1);
+    float *tmp_fixup = schedule.fixup_needed
+        ? static_cast<float *>(scratch) : nullptr;
+    if ((M % kMTile) == 0) {
+        dense_q4_16warp_streamk_kernel<false>
+            <<<grid, block, kSharedBytes, stream>>>(
+                static_cast<const block_q4_K *>(W),
+                static_cast<const block_q8_1_mmq *>(q8_ds4),
+                out, tmp_fixup, M, N, K);
+    } else {
+        dense_q4_16warp_streamk_kernel<true>
+            <<<grid, block, kSharedBytes, stream>>>(
+                static_cast<const block_q4_K *>(W),
+                static_cast<const block_q8_1_mmq *>(q8_ds4),
+                out, tmp_fixup, M, N, K);
+    }
+    cudaError_t err = cudaGetLastError();
+    if (err != cudaSuccess) {
+        return -4;
+    }
+
+    if (!schedule.fixup_needed) {
+        return 0;
+    }
+
+    const uint3 blocks_per_ne00_fd = init_fastdiv_values(blocks_per_row);
+    const uint3 one_fd = init_fastdiv_values(1);
+    const uint3 ntx_fd = init_fastdiv_values(schedule.ntx);
+    const dim3 fixup_grid(schedule.grid_x, kMTile / 32, 1);
+    const dim3 fixup_block(32, 4, 1);
+    if ((M % kMTile) == 0) {
+        constexpr bool need_check = false;
+        mul_mat_q_stream_k_fixup<
+            GGML_TYPE_Q4_K, kNTile, need_check, false>
+            <<<fixup_grid, fixup_block, 0, stream>>>(
+                /*ids_dst=*/nullptr, /*expert_bounds=*/nullptr, out,
+                tmp_fixup, blocks_per_ne00_fd, M, N, M,
+                one_fd, /*stride_channel_dst=*/0,
+                one_fd, /*stride_sample_dst=*/0, ntx_fd);
+    } else {
+        constexpr bool need_check = true;
+        mul_mat_q_stream_k_fixup<
+            GGML_TYPE_Q4_K, kNTile, need_check, false>
+            <<<fixup_grid, fixup_block, 0, stream>>>(
+                /*ids_dst=*/nullptr, /*expert_bounds=*/nullptr, out,
+                tmp_fixup, blocks_per_ne00_fd, M, N, M,
+                one_fd, /*stride_channel_dst=*/0,
+                one_fd, /*stride_sample_dst=*/0, ntx_fd);
+    }
+    err = cudaGetLastError();
+    return err == cudaSuccess ? 0 : -5;
+}
diff --git a/cuda/mmq/ds4_mmq_q4_16warp.cuh b/cuda/mmq/ds4_mmq_q4_16warp.cuh
new file mode 100644
index 0000000000..2434330a25
--- /dev/null
+++ b/cuda/mmq/ds4_mmq_q4_16warp.cuh
@@ -0,0 +1,94 @@
+// SPDX-License-Identifier: MIT
+// Internal CUDA Q4_K dense-prefill experiment.  This header intentionally
+// exposes only the pre-quantized enqueue boundary; allocation and Q8_1
+// quantization stay owned by ds4_mmq.cu.
+
+#pragma once
+
+#if defined(GGML_USE_HIP)
+#include "vendors/hip.h"
+#else
+#include <cuda_runtime.h>
+#endif
+
+#include <stddef.h>
+
+#if !defined(GGML_USE_HIP)
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// Returns non-zero when the requested CUDA compute capability can execute
+// the m128n128, 16-warp integer-MMA kernel.
+int ds4_mmq_q4_K_dense_16warp_available(int cc);
+
+// Conservative standalone production admission gate. Availability and shape
+// are both checked; it admits M>=1024 and only complete 128x128 output tiles.
+// The K envelope covers the production 8192-wide attention output projection.
+// The dispatcher separately enforces the m128n128 reference selector and its
+// candidate-grid efficiency gate. A false result must fall back. The pair
+// dispatcher has a separate per-leg M>=512 gate.
+int ds4_mmq_q4_K_dense_16warp_supported(int cc, int M, int N, int K);
+
+// Opt in the 56 KiB dynamic-shared-memory launch on the current device.
+// Call once during device initialization (and once again after switching to a
+// different device) before enqueue.  The operation is idempotent.
+int ds4_mmq_q4_K_dense_16warp_prepare(void);
+
+// Enqueue-only dense Q4_K GEMM over an already resident canonical MMQ Q8_1
+// activation buffer.
+//
+//   W      raw row-major block_q4_K, [M][K/256]
+//   q8_ds4 block_q8_1_mmq DS4 (half scale + half sum), [K/128][N]
+//   out    column-major float, [N][M]
+//
+// ds4_mmq_q4_K_dense_16warp_prepare must have succeeded on the current device.
+// The kernel owns the complete K reduction for every output tile: it never
+// uses stream-K and writes every valid output exactly once.  No allocation,
+// memset, quantization, synchronization, or host/device copy is performed.
+// Returns 0 after a successful enqueue and a negative value otherwise.
+int ds4_mmq_q4_K_dense_16warp_enqueue(
+    const void   * W,
+    const void   * q8_ds4,
+    float        * out,
+    int            M,
+    int            N,
+    int            K,
+    cudaStream_t   stream);
+
+// Return the caller-owned fixup storage required by the canonical Stream-K
+// partition for this dense MxN output shape and SM count.  Zero means either
+// that no fixup is necessary (the selected grid owns complete tiles) or that
+// the arguments/size cannot be represented; enqueue repeats all validation.
+// The storage, when non-zero, is a byte buffer and need only remain valid until
+// the work already enqueued on `stream` has completed.
+size_t ds4_mmq_q4_K_dense_16warp_streamk_scratch_bytes(
+    int M,
+    int N,
+    int nsm);
+
+// Enqueue the same 16-warp arithmetic using canonical CUDA MMQ Stream-K
+// scheduling and its exact Q4_K fixup reduction tree.  W, q8_ds4 and out use
+// the layouts documented above.  `scratch` must provide at least the size
+// returned by ds4_mmq_q4_K_dense_16warp_streamk_scratch_bytes; it may be null
+// when that function returns zero.  The routine performs only asynchronous
+// memset/kernel operations and does not allocate or synchronize.
+// ds4_mmq_q4_K_dense_16warp_prepare must have succeeded on the current device.
+int ds4_mmq_q4_K_dense_16warp_streamk_enqueue(
+    const void   * W,
+    const void   * q8_ds4,
+    float        * out,
+    void         * scratch,
+    size_t         scratch_bytes,
+    int            M,
+    int            N,
+    int            K,
+    int            nsm,
+    cudaStream_t   stream);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // !defined(GGML_USE_HIP)
diff --git a/cuda/mmq/mmq.cuh b/cuda/mmq/mmq.cuh
index 0f22964d61..ad78a951ea 100644
--- a/cuda/mmq/mmq.cuh
+++ b/cuda/mmq/mmq.cuh
@@ -3663,6 +3663,7 @@ static __device__ __forceinline__ void mul_mat_q_process_tile(
         const int * __restrict__ ids_dst, float * __restrict__ dst, float * __restrict__ tmp_fixup,
         const int stride_row_x, const int ncols_y, const int stride_col_dst,
         const int tile_x_max_i, const int tile_y_max_j, const int kb0_start, const int kb0_stop,
+        const int blocks_per_ne00_total,
         const char * __restrict__ x_soa, const int64_t soa_blocks) {
 
     constexpr int              warp_size  = ggml_cuda_get_physical_warp_size();
@@ -3756,6 +3757,22 @@ static __device__ __forceinline__ void mul_mat_q_process_tile(
         __syncthreads();
     }
 
+    /* AProjQ4 dense prefill used to run a separate full-output sanitize
+     * kernel after every MMQ.  Preserve that contract in the producer
+     * epilogue instead.  A stream-K block may publish only the leading
+     * partial of a split tile; sanitizing that partial would change the
+     * eventual sum, so only a block that owns the complete K range may fold
+     * non-finite values here.  Split tiles are handled after their final
+     * accumulation in mul_mat_q_stream_k_fixup below. */
+    if constexpr (type == GGML_TYPE_Q4_K && !fixup) {
+        if (kb0_start == 0 && kb0_stop == blocks_per_ne00_total) {
+#pragma unroll
+            for (int l = 0; l < mmq_x*mmq_y / (nwarps*warp_size); ++l) {
+                if (!isfinite(sum[l])) sum[l] = 0.0f;
+            }
+        }
+    }
+
     if (fixup) {
         write_back(sum, nullptr, tmp_fixup + blockIdx.x*(mmq_x*mmq_y), mmq_y, mmq_y, mmq_x);
     } else {
@@ -3766,7 +3783,7 @@ static __device__ __forceinline__ void mul_mat_q_process_tile(
 
 // The mul_mat_q kernel implements "stream-k" work partitioning as described in https://arxiv.org/abs/2301.03598
 
-template <ggml_type type, int mmq_x, bool need_check>
+template <ggml_type type, int mmq_x, bool need_check, bool grid_z_channels = false>
 #if defined(GGML_USE_HIP)
 #if defined(RDNA4) || defined(RDNA3) || defined(RDNA2) || defined(CDNA) || defined(GCN)
     __launch_bounds__(ggml_cuda_get_physical_warp_size()*mmq_get_nwarps_device(), 2)
@@ -3800,6 +3817,22 @@ static __global__ void mul_mat_q(
 
     const uint32_t nty = (nrows_x + mmq_y - 1) / mmq_y; // Number of tiles y
 
+    /* Dense grouped dispatch: grid.z is an outer channel selector while each
+     * z-slice keeps the exact grid.x / stream-k partition of the former
+     * one-launch-per-channel path.  Offset the three channel bases here and
+     * let the ordinary tile code see a single logical channel.  This preserves
+     * each output's K reduction tree while removing the host launch loop. */
+    const int grid_z_channel = grid_z_channels ? (int)blockIdx.z : 0;
+    const int grid_z_offset_x = grid_z_channels
+        ? grid_z_channel * stride_channel_x : 0;
+    if constexpr (grid_z_channels) {
+        y += (int64_t)grid_z_channel * stride_channel_y;
+        dst += (int64_t)grid_z_channel * stride_channel_dst;
+        if (tmp_fixup != nullptr) {
+            tmp_fixup += (int64_t)grid_z_channel * gridDim.x * mmq_x * mmq_y;
+        }
+    }
+
     // Initialize the ids for writing back data with just the index.
     // For regular matrix multiplications this is never changed.
     // For MoE the correct indices are loaded from ids_dst.
@@ -3819,9 +3852,13 @@ static __global__ void mul_mat_q(
     // On non-CDNA AMD or old CUDA the performance with stream-k was worse, use conventional tiling instead:
 #if (defined(GGML_USE_HIP) && !defined(CDNA)) || __CUDA_ARCH__ < GGML_CUDA_CC_VOLTA
     {
-        const uint2 tmp2 = fast_div_modulo(blockIdx.z, nchannels_y);
-        const int wt = tmp2.x;
-        const int zt = tmp2.y;
+        int wt = 0;
+        int zt = 0;
+        if constexpr (!grid_z_channels) {
+            const uint2 tmp2 = fast_div_modulo(blockIdx.z, nchannels_y);
+            wt = tmp2.x;
+            zt = tmp2.y;
+        }
         const int jt = blockIdx.y;
         const int it = blockIdx.x;
 
@@ -3870,12 +3907,13 @@ static __global__ void mul_mat_q(
         const int tile_x_max_i = nrows_x  - it*mmq_y - 1;
         const int tile_y_max_j = col_diff - jt*mmq_x - 1;
 
-        const int offset_x = fastdiv(wt, sample_ratio)*stride_sample_x + fastdiv(zt, channel_ratio)*stride_channel_x + it*mmq_y*stride_row_x;
+        const int offset_x = grid_z_offset_x + fastdiv(wt, sample_ratio)*stride_sample_x + fastdiv(zt, channel_ratio)*stride_channel_x + it*mmq_y*stride_row_x;
 
         constexpr bool fixup = false;
         mul_mat_q_process_tile<type, mmq_x, need_check, fixup>
             (x, offset_x, y + offset_y, ids_dst_shared, dst + offset_dst, tmp_fixup, stride_row_x, ncols_y, stride_col_dst,
-             tile_x_max_i, tile_y_max_j, 0, blocks_per_ne00.z, x_soa, soa_blocks);
+             tile_x_max_i, tile_y_max_j, 0, blocks_per_ne00.z,
+             blocks_per_ne00.z, x_soa, soa_blocks);
         return;
     }
 #endif // (defined(GGML_USE_HIP) && !defined(CDNA4) && !defined(CDNA3)) || __CUDA_ARCH__ < GGML_CUDA_CC_VOLTA
@@ -3956,12 +3994,13 @@ static __global__ void mul_mat_q(
         const int tile_x_max_i = nrows_x  - it*mmq_y - 1;
         const int tile_y_max_j = col_diff - jt*mmq_x - 1;
 
-        const int offset_x = fastdiv(wt, sample_ratio)*stride_sample_x + fastdiv(zt, channel_ratio)*stride_channel_x + it*mmq_y*stride_row_x;
+        const int offset_x = grid_z_offset_x + fastdiv(wt, sample_ratio)*stride_sample_x + fastdiv(zt, channel_ratio)*stride_channel_x + it*mmq_y*stride_row_x;
 
         constexpr bool fixup = false; // All but (potentially) the last iterations write their data to dst rather than the fixup buffer.
         mul_mat_q_process_tile<type, mmq_x, need_check, fixup>
             (x, offset_x, y + offset_y, ids_dst_shared, dst + offset_dst, tmp_fixup, stride_row_x, ncols_y, stride_col_dst,
-             tile_x_max_i, tile_y_max_j, kb0_start, kb0_stop, x_soa, soa_blocks);
+             tile_x_max_i, tile_y_max_j, kb0_start, kb0_stop,
+             blocks_per_ne00.z, x_soa, soa_blocks);
 
         kbc += blocks_per_ne00.z;
         kbc -= fastmodulo(kbc, blocks_per_ne00);
@@ -4025,15 +4064,16 @@ static __global__ void mul_mat_q(
     const int tile_x_max_i = nrows_x  - it*mmq_y - 1;
     const int tile_y_max_j = col_diff - jt*mmq_x - 1;
 
-    const int offset_x = fastdiv(wt, sample_ratio)*stride_sample_x + fastdiv(zt, channel_ratio)*stride_channel_x + it*mmq_y*stride_row_x;
+    const int offset_x = grid_z_offset_x + fastdiv(wt, sample_ratio)*stride_sample_x + fastdiv(zt, channel_ratio)*stride_channel_x + it*mmq_y*stride_row_x;
 
     constexpr bool fixup = true; // Last index writes its data to fixup buffer to avoid data races with other blocks.
     mul_mat_q_process_tile<type, mmq_x, need_check, fixup>
         (x, offset_x, y + offset_y, ids_dst_shared, dst + offset_dst, tmp_fixup, stride_row_x, ncols_y, stride_col_dst,
-         tile_x_max_i, tile_y_max_j, kb0_start, kb0_stop, x_soa, soa_blocks);
+         tile_x_max_i, tile_y_max_j, kb0_start, kb0_stop,
+         blocks_per_ne00.z, x_soa, soa_blocks);
 }
 
-template <ggml_type type, int mmq_x, bool need_check>
+template <ggml_type type, int mmq_x, bool need_check, bool grid_z_channels = false>
 __launch_bounds__(ggml_cuda_get_physical_warp_size()*mmq_get_nwarps_device()/2, 1)
 static __global__ void mul_mat_q_stream_k_fixup(
         const int32_t * __restrict__ ids_dst, const int32_t * __restrict__ expert_bounds, float * __restrict__ dst,
@@ -4048,6 +4088,11 @@ static __global__ void mul_mat_q_stream_k_fixup(
     constexpr int nwarps = mmq_get_nwarps_device()/2;
     constexpr int warp_size = ggml_cuda_get_physical_warp_size();
 
+    if constexpr (grid_z_channels) {
+        dst += (int64_t)blockIdx.z * stride_channel_dst;
+        tmp_last_tile += (int64_t)blockIdx.z * gridDim.x * mmq_x * mmq_y;
+    }
+
     float sum[mmq_x / nwarps] = {0.0f};
     const int i = blockIdx.y*warp_size + threadIdx.x;
 
@@ -4136,7 +4181,12 @@ static __global__ void mul_mat_q_stream_k_fixup(
                 return;
             }
 
-            dst[j*stride_col_dst + i] += sum[j0/nwarps];
+            const int dst_idx = j*stride_col_dst + i;
+            float value = dst[dst_idx] + sum[j0/nwarps];
+            if constexpr (type == GGML_TYPE_Q4_K) {
+                if (!isfinite(value)) value = 0.0f;
+            }
+            dst[dst_idx] = value;
         }
         return;
     }
@@ -4172,7 +4222,12 @@ static __global__ void mul_mat_q_stream_k_fixup(
             return;
         }
 
-        dst[ids_dst_shared[j]*stride_col_dst + i] += sum[j0/nwarps];
+        const int dst_idx = ids_dst_shared[j]*stride_col_dst + i;
+        float value = dst[dst_idx] + sum[j0/nwarps];
+        if constexpr (type == GGML_TYPE_Q4_K) {
+            if (!isfinite(value)) value = 0.0f;
+        }
+        dst[dst_idx] = value;
     }
 }
 
@@ -4187,6 +4242,10 @@ struct mmq_args {
     // ignored; soa_blocks = pair count (Q2_K) or block count (IQ2_XXS).
     // Trailing fields so existing aggregate initializers value-init them.
     const char * x_soa; int64_t soa_blocks;
+    // Optional caller-owned Stream-K fixup tile storage.  Production callers
+    // normally leave this null and use the CUDA pool; kernel-only A/B harnesses
+    // provide it so cudaMallocAsync/cudaFreeAsync are outside their events.
+    float * stream_k_fixup; size_t stream_k_fixup_elements;
 };
 
 template<ggml_type type>
@@ -4199,7 +4258,7 @@ static size_t mmq_get_nbytes_shared(const int mmq_x, const int mmq_y, const int
     return nbs_ids + nbs_x + GGML_PAD(nbs_y, nwarps*warp_size*sizeof(int));
 }
 
-template <ggml_type type, int mmq_x>
+template <ggml_type type, int mmq_x, bool grid_z_channels = false>
 static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) {
     const int id = ggml_cuda_get_device();
     const int cc = ggml_cuda_info().devices[id].cc;
@@ -4212,30 +4271,38 @@ static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & a
 
     const int nbytes_shared = mmq_get_nbytes_shared<type>(mmq_x, mmq_y, cc, warp_size, nwarps);
 
-    CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q<type, mmq_x, false>), nbytes_shared);
-    CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q<type, mmq_x,  true>), nbytes_shared);
+    CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q<type, mmq_x, false, grid_z_channels>), nbytes_shared);
+    CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q<type, mmq_x,  true, grid_z_channels>), nbytes_shared);
 
     const int nty  = (args.nrows_x   + mmq_y - 1) / mmq_y;
     const int ntx  = (args.ncols_max + mmq_x - 1) / mmq_x;
-    const int ntzw = args.nchannels_y * args.nsamples_y;
-    const dim3 block_nums_xy_tiling(nty, ntx, ntzw);
+    const int ntzw = grid_z_channels ? 1 : args.nchannels_y * args.nsamples_y;
+    const int grid_z = grid_z_channels ? args.nchannels_y : ntzw;
+    const dim3 block_nums_xy_tiling(nty, ntx, grid_z);
 
     GGML_ASSERT(args.nchannels_y % args.nchannels_x == 0);
     GGML_ASSERT(args.nsamples_y  % args.nsamples_x  == 0);
-    const int channel_ratio = args.nchannels_y / args.nchannels_x;
-    const int sample_ratio  = args.nsamples_y  / args.nsamples_x;
+    if constexpr (grid_z_channels) {
+        GGML_ASSERT(args.ids_dst == nullptr && args.expert_bounds == nullptr);
+        GGML_ASSERT(args.nchannels_x == args.nchannels_y);
+        GGML_ASSERT(args.nsamples_x == 1 && args.nsamples_y == 1);
+    }
+    const int channel_ratio = grid_z_channels ? 1 : args.nchannels_y / args.nchannels_x;
+    const int sample_ratio  = grid_z_channels ? 1 : args.nsamples_y  / args.nsamples_x;
+    const int logical_nchannels_y = grid_z_channels ? 1 : args.nchannels_y;
+    const int logical_nsamples_y = grid_z_channels ? 1 : args.nsamples_y;
 
     const uint3 blocks_per_ne00_fd = init_fastdiv_values(args.ncols_x / ggml_cuda_type_traits<type>::qk);
     const uint3 ntx_fd             = init_fastdiv_values(ntx);
-    const uint3 nchannels_y_fd     = init_fastdiv_values(args.nchannels_y);
-    const uint3 nsamples_y_fd      = init_fastdiv_values(args.nsamples_y);
+    const uint3 nchannels_y_fd     = init_fastdiv_values(logical_nchannels_y);
+    const uint3 nsamples_y_fd      = init_fastdiv_values(logical_nsamples_y);
     const uint3 channel_ratio_fd   = init_fastdiv_values(channel_ratio);
     const uint3 sample_ratio_fd    = init_fastdiv_values(sample_ratio);
 
     if (!args.use_stream_k) {
         if (args.nrows_x % mmq_y == 0) {
             constexpr bool need_check = false;
-            mul_mat_q<type, mmq_x, need_check><<<block_nums_xy_tiling, block_dims, nbytes_shared, stream>>>
+            mul_mat_q<type, mmq_x, need_check, grid_z_channels><<<block_nums_xy_tiling, block_dims, nbytes_shared, stream>>>
                 (args.x, args.y, args.ids_dst, args.expert_bounds, args.dst, nullptr,
                  blocks_per_ne00_fd, args.nrows_x, args.ncols_dst, args.stride_row_x, args.ncols_y, args.nrows_dst,
                  channel_ratio_fd, nchannels_y_fd, args.stride_channel_x, args.stride_channel_y, args.stride_channel_dst,
@@ -4243,7 +4310,7 @@ static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & a
                  ntx_fd, args.x_soa, args.soa_blocks);
         } else {
             constexpr bool need_check = true;
-            mul_mat_q<type, mmq_x, need_check><<<block_nums_xy_tiling, block_dims, nbytes_shared, stream>>>
+            mul_mat_q<type, mmq_x, need_check, grid_z_channels><<<block_nums_xy_tiling, block_dims, nbytes_shared, stream>>>
                 (args.x, args.y, args.ids_dst, args.expert_bounds, args.dst, nullptr,
                  blocks_per_ne00_fd, args.nrows_x, args.ncols_dst, args.stride_row_x, args.ncols_y, args.nrows_dst,
                  channel_ratio_fd, nchannels_y_fd, args.stride_channel_x, args.stride_channel_y, args.stride_channel_dst,
@@ -4258,7 +4325,8 @@ static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & a
     const int ntiles_dst = ntx * nty * ntzw;
     const int tiles_nwaves = (ntiles_dst + nsm - 1) / nsm;
     const int tiles_efficiency_percent = 100 * ntiles_dst / (nsm*tiles_nwaves);
-    const dim3 block_nums_stream_k(GGML_CUDA_CC_IS_NVIDIA(cc) && tiles_efficiency_percent >= 90 ? ntiles_dst : nsm, 1, 1);
+    const unsigned stream_k_grid_x = GGML_CUDA_CC_IS_NVIDIA(cc) && tiles_efficiency_percent >= 90 ? ntiles_dst : nsm;
+    const dim3 block_nums_stream_k(stream_k_grid_x, 1, grid_z_channels ? args.nchannels_y : 1);
 
     GGML_ASSERT(ntiles_dst * blocks_per_ne00_fd.z < (1 << 30)); // Assert that variable kbc will not overflow.
 
@@ -4266,20 +4334,28 @@ static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & a
 
     ggml_cuda_pool & pool = ctx.pool(id);
     ggml_cuda_pool_alloc<float> tmp_fixup(pool);
+    float * tmp_fixup_ptr = nullptr;
     if (fixup_needed) {
-        tmp_fixup.alloc(block_nums_stream_k.x * mmq_x*mmq_y);
-        CUDA_CHECK(cudaMemsetAsync(tmp_fixup.ptr, 0,
-                    (size_t)block_nums_stream_k.x * (size_t)mmq_x * (size_t)mmq_y * sizeof(float),
-                    stream));
+        const size_t fixup_elements =
+            (size_t)block_nums_stream_k.x *
+            (size_t)block_nums_stream_k.z * mmq_x * mmq_y;
+        if (args.stream_k_fixup != nullptr) {
+            GGML_ASSERT(args.stream_k_fixup_elements >= fixup_elements);
+            tmp_fixup_ptr = args.stream_k_fixup;
+        } else {
+            tmp_fixup_ptr = tmp_fixup.alloc(fixup_elements);
+        }
+        CUDA_CHECK(cudaMemsetAsync(
+            tmp_fixup_ptr, 0, fixup_elements * sizeof(float), stream));
     }
 
-    const dim3 block_nums_fixup(block_nums_stream_k.x, mmq_y/warp_size, 1);
+    const dim3 block_nums_fixup(block_nums_stream_k.x, mmq_y/warp_size, block_nums_stream_k.z);
     const dim3 block_dims_fixup(block_dims.x, block_dims.y/2, block_dims.z);
 
     if (args.nrows_x % mmq_y == 0) {
         constexpr bool need_check = false;
-        mul_mat_q<type, mmq_x, need_check><<<block_nums_stream_k, block_dims, nbytes_shared, stream>>>
-            (args.x, args.y, args.ids_dst, args.expert_bounds, args.dst, tmp_fixup.ptr,
+        mul_mat_q<type, mmq_x, need_check, grid_z_channels><<<block_nums_stream_k, block_dims, nbytes_shared, stream>>>
+            (args.x, args.y, args.ids_dst, args.expert_bounds, args.dst, tmp_fixup_ptr,
              blocks_per_ne00_fd, args.nrows_x, args.ncols_dst, args.stride_row_x, args.ncols_y, args.nrows_dst,
              channel_ratio_fd, nchannels_y_fd, args.stride_channel_x, args.stride_channel_y, args.stride_channel_dst,
              sample_ratio_fd, nsamples_y_fd, args.stride_sample_x, args.stride_sample_y, args.stride_sample_dst,
@@ -4290,14 +4366,14 @@ static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & a
         }
 
         CUDA_CHECK(cudaGetLastError());
-        mul_mat_q_stream_k_fixup<type, mmq_x, need_check><<<block_nums_fixup, block_dims_fixup, 0, stream>>>
-            (args.ids_dst, args.expert_bounds, args.dst, tmp_fixup.ptr, blocks_per_ne00_fd, args.nrows_x, args.ncols_dst,
+        mul_mat_q_stream_k_fixup<type, mmq_x, need_check, grid_z_channels><<<block_nums_fixup, block_dims_fixup, 0, stream>>>
+            (args.ids_dst, args.expert_bounds, args.dst, tmp_fixup_ptr, blocks_per_ne00_fd, args.nrows_x, args.ncols_dst,
              args.nrows_dst, nchannels_y_fd, args.stride_channel_dst, nsamples_y_fd, args.stride_sample_dst,
              ntx_fd);
     } else {
         constexpr bool need_check = true;
-        mul_mat_q<type, mmq_x, need_check><<<block_nums_stream_k, block_dims, nbytes_shared, stream>>>
-            (args.x, args.y, args.ids_dst, args.expert_bounds, args.dst, tmp_fixup.ptr,
+        mul_mat_q<type, mmq_x, need_check, grid_z_channels><<<block_nums_stream_k, block_dims, nbytes_shared, stream>>>
+            (args.x, args.y, args.ids_dst, args.expert_bounds, args.dst, tmp_fixup_ptr,
              blocks_per_ne00_fd, args.nrows_x, args.ncols_dst, args.stride_row_x, args.ncols_y, args.nrows_dst,
              channel_ratio_fd, nchannels_y_fd, args.stride_channel_x, args.stride_channel_y, args.stride_channel_dst,
              sample_ratio_fd, nsamples_y_fd, args.stride_sample_x, args.stride_sample_y, args.stride_sample_dst,
@@ -4308,15 +4384,15 @@ static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & a
         }
 
         CUDA_CHECK(cudaGetLastError());
-        mul_mat_q_stream_k_fixup<type, mmq_x, need_check><<<block_nums_fixup, block_dims_fixup, 0, stream>>>
-            (args.ids_dst, args.expert_bounds, args.dst, tmp_fixup.ptr, blocks_per_ne00_fd, args.nrows_x, args.ncols_dst,
+        mul_mat_q_stream_k_fixup<type, mmq_x, need_check, grid_z_channels><<<block_nums_fixup, block_dims_fixup, 0, stream>>>
+            (args.ids_dst, args.expert_bounds, args.dst, tmp_fixup_ptr, blocks_per_ne00_fd, args.nrows_x, args.ncols_dst,
              args.nrows_dst, nchannels_y_fd, args.stride_channel_dst, nsamples_y_fd, args.stride_sample_dst,
              ntx_fd);
     }
 }
 
-template <ggml_type type>
-void mul_mat_q_case(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) {
+template <ggml_type type, bool grid_z_channels>
+static void mul_mat_q_case_impl(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) {
     const int    id     = ggml_cuda_get_device();
     const int    cc     = ggml_cuda_info().devices[id].cc;
     const size_t smpbo  = ggml_cuda_info().devices[id].smpbo;
@@ -4346,52 +4422,52 @@ void mul_mat_q_case(ggml_backend_cuda_context & ctx, const mmq_args & args, cuda
 
     switch (mmq_x_best) {
         case   8:
-            launch_mul_mat_q<type,   8>(ctx, args, stream);
+            launch_mul_mat_q<type,   8, grid_z_channels>(ctx, args, stream);
             break;
         case  16:
-            launch_mul_mat_q<type,  16>(ctx, args, stream);
+            launch_mul_mat_q<type,  16, grid_z_channels>(ctx, args, stream);
             break;
         case  24:
-            launch_mul_mat_q<type,  24>(ctx, args, stream);
+            launch_mul_mat_q<type,  24, grid_z_channels>(ctx, args, stream);
             break;
         case  32:
-            launch_mul_mat_q<type,  32>(ctx, args, stream);
+            launch_mul_mat_q<type,  32, grid_z_channels>(ctx, args, stream);
             break;
         case  40:
-            launch_mul_mat_q<type,  40>(ctx, args, stream);
+            launch_mul_mat_q<type,  40, grid_z_channels>(ctx, args, stream);
             break;
         case  48:
-            launch_mul_mat_q<type,  48>(ctx, args, stream);
+            launch_mul_mat_q<type,  48, grid_z_channels>(ctx, args, stream);
             break;
         case  56:
-            launch_mul_mat_q<type,  56>(ctx, args, stream);
+            launch_mul_mat_q<type,  56, grid_z_channels>(ctx, args, stream);
             break;
         case  64:
-            launch_mul_mat_q<type,  64>(ctx, args, stream);
+            launch_mul_mat_q<type,  64, grid_z_channels>(ctx, args, stream);
             break;
         case  72:
-            launch_mul_mat_q<type,  72>(ctx, args, stream);
+            launch_mul_mat_q<type,  72, grid_z_channels>(ctx, args, stream);
             break;
         case  80:
-            launch_mul_mat_q<type,  80>(ctx, args, stream);
+            launch_mul_mat_q<type,  80, grid_z_channels>(ctx, args, stream);
             break;
         case  88:
-            launch_mul_mat_q<type,  88>(ctx, args, stream);
+            launch_mul_mat_q<type,  88, grid_z_channels>(ctx, args, stream);
             break;
         case  96:
-            launch_mul_mat_q<type,  96>(ctx, args, stream);
+            launch_mul_mat_q<type,  96, grid_z_channels>(ctx, args, stream);
             break;
         case 104:
-            launch_mul_mat_q<type, 104>(ctx, args, stream);
+            launch_mul_mat_q<type, 104, grid_z_channels>(ctx, args, stream);
             break;
         case 112:
-            launch_mul_mat_q<type, 112>(ctx, args, stream);
+            launch_mul_mat_q<type, 112, grid_z_channels>(ctx, args, stream);
             break;
         case 120:
-            launch_mul_mat_q<type, 120>(ctx, args, stream);
+            launch_mul_mat_q<type, 120, grid_z_channels>(ctx, args, stream);
             break;
         case 128:
-            launch_mul_mat_q<type, 128>(ctx, args, stream);
+            launch_mul_mat_q<type, 128, grid_z_channels>(ctx, args, stream);
             break;
         default:
             fprintf(stderr, "mmq_x_best=%d\n", mmq_x_best);
@@ -4400,6 +4476,20 @@ void mul_mat_q_case(ggml_backend_cuda_context & ctx, const mmq_args & args, cuda
     }
 }
 
+template <ggml_type type>
+void mul_mat_q_case(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) {
+    mul_mat_q_case_impl<type, false>(ctx, args, stream);
+}
+
+/* Dense-only grouped entry used by ds4's Q4 attention output-A path.  Each
+ * grid.z slice is reduction-isolated, so the result remains bit-identical to
+ * invoking mul_mat_q_case once per channel on the same stream. */
+template <ggml_type type>
+void mul_mat_q_case_grouped_channels(
+        ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) {
+    mul_mat_q_case_impl<type, true>(ctx, args, stream);
+}
+
 #define DECL_MMQ_CASE(type)                                                        \
     template void mul_mat_q_case<type>(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) \
 
diff --git a/cuda/mmq/quantize.cu b/cuda/mmq/quantize.cu
index 52f664719a..a8066c4845 100644
--- a/cuda/mmq/quantize.cu
+++ b/cuda/mmq/quantize.cu
@@ -366,6 +366,77 @@ static __global__ void quantize_mmq_q8_1(
     }
 }
 
+#if !defined(GGML_USE_HIP)
+// Q4 grouped attention-A has one hot, fixed source geometry:
+// [token][group=8][K=4096].  The generic MMQ quantizer launches four-warp
+// CTAs, each producing four 128-value block_q8_1_mmq records.  This kernel
+// keeps the exact DS4 arithmetic and byte layout, but uses eight warps and
+// lets each warp produce two consecutive records.  One CTA therefore covers
+// 2048 source values and the launch uses four times fewer CTAs, without the
+// 16-warp geometry that regressed on GB10.
+__launch_bounds__(8*WARP_SIZE)
+static __global__ void quantize_mmq_q8_1_q4_grouped_k4096_g8x2(
+        const float * __restrict__ x, void * __restrict__ vy, const int ne1) {
+    constexpr int k_groups = 8;
+    constexpr int k_values = 4096;
+    constexpr int k_q8_blocks = k_values / (4*QK8_1);
+    constexpr int k_warps = 8;
+    constexpr int k_blocks_per_warp = 2;
+    constexpr int k_blocks_per_cta = k_warps * k_blocks_per_warp;
+    static_assert(WARP_SIZE == 32, "CUDA Q8_1 quantizer requires 32-lane warps");
+    static_assert(k_q8_blocks == 32, "unexpected grouped Q8_1 block count");
+    static_assert(2*k_blocks_per_cta == k_q8_blocks,
+                  "two CTAs must cover one grouped activation row");
+
+    const int token = (int)blockIdx.x;
+    const int group = (int)blockIdx.z;
+    const int warp = (int)threadIdx.x / WARP_SIZE;
+    const int lane = (int)threadIdx.x % WARP_SIZE;
+    const uint32_t input_base =
+        ((uint32_t)token * k_groups + (uint32_t)group) * k_values;
+    const float4 * __restrict__ x4 = (const float4 *)x;
+    block_q8_1_mmq * __restrict__ y = (block_q8_1_mmq *)vy;
+
+#pragma unroll
+    for (int j = 0; j < k_blocks_per_warp; ++j) {
+        const int q8_block =
+            (int)blockIdx.y * k_blocks_per_cta + j * k_warps + warp;
+        const uint32_t input = input_base +
+            (uint32_t)q8_block * (4*QK8_1) + (uint32_t)lane * 4u;
+        const float4 xi = x4[input / 4u];
+
+        float amax = fabsf(xi.x);
+        amax = fmaxf(amax, fabsf(xi.y));
+        amax = fmaxf(amax, fabsf(xi.z));
+        amax = fmaxf(amax, fabsf(xi.w));
+#pragma unroll
+        for (int offset = 4; offset > 0; offset >>= 1) {
+            amax = fmaxf(
+                amax, __shfl_xor_sync(0xFFFFFFFF, amax, offset, WARP_SIZE));
+        }
+
+        float sum = xi.x + xi.y + xi.z + xi.w;
+#pragma unroll
+        for (int offset = 4; offset > 0; offset >>= 1) {
+            sum += __shfl_xor_sync(0xFFFFFFFF, sum, offset, WARP_SIZE);
+        }
+
+        const float d_inv = 127.0f / amax;
+        char4 q;
+        q.x = roundf(xi.x*d_inv);
+        q.y = roundf(xi.y*d_inv);
+        q.z = roundf(xi.z*d_inv);
+        q.w = roundf(xi.w*d_inv);
+
+        const int ib = (group*k_q8_blocks + q8_block)*ne1 + token;
+        ((char4 *)y[ib].qs)[lane] = q;
+        if ((lane % 8) == 0) {
+            y[ib].ds4[lane / 8] = make_half2(1.0f / d_inv, sum);
+        }
+    }
+}
+#endif
+
 void quantize_row_q8_1_cuda(
         const float * x, const int32_t * ids, void * vy, const ggml_type type_src0,
         const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03,
@@ -412,6 +483,22 @@ void quantize_mmq_q8_1_cuda(
     }
 }
 
+#if !defined(GGML_USE_HIP)
+void quantize_mmq_q8_1_q4_grouped_k4096_g8x2_cuda(
+        const float * x, void * vy, int ne1, cudaStream_t stream) {
+    GGML_ASSERT(x);
+    GGML_ASSERT(vy);
+    GGML_ASSERT(ne1 > 0 && ne1 <= INT32_MAX / (8*4096));
+
+    constexpr int k_warps = 8;
+    constexpr int k_grid_y = 2;
+    const dim3 num_blocks(ne1, k_grid_y, 8);
+    const dim3 block_size(k_warps*WARP_SIZE, 1, 1);
+    quantize_mmq_q8_1_q4_grouped_k4096_g8x2<<<
+        num_blocks, block_size, 0, stream>>>(x, vy, ne1);
+}
+#endif
+
 void quantize_mmq_fp4_cuda(
         const float * x, const int32_t * ids, void * vy, const ggml_type type_src0,
         const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03,
diff --git a/cuda/mmq/quantize.cuh b/cuda/mmq/quantize.cuh
index 768a3ae6de..ecc1ca6dfe 100644
--- a/cuda/mmq/quantize.cuh
+++ b/cuda/mmq/quantize.cuh
@@ -26,6 +26,13 @@ void quantize_mmq_q8_1_cuda(
         ggml_type type_src0, int64_t ne00, int64_t s01, int64_t s02, int64_t s03,
         int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3, cudaStream_t stream);
 
+#if !defined(GGML_USE_HIP)
+// Fixed-layout Q4 attention-A prefill producer.  X is [N][8][4096] and the
+// output is canonical block_q8_1_mmq DS4, group-major [8][32][N].
+void quantize_mmq_q8_1_q4_grouped_k4096_g8x2_cuda(
+        const float * x, void * vy, int ne1, cudaStream_t stream);
+#endif
+
 void quantize_mmq_fp4_cuda(const float *   x,
                              const int32_t * ids,
                              void *          vy,
diff --git a/cuda/mmq/test/d2r_stubs.cu b/cuda/mmq/test/d2r_stubs.cu
index e9249e7909..81fb1a9e3b 100644
--- a/cuda/mmq/test/d2r_stubs.cu
+++ b/cuda/mmq/test/d2r_stubs.cu
@@ -1,10 +1,12 @@
 #include "ds4_mmq_d2r.cuh"
 
 extern "C" int ds4_cuda_q8_fold_take_q81(
-        const void *src, uint64_t in_dim, const void **q81) {
+        const void *src, uint64_t in_dim, cudaStream_t stream,
+        const void **q81) {
     (void)src;
     (void)in_dim;
-    (void)q81;
+    (void)stream;
+    if (q81) *q81 = nullptr;
     return 0;
 }
 
diff --git a/cuda/mmq/test/proto_gemm_dense_q8_d2r.cu b/cuda/mmq/test/proto_gemm_dense_q8_d2r.cu
index 78496c5a13..8dc8d42758 100644
--- a/cuda/mmq/test/proto_gemm_dense_q8_d2r.cu
+++ b/cuda/mmq/test/proto_gemm_dense_q8_d2r.cu
@@ -42,13 +42,6 @@
 #include "ds4_mmq.h"
 #include "quantize.cuh"
 
-extern "C" int ds4_cuda_q8_fold_take_q81(const void *src, uint64_t in_dim, void *out) {
-    (void)src;
-    (void)in_dim;
-    (void)out;
-    return 0;
-}
-
 #include <cuda_fp16.h>
 #include <cuda_runtime.h>
 
diff --git a/cuda/mmq/test/test_mmq_parity.cu b/cuda/mmq/test/test_mmq_parity.cu
index 29edf8275c..cdf4aa341a 100644
--- a/cuda/mmq/test/test_mmq_parity.cu
+++ b/cuda/mmq/test/test_mmq_parity.cu
@@ -20,6 +20,7 @@
 //        -o test_mmq_parity
 
 #include "ds4_mmq.h"
+#include "ds4_mmq_q4_16warp.cuh"
 #include "iq2_host_tables.h"
 
 // Pull in the block_* struct definitions.  We use the CUDA decl/impl mode
@@ -48,18 +49,98 @@
 #endif
 
 #include <algorithm>
+#include <cerrno>
 #include <cmath>
 #include <cstdint>
 #include <cstdio>
 #include <cstdlib>
 #include <cstring>
 #include <random>
+#include <string>
 #include <vector>
 
 namespace {
 
 constexpr int QK_K_LOCAL = 256;
 
+class scoped_env_override {
+public:
+    explicit scoped_env_override(const char *name) : name_(name) {
+        const char *value = std::getenv(name_);
+        if (value) {
+            had_original_ = true;
+            original_value_ = value;
+        }
+    }
+
+    ~scoped_env_override() {
+        (void)restore();
+    }
+
+    bool set(const char *value) {
+        if (setenv(name_, value, 1) != 0) {
+            const int saved_errno = errno;
+            fprintf(stderr, "setenv(%s=%s) failed: %s\n",
+                    name_, value, std::strerror(saved_errno));
+            return false;
+        }
+        active_ = true;
+        return true;
+    }
+
+    bool restore() {
+        if (!active_) return true;
+        const int rc = had_original_
+            ? setenv(name_, original_value_.c_str(), 1)
+            : unsetenv(name_);
+        if (rc != 0) {
+            const int saved_errno = errno;
+            fprintf(stderr, "restoring %s failed: %s\n",
+                    name_, std::strerror(saved_errno));
+            return false;
+        }
+        active_ = false;
+        return true;
+    }
+
+    scoped_env_override(const scoped_env_override &) = delete;
+    scoped_env_override &operator=(const scoped_env_override &) = delete;
+
+private:
+    const char *name_;
+    std::string original_value_;
+    bool had_original_ = false;
+    bool active_ = false;
+};
+
+cudaError_t enqueue_scratch_guard_copy(
+        const void *storage, size_t payload_bytes, size_t guard_bytes,
+        uint8_t *host_guards, cudaStream_t stream) {
+    if (!storage || !host_guards || guard_bytes == 0) {
+        // cudaError_t is an integer-compatible type in both the CUDA runtime and
+        // the host-only syntax-check stub; avoid depending on a stubbed enum.
+        return static_cast<cudaError_t>(1);
+    }
+    const auto *bytes = static_cast<const uint8_t *>(storage);
+    cudaError_t err = cudaMemcpyAsync(
+        host_guards, bytes, guard_bytes, cudaMemcpyDeviceToHost, stream);
+    if (err == cudaSuccess) {
+        err = cudaMemcpyAsync(
+            host_guards + guard_bytes, bytes + guard_bytes + payload_bytes,
+            guard_bytes, cudaMemcpyDeviceToHost, stream);
+    }
+    return err;
+}
+
+size_t scratch_guard_mismatches(
+        const std::vector<uint8_t> &guards, uint8_t expected) {
+    size_t bad = 0;
+    for (uint8_t value : guards) {
+        if (value != expected) bad++;
+    }
+    return bad;
+}
+
 // --------------------------------------------------------------------------
 // Half-precision conversion (standalone, no CUDA host fp16 needed).
 // --------------------------------------------------------------------------
@@ -504,6 +585,1176 @@ bool run_q4_K(int M, int N, int K, uint32_t seed, float abs_scale = 0.20f) {
     return ok;
 }
 
+// Resident-kernel oracle for the opt-in m128n128/16-warp Q4_K prefill
+// candidate. The raw canonical/candidate arms consume the same caller-owned
+// DS4 Q8_1 activation. The canonical arm uses the production stream-K policy,
+// including fixup when selected. The production API can be checked as a third,
+// independently guarded output without folding its quantizer into the raw
+// kernel comparison.
+bool run_q4_K_dense_16warp_parity(
+        int M, int N, int K, int nsm, uint32_t seed,
+        bool check_public_dense, bool check_rejection) {
+    fprintf(stderr,
+            "=== Q4_K/DENSE_16WARP  M=%d N=%d K=%d seed=%u%s ===\n",
+            M, N, K, seed, check_public_dense ? " production" : "");
+
+    if (M <= 0 || N <= 0 || K <= 0 || nsm <= 0 ||
+        K % QK_K_LOCAL != 0 ||
+        (size_t)M > SIZE_MAX / (size_t)N / sizeof(float)) {
+        fprintf(stderr, "invalid 16-warp parity shape\n\n");
+        return false;
+    }
+
+    std::mt19937 rng(seed);
+    std::normal_distribution<float> nd(0.0f, 1.0f);
+    const int blocks_per_row = K / QK_K_LOCAL;
+    std::vector<block_q4_K> W((size_t)M * blocks_per_row);
+    for (auto &blk : W) generate_random_block_q4_K(&blk, rng);
+    std::vector<float> X((size_t)N * K);
+    for (float &v : X) v = nd(rng);
+
+    const size_t q8_bytes = ds4_mmq_q4_K_q8_1_scratch_bytes(N, K);
+    if (q8_bytes == 0) {
+        fprintf(stderr, "Q4_K 16-warp scratch-size query rejected shape\n\n");
+        return false;
+    }
+
+    constexpr size_t guard_floats = 64;
+    constexpr uint8_t ref_guard_byte = 0xa5;
+    constexpr uint8_t prod_guard_byte = 0xc3;
+    constexpr uint8_t got_guard_byte = 0x5a;
+    constexpr uint8_t reject_byte = 0x3c;
+    constexpr uint8_t scratch_guard_byte = 0x7d;
+    const size_t output_count = (size_t)M * N;
+    const size_t output_bytes = output_count * sizeof(float);
+    const size_t guard_bytes = guard_floats * sizeof(float);
+    const size_t guarded_bytes = output_bytes + 2u * guard_bytes;
+    const size_t scratch_bytes =
+        ds4_mmq_q4_K_dense_16warp_streamk_scratch_bytes(M, N, nsm);
+    if (scratch_bytes > SIZE_MAX - 2u * guard_bytes) {
+        fprintf(stderr, "Q4_K 16-warp scratch guard size overflow\n\n");
+        return false;
+    }
+    const size_t scratch_guarded_bytes = scratch_bytes + 2u * guard_bytes;
+
+    cudaStream_t stream = nullptr;
+    void *dW = nullptr;
+    float *dX = nullptr;
+    void *dQ8 = nullptr;
+    float *dRefStorage = nullptr;
+    float *dProdStorage = nullptr;
+    float *dGotStorage = nullptr;
+    void *dScratchStorage = nullptr;
+    const bool allocated = cudaStreamCreate(&stream) == cudaSuccess &&
+        cudaMalloc(&dW, W.size() * sizeof(block_q4_K)) == cudaSuccess &&
+        cudaMalloc(&dX, X.size() * sizeof(float)) == cudaSuccess &&
+        cudaMalloc(&dQ8, q8_bytes) == cudaSuccess &&
+        cudaMalloc(&dRefStorage, guarded_bytes) == cudaSuccess &&
+        (!check_public_dense ||
+         cudaMalloc(&dProdStorage, guarded_bytes) == cudaSuccess) &&
+        cudaMalloc(&dGotStorage, guarded_bytes) == cudaSuccess &&
+        cudaMalloc(&dScratchStorage, scratch_guarded_bytes) == cudaSuccess;
+    const auto cleanup = [&]() {
+        if (dScratchStorage) cudaFree(dScratchStorage);
+        if (dGotStorage) cudaFree(dGotStorage);
+        if (dProdStorage) cudaFree(dProdStorage);
+        if (dRefStorage) cudaFree(dRefStorage);
+        if (dQ8) cudaFree(dQ8);
+        if (dX) cudaFree(dX);
+        if (dW) cudaFree(dW);
+        if (stream) cudaStreamDestroy(stream);
+    };
+    if (!allocated) {
+        fprintf(stderr, "Q4_K 16-warp parity allocation failed: %s\n\n",
+                cudaGetErrorString(cudaGetLastError()));
+        cleanup();
+        return false;
+    }
+
+    float *const dRef = dRefStorage + guard_floats;
+    float *const dProd = check_public_dense
+        ? dProdStorage + guard_floats : nullptr;
+    float *const dGot = dGotStorage + guard_floats;
+    void *const dScratch =
+        static_cast<uint8_t *>(dScratchStorage) + guard_bytes;
+    cudaError_t enqueue_err = cudaMemcpyAsync(
+        dW, W.data(), W.size() * sizeof(block_q4_K),
+        cudaMemcpyHostToDevice, stream);
+    if (enqueue_err == cudaSuccess) {
+        enqueue_err = cudaMemcpyAsync(
+            dX, X.data(), X.size() * sizeof(float),
+            cudaMemcpyHostToDevice, stream);
+    }
+    if (enqueue_err == cudaSuccess) {
+        enqueue_err = cudaMemsetAsync(
+            dRefStorage, ref_guard_byte, guarded_bytes, stream);
+    }
+    if (enqueue_err == cudaSuccess && check_public_dense) {
+        enqueue_err = cudaMemsetAsync(
+            dProdStorage, prod_guard_byte, guarded_bytes, stream);
+    }
+    if (enqueue_err == cudaSuccess) {
+        enqueue_err = cudaMemsetAsync(
+            dGotStorage, got_guard_byte, guarded_bytes, stream);
+    }
+    if (enqueue_err == cudaSuccess) {
+        enqueue_err = cudaMemsetAsync(
+            dScratchStorage, scratch_guard_byte, scratch_guarded_bytes,
+            stream);
+    }
+
+    const int rc_quant = enqueue_err == cudaSuccess
+        ? ds4_mmq_q4_K_quantize_q8_1_for_test(
+              dX, dQ8, q8_bytes, N, K, stream)
+        : -100;
+    const int rc_ref = rc_quant == 0
+        ? ds4_mmq_q4_K_dense_preq_reference_for_test(
+              dW, dQ8, q8_bytes, dRef, M, N, K,
+              /*use_stream_k=*/1,
+              dScratch, scratch_bytes,
+              stream)
+        : -100;
+    std::vector<uint8_t> ref_scratch_guards(2u * guard_bytes);
+    if (enqueue_err == cudaSuccess && rc_ref == 0) {
+        enqueue_err = enqueue_scratch_guard_copy(
+            dScratchStorage, scratch_bytes, guard_bytes,
+            ref_scratch_guards.data(), stream);
+    }
+    const int rc_prod = rc_ref != 0 || enqueue_err != cudaSuccess
+        ? -100 : check_public_dense
+        ? ds4_mmq_q4_K_dense(
+              dW, dX, dProd, M, N, K, stream)
+        : 0;
+    const int rc_got = rc_prod == 0
+        ? ds4_mmq_q4_K_dense_16warp_streamk_enqueue(
+              dW, dQ8, dGot, dScratch, scratch_bytes,
+              M, N, K, nsm, stream)
+        : -100;
+    std::vector<uint8_t> got_scratch_guards(2u * guard_bytes);
+    if (enqueue_err == cudaSuccess && rc_got == 0) {
+        enqueue_err = enqueue_scratch_guard_copy(
+            dScratchStorage, scratch_bytes, guard_bytes,
+            got_scratch_guards.data(), stream);
+    }
+
+    std::vector<uint8_t> ref_guarded(guarded_bytes);
+    std::vector<uint8_t> prod_guarded(
+        check_public_dense ? guarded_bytes : 0u);
+    std::vector<uint8_t> got_guarded(guarded_bytes);
+    if (rc_got == 0) {
+        enqueue_err = cudaMemcpyAsync(
+            ref_guarded.data(), dRefStorage, guarded_bytes,
+            cudaMemcpyDeviceToHost, stream);
+    }
+    if (enqueue_err == cudaSuccess && rc_got == 0 && check_public_dense) {
+        enqueue_err = cudaMemcpyAsync(
+            prod_guarded.data(), dProdStorage, guarded_bytes,
+            cudaMemcpyDeviceToHost, stream);
+    }
+    if (enqueue_err == cudaSuccess && rc_got == 0) {
+        enqueue_err = cudaMemcpyAsync(
+            got_guarded.data(), dGotStorage, guarded_bytes,
+            cudaMemcpyDeviceToHost, stream);
+    }
+    const cudaError_t sync_err = cudaStreamSynchronize(stream);
+
+    size_t ref_got_mismatches = 0;
+    size_t ref_prod_mismatches = 0;
+    size_t prod_got_mismatches = 0;
+    size_t nonfinite_ref = 0;
+    size_t nonfinite_prod = 0;
+    size_t nonfinite_got = 0;
+    uint32_t first_ref_bits = 0;
+    uint32_t first_got_bits = 0;
+    uint32_t first_prod_bits = 0;
+    size_t first_ref_got_bad = SIZE_MAX;
+    size_t first_ref_prod_bad = SIZE_MAX;
+    if (enqueue_err == cudaSuccess && sync_err == cudaSuccess && rc_got == 0) {
+        for (size_t i = 0; i < output_count; ++i) {
+            const uint8_t *const ref_bits_ptr =
+                ref_guarded.data() + guard_bytes + i * sizeof(float);
+            const uint8_t *const got_bits_ptr =
+                got_guarded.data() + guard_bytes + i * sizeof(float);
+            const uint8_t *const prod_bits_ptr = check_public_dense
+                ? prod_guarded.data() + guard_bytes + i * sizeof(float)
+                : nullptr;
+            float ref_value = 0.0f;
+            float prod_value = 0.0f;
+            float got_value = 0.0f;
+            std::memcpy(&ref_value, ref_bits_ptr, sizeof(ref_value));
+            std::memcpy(&got_value, got_bits_ptr, sizeof(got_value));
+            if (check_public_dense) {
+                std::memcpy(&prod_value, prod_bits_ptr, sizeof(prod_value));
+            }
+            if (!std::isfinite(ref_value)) nonfinite_ref++;
+            if (check_public_dense && !std::isfinite(prod_value)) {
+                nonfinite_prod++;
+            }
+            if (!std::isfinite(got_value)) nonfinite_got++;
+            if (std::memcmp(ref_bits_ptr, got_bits_ptr, sizeof(float)) != 0) {
+                if (first_ref_got_bad == SIZE_MAX) {
+                    first_ref_got_bad = i;
+                    std::memcpy(&first_ref_bits, ref_bits_ptr,
+                                sizeof(first_ref_bits));
+                    std::memcpy(&first_got_bits, got_bits_ptr,
+                                sizeof(first_got_bits));
+                }
+                ref_got_mismatches++;
+            }
+            if (check_public_dense &&
+                std::memcmp(ref_bits_ptr, prod_bits_ptr, sizeof(float)) != 0) {
+                if (first_ref_prod_bad == SIZE_MAX) {
+                    first_ref_prod_bad = i;
+                    std::memcpy(&first_prod_bits, prod_bits_ptr,
+                                sizeof(first_prod_bits));
+                }
+                ref_prod_mismatches++;
+            }
+            if (check_public_dense &&
+                std::memcmp(prod_bits_ptr, got_bits_ptr, sizeof(float)) != 0) {
+                prod_got_mismatches++;
+            }
+        }
+    }
+
+    const auto guard_mismatches = [=](
+            const std::vector<uint8_t> &bytes, uint8_t expected) {
+        size_t bad = 0;
+        for (size_t i = 0; i < guard_bytes; ++i) {
+            if (bytes[i] != expected) bad++;
+        }
+        for (size_t i = guard_bytes + output_bytes; i < bytes.size(); ++i) {
+            if (bytes[i] != expected) bad++;
+        }
+        return bad;
+    };
+    const size_t ref_canary = guard_mismatches(ref_guarded, ref_guard_byte);
+    const size_t prod_canary = check_public_dense
+        ? guard_mismatches(prod_guarded, prod_guard_byte) : 0;
+    const size_t got_canary = guard_mismatches(got_guarded, got_guard_byte);
+    const size_t ref_scratch_canary =
+        scratch_guard_mismatches(ref_scratch_guards, scratch_guard_byte);
+    const size_t got_scratch_canary =
+        scratch_guard_mismatches(got_scratch_guards, scratch_guard_byte);
+
+    int rc_reject = DS4_MMQ_NOT_APPLICABLE;
+    size_t reject_writes = 0;
+    cudaError_t reject_sync = cudaSuccess;
+    if (check_rejection && enqueue_err == cudaSuccess &&
+        sync_err == cudaSuccess) {
+        cudaError_t reject_err = cudaMemsetAsync(
+            dGotStorage, reject_byte, guarded_bytes, stream);
+        rc_reject = reject_err == cudaSuccess
+            ? ds4_mmq_q4_K_dense_preq_16warp_for_test(
+                  dW, dQ8, q8_bytes, dGot, M, /*N=*/511, K, stream)
+            : -100;
+        std::vector<uint8_t> rejected(guarded_bytes);
+        if (reject_err == cudaSuccess) {
+            reject_err = cudaMemcpyAsync(
+                rejected.data(), dGotStorage, guarded_bytes,
+                cudaMemcpyDeviceToHost, stream);
+        }
+        reject_sync = cudaStreamSynchronize(stream);
+        if (reject_err == cudaSuccess && reject_sync == cudaSuccess) {
+            for (uint8_t byte : rejected) {
+                if (byte != reject_byte) reject_writes++;
+            }
+        } else {
+            reject_writes = SIZE_MAX;
+        }
+    }
+
+    const bool rejection_ok = !check_rejection ||
+        (rc_reject == DS4_MMQ_NOT_APPLICABLE &&
+         reject_sync == cudaSuccess && reject_writes == 0);
+    const bool production_ok = !check_public_dense ||
+        (ref_prod_mismatches == 0 && prod_got_mismatches == 0 &&
+         nonfinite_prod == 0 && prod_canary == 0);
+    const bool ok = rc_quant == 0 && rc_ref == 0 && rc_prod == 0 &&
+        rc_got == 0 && enqueue_err == cudaSuccess &&
+        sync_err == cudaSuccess && ref_got_mismatches == 0 &&
+        nonfinite_ref == 0 && nonfinite_got == 0 && ref_canary == 0 &&
+        got_canary == 0 && ref_scratch_canary == 0 &&
+        got_scratch_canary == 0 && production_ok && rejection_ok;
+    fprintf(stderr,
+            "quant/ref/production/16w=%d/%d/%d/%d enqueue=%s sync=%s "
+            "bits(ref-16w/ref-production/production-16w)=%zu/%zu/%zu "
+            "nonfinite=%zu/%zu/%zu canary=%zu/%zu/%zu "
+            "scratch_canary(ref/16w)=%zu/%zu reject=%d/%zu: %s\n",
+            rc_quant, rc_ref, rc_prod, rc_got,
+            cudaGetErrorString(enqueue_err), cudaGetErrorString(sync_err),
+            ref_got_mismatches, ref_prod_mismatches, prod_got_mismatches,
+            nonfinite_ref, nonfinite_prod, nonfinite_got,
+            ref_canary, prod_canary, got_canary,
+            ref_scratch_canary, got_scratch_canary,
+            rc_reject, reject_writes,
+            ok ? "PASS" : "FAIL");
+    if (first_ref_got_bad != SIZE_MAX) {
+        fprintf(stderr,
+                "first ref/16w mismatch at output[%zu]: "
+                "ref=0x%08x got=0x%08x\n",
+                first_ref_got_bad, first_ref_bits, first_got_bits);
+    }
+    if (first_ref_prod_bad != SIZE_MAX) {
+        uint32_t ref_bits = 0;
+        const uint8_t *const ptr = ref_guarded.data() + guard_bytes +
+            first_ref_prod_bad * sizeof(float);
+        std::memcpy(&ref_bits, ptr, sizeof(ref_bits));
+        fprintf(stderr,
+                "first ref/production mismatch at output[%zu]: "
+                "ref=0x%08x production=0x%08x\n",
+                first_ref_prod_bad, ref_bits, first_prod_bits);
+    }
+    fputc('\n', stderr);
+    cleanup();
+    return ok;
+}
+
+// Kernel-only Q-A/KV pair oracle. Quantize X exactly once, then compare two
+// canonical stream-K launches against two 16-warp stream-K launches over the
+// same immutable Q8_1 DS4 buffer and the real required production pair API.
+// Keeping every output independently guarded catches a bad M0/M1 stride or a
+// cross-leg overwrite as well as a numerical mismatch.
+bool run_q4_K_dense_pair_16warp_parity(
+        int M0, int M1, int N, int K, int nsm, uint32_t seed) {
+    fprintf(stderr,
+            "=== Q4_K/DENSE_PAIR_16WARP  M0=%d M1=%d N=%d K=%d seed=%u ===\n",
+            M0, M1, N, K, seed);
+
+    if (M0 <= 0 || M1 <= 0 || N <= 0 || K <= 0 || nsm <= 0 ||
+        K % QK_K_LOCAL != 0 ||
+        (size_t)M0 > SIZE_MAX / (size_t)N / sizeof(float) ||
+        (size_t)M1 > SIZE_MAX / (size_t)N / sizeof(float)) {
+        fprintf(stderr, "invalid 16-warp pair parity shape\n\n");
+        return false;
+    }
+
+    std::mt19937 rng(seed);
+    std::normal_distribution<float> nd(0.0f, 1.0f);
+    const int blocks_per_row = K / QK_K_LOCAL;
+    std::vector<block_q4_K> W0((size_t)M0 * blocks_per_row);
+    std::vector<block_q4_K> W1((size_t)M1 * blocks_per_row);
+    for (auto &blk : W0) generate_random_block_q4_K(&blk, rng);
+    for (auto &blk : W1) generate_random_block_q4_K(&blk, rng);
+    std::vector<float> X((size_t)N * K);
+    for (float &v : X) v = nd(rng);
+
+    const size_t q8_bytes = ds4_mmq_q4_K_q8_1_scratch_bytes(N, K);
+    if (q8_bytes == 0) {
+        fprintf(stderr, "Q4_K 16-warp pair scratch-size query rejected shape\n\n");
+        return false;
+    }
+
+    constexpr size_t guard_floats = 64;
+    constexpr size_t guard_bytes = guard_floats * sizeof(float);
+    constexpr uint8_t ref0_guard_byte = 0xa5;
+    constexpr uint8_t ref1_guard_byte = 0xb6;
+    constexpr uint8_t got0_guard_byte = 0x5a;
+    constexpr uint8_t got1_guard_byte = 0x69;
+    constexpr uint8_t prod0_guard_byte = 0xc3;
+    constexpr uint8_t prod1_guard_byte = 0xd4;
+    constexpr uint8_t reject_guard_byte = 0x3c;
+    constexpr uint8_t scratch_guard_byte = 0x7d;
+    const size_t count0 = (size_t)M0 * N;
+    const size_t count1 = (size_t)M1 * N;
+    const size_t bytes0 = count0 * sizeof(float);
+    const size_t bytes1 = count1 * sizeof(float);
+    const size_t guarded0 = bytes0 + 2u * guard_bytes;
+    const size_t guarded1 = bytes1 + 2u * guard_bytes;
+    const size_t scratch0_bytes =
+        ds4_mmq_q4_K_dense_16warp_streamk_scratch_bytes(M0, N, nsm);
+    const size_t scratch1_bytes =
+        ds4_mmq_q4_K_dense_16warp_streamk_scratch_bytes(M1, N, nsm);
+    const size_t scratch_bytes =
+        scratch0_bytes > scratch1_bytes ? scratch0_bytes : scratch1_bytes;
+    if (scratch_bytes > SIZE_MAX - 2u * guard_bytes) {
+        fprintf(stderr, "Q4_K 16-warp pair scratch guard size overflow\n\n");
+        return false;
+    }
+    const size_t scratch_guarded_bytes = scratch_bytes + 2u * guard_bytes;
+
+    cudaStream_t stream = nullptr;
+    void *dW0 = nullptr;
+    void *dW1 = nullptr;
+    float *dX = nullptr;
+    void *dQ8 = nullptr;
+    float *dRef0Storage = nullptr;
+    float *dRef1Storage = nullptr;
+    float *dGot0Storage = nullptr;
+    float *dGot1Storage = nullptr;
+    float *dProd0Storage = nullptr;
+    float *dProd1Storage = nullptr;
+    void *dScratchStorage = nullptr;
+    const bool allocated = cudaStreamCreate(&stream) == cudaSuccess &&
+        cudaMalloc(&dW0, W0.size() * sizeof(block_q4_K)) == cudaSuccess &&
+        cudaMalloc(&dW1, W1.size() * sizeof(block_q4_K)) == cudaSuccess &&
+        cudaMalloc(&dX, X.size() * sizeof(float)) == cudaSuccess &&
+        cudaMalloc(&dQ8, q8_bytes) == cudaSuccess &&
+        cudaMalloc(&dRef0Storage, guarded0) == cudaSuccess &&
+        cudaMalloc(&dRef1Storage, guarded1) == cudaSuccess &&
+        cudaMalloc(&dGot0Storage, guarded0) == cudaSuccess &&
+        cudaMalloc(&dGot1Storage, guarded1) == cudaSuccess &&
+        cudaMalloc(&dProd0Storage, guarded0) == cudaSuccess &&
+        cudaMalloc(&dProd1Storage, guarded1) == cudaSuccess &&
+        cudaMalloc(&dScratchStorage, scratch_guarded_bytes) == cudaSuccess;
+    const auto cleanup = [&]() {
+        if (dScratchStorage) cudaFree(dScratchStorage);
+        if (dProd1Storage) cudaFree(dProd1Storage);
+        if (dProd0Storage) cudaFree(dProd0Storage);
+        if (dGot1Storage) cudaFree(dGot1Storage);
+        if (dGot0Storage) cudaFree(dGot0Storage);
+        if (dRef1Storage) cudaFree(dRef1Storage);
+        if (dRef0Storage) cudaFree(dRef0Storage);
+        if (dQ8) cudaFree(dQ8);
+        if (dX) cudaFree(dX);
+        if (dW1) cudaFree(dW1);
+        if (dW0) cudaFree(dW0);
+        if (stream) cudaStreamDestroy(stream);
+    };
+    if (!allocated) {
+        fprintf(stderr, "Q4_K 16-warp pair allocation failed: %s\n\n",
+                cudaGetErrorString(cudaGetLastError()));
+        cleanup();
+        return false;
+    }
+
+    float *const dRef0 = dRef0Storage + guard_floats;
+    float *const dRef1 = dRef1Storage + guard_floats;
+    float *const dGot0 = dGot0Storage + guard_floats;
+    float *const dGot1 = dGot1Storage + guard_floats;
+    float *const dProd0 = dProd0Storage + guard_floats;
+    float *const dProd1 = dProd1Storage + guard_floats;
+    void *const dScratch =
+        static_cast<uint8_t *>(dScratchStorage) + guard_bytes;
+    cudaError_t enqueue_err = cudaMemcpyAsync(
+        dW0, W0.data(), W0.size() * sizeof(block_q4_K),
+        cudaMemcpyHostToDevice, stream);
+    if (enqueue_err == cudaSuccess) {
+        enqueue_err = cudaMemcpyAsync(
+            dW1, W1.data(), W1.size() * sizeof(block_q4_K),
+            cudaMemcpyHostToDevice, stream);
+    }
+    if (enqueue_err == cudaSuccess) {
+        enqueue_err = cudaMemcpyAsync(
+            dX, X.data(), X.size() * sizeof(float),
+            cudaMemcpyHostToDevice, stream);
+    }
+    if (enqueue_err == cudaSuccess) {
+        enqueue_err = cudaMemsetAsync(
+            dRef0Storage, ref0_guard_byte, guarded0, stream);
+    }
+    if (enqueue_err == cudaSuccess) {
+        enqueue_err = cudaMemsetAsync(
+            dRef1Storage, ref1_guard_byte, guarded1, stream);
+    }
+    if (enqueue_err == cudaSuccess) {
+        enqueue_err = cudaMemsetAsync(
+            dGot0Storage, got0_guard_byte, guarded0, stream);
+    }
+    if (enqueue_err == cudaSuccess) {
+        enqueue_err = cudaMemsetAsync(
+            dGot1Storage, got1_guard_byte, guarded1, stream);
+    }
+    if (enqueue_err == cudaSuccess) {
+        enqueue_err = cudaMemsetAsync(
+            dProd0Storage, prod0_guard_byte, guarded0, stream);
+    }
+    if (enqueue_err == cudaSuccess) {
+        enqueue_err = cudaMemsetAsync(
+            dProd1Storage, prod1_guard_byte, guarded1, stream);
+    }
+    if (enqueue_err == cudaSuccess) {
+        enqueue_err = cudaMemsetAsync(
+            dScratchStorage, scratch_guard_byte, scratch_guarded_bytes,
+            stream);
+    }
+
+    const int rc_quant = enqueue_err == cudaSuccess
+        ? ds4_mmq_q4_K_quantize_q8_1_for_test(
+              dX, dQ8, q8_bytes, N, K, stream)
+        : -100;
+    const int rc_ref0 = rc_quant == 0
+        ? ds4_mmq_q4_K_dense_preq_reference_for_test(
+              dW0, dQ8, q8_bytes, dRef0, M0, N, K,
+              /*use_stream_k=*/1,
+              dScratch, scratch_bytes,
+              stream)
+        : -100;
+    const int rc_ref1 = rc_ref0 == 0
+        ? ds4_mmq_q4_K_dense_preq_reference_for_test(
+              dW1, dQ8, q8_bytes, dRef1, M1, N, K,
+              /*use_stream_k=*/1,
+              dScratch, scratch_bytes,
+              stream)
+        : -100;
+    std::vector<uint8_t> ref_scratch_guards(2u * guard_bytes);
+    if (enqueue_err == cudaSuccess && rc_ref1 == 0) {
+        enqueue_err = enqueue_scratch_guard_copy(
+            dScratchStorage, scratch_bytes, guard_bytes,
+            ref_scratch_guards.data(), stream);
+    }
+    const int rc_got0 = rc_ref1 == 0 && enqueue_err == cudaSuccess
+        ? ds4_mmq_q4_K_dense_16warp_streamk_enqueue(
+              dW0, dQ8, dGot0, dScratch, scratch_bytes,
+              M0, N, K, nsm, stream)
+        : -100;
+    const int rc_got1 = rc_got0 == 0
+        ? ds4_mmq_q4_K_dense_16warp_streamk_enqueue(
+              dW1, dQ8, dGot1, dScratch, scratch_bytes,
+              M1, N, K, nsm, stream)
+        : -100;
+    std::vector<uint8_t> got_scratch_guards(2u * guard_bytes);
+    if (enqueue_err == cudaSuccess && rc_got1 == 0) {
+        enqueue_err = enqueue_scratch_guard_copy(
+            dScratchStorage, scratch_bytes, guard_bytes,
+            got_scratch_guards.data(), stream);
+    }
+    const int rc_pair = rc_got1 == 0 && enqueue_err == cudaSuccess
+        ? ds4_mmq_q4_K_dense_pair(
+              dW0, dW1, dX, dProd0, dProd1, M0, M1, N, K, stream)
+        : -100;
+
+    std::vector<uint8_t> ref0_guarded(guarded0);
+    std::vector<uint8_t> ref1_guarded(guarded1);
+    std::vector<uint8_t> got0_guarded(guarded0);
+    std::vector<uint8_t> got1_guarded(guarded1);
+    std::vector<uint8_t> prod0_guarded(guarded0);
+    std::vector<uint8_t> prod1_guarded(guarded1);
+    if (rc_pair == 0) {
+        enqueue_err = cudaMemcpyAsync(
+            ref0_guarded.data(), dRef0Storage, guarded0,
+            cudaMemcpyDeviceToHost, stream);
+    }
+    if (enqueue_err == cudaSuccess && rc_pair == 0) {
+        enqueue_err = cudaMemcpyAsync(
+            ref1_guarded.data(), dRef1Storage, guarded1,
+            cudaMemcpyDeviceToHost, stream);
+    }
+    if (enqueue_err == cudaSuccess && rc_pair == 0) {
+        enqueue_err = cudaMemcpyAsync(
+            got0_guarded.data(), dGot0Storage, guarded0,
+            cudaMemcpyDeviceToHost, stream);
+    }
+    if (enqueue_err == cudaSuccess && rc_pair == 0) {
+        enqueue_err = cudaMemcpyAsync(
+            got1_guarded.data(), dGot1Storage, guarded1,
+            cudaMemcpyDeviceToHost, stream);
+    }
+    if (enqueue_err == cudaSuccess && rc_pair == 0) {
+        enqueue_err = cudaMemcpyAsync(
+            prod0_guarded.data(), dProd0Storage, guarded0,
+            cudaMemcpyDeviceToHost, stream);
+    }
+    if (enqueue_err == cudaSuccess && rc_pair == 0) {
+        enqueue_err = cudaMemcpyAsync(
+            prod1_guarded.data(), dProd1Storage, guarded1,
+            cudaMemcpyDeviceToHost, stream);
+    }
+    const cudaError_t sync_err = cudaStreamSynchronize(stream);
+
+    struct leg_result {
+        size_t mismatches = 0;
+        size_t nonfinite_ref = 0;
+        size_t nonfinite_got = 0;
+        size_t ref_canary = 0;
+        size_t got_canary = 0;
+        size_t first_bad = SIZE_MAX;
+        uint32_t first_ref_bits = 0;
+        uint32_t first_got_bits = 0;
+    };
+    const auto inspect_leg = [&](const std::vector<uint8_t> &ref,
+                                  const std::vector<uint8_t> &got,
+                                  size_t count, size_t output_bytes,
+                                  uint8_t expected_ref,
+                                  uint8_t expected_got) {
+        leg_result result;
+        if (enqueue_err != cudaSuccess || sync_err != cudaSuccess ||
+            rc_pair != 0) {
+            return result;
+        }
+        for (size_t i = 0; i < count; ++i) {
+            const uint8_t *const ref_ptr =
+                ref.data() + guard_bytes + i * sizeof(float);
+            const uint8_t *const got_ptr =
+                got.data() + guard_bytes + i * sizeof(float);
+            float ref_value = 0.0f;
+            float got_value = 0.0f;
+            std::memcpy(&ref_value, ref_ptr, sizeof(ref_value));
+            std::memcpy(&got_value, got_ptr, sizeof(got_value));
+            if (!std::isfinite(ref_value)) result.nonfinite_ref++;
+            if (!std::isfinite(got_value)) result.nonfinite_got++;
+            if (std::memcmp(ref_ptr, got_ptr, sizeof(float)) != 0) {
+                if (result.first_bad == SIZE_MAX) {
+                    result.first_bad = i;
+                    std::memcpy(&result.first_ref_bits, ref_ptr,
+                                sizeof(result.first_ref_bits));
+                    std::memcpy(&result.first_got_bits, got_ptr,
+                                sizeof(result.first_got_bits));
+                }
+                result.mismatches++;
+            }
+        }
+        for (size_t i = 0; i < guard_bytes; ++i) {
+            if (ref[i] != expected_ref) result.ref_canary++;
+            if (got[i] != expected_got) result.got_canary++;
+        }
+        for (size_t i = guard_bytes + output_bytes; i < ref.size(); ++i) {
+            if (ref[i] != expected_ref) result.ref_canary++;
+        }
+        for (size_t i = guard_bytes + output_bytes; i < got.size(); ++i) {
+            if (got[i] != expected_got) result.got_canary++;
+        }
+        return result;
+    };
+
+    const leg_result leg0 = inspect_leg(
+        ref0_guarded, got0_guarded, count0, bytes0,
+        ref0_guard_byte, got0_guard_byte);
+    const leg_result leg1 = inspect_leg(
+        ref1_guarded, got1_guarded, count1, bytes1,
+        ref1_guard_byte, got1_guard_byte);
+    const leg_result prod0 = inspect_leg(
+        ref0_guarded, prod0_guarded, count0, bytes0,
+        ref0_guard_byte, prod0_guard_byte);
+    const leg_result prod1 = inspect_leg(
+        ref1_guarded, prod1_guarded, count1, bytes1,
+        ref1_guard_byte, prod1_guard_byte);
+    const size_t ref_scratch_canary =
+        scratch_guard_mismatches(ref_scratch_guards, scratch_guard_byte);
+    const size_t got_scratch_canary =
+        scratch_guard_mismatches(got_scratch_guards, scratch_guard_byte);
+
+    // REQUIRE must reject the whole pair before either leg can enqueue. A
+    // 384-row second leg is aligned but below the pair admission floor; poison
+    // both complete guarded ranges so even a partial first-leg launch is seen.
+    int rc_reject = -100;
+    size_t reject_writes = SIZE_MAX;
+    cudaError_t reject_sync = cudaSuccess;
+    if (enqueue_err == cudaSuccess && sync_err == cudaSuccess && rc_pair == 0) {
+        cudaError_t reject_err = cudaMemsetAsync(
+            dProd0Storage, reject_guard_byte, guarded0, stream);
+        if (reject_err == cudaSuccess) {
+            reject_err = cudaMemsetAsync(
+                dProd1Storage, reject_guard_byte, guarded1, stream);
+        }
+        rc_reject = reject_err == cudaSuccess
+            ? ds4_mmq_q4_K_dense_pair(
+                  dW0, dW1, dX, dProd0, dProd1,
+                  M0, /*M1=*/384, N, K, stream)
+            : -100;
+        std::vector<uint8_t> rejected0(guarded0);
+        std::vector<uint8_t> rejected1(guarded1);
+        if (reject_err == cudaSuccess) {
+            reject_err = cudaMemcpyAsync(
+                rejected0.data(), dProd0Storage, guarded0,
+                cudaMemcpyDeviceToHost, stream);
+        }
+        if (reject_err == cudaSuccess) {
+            reject_err = cudaMemcpyAsync(
+                rejected1.data(), dProd1Storage, guarded1,
+                cudaMemcpyDeviceToHost, stream);
+        }
+        reject_sync = cudaStreamSynchronize(stream);
+        if (reject_err == cudaSuccess && reject_sync == cudaSuccess) {
+            reject_writes = 0;
+            for (uint8_t byte : rejected0) {
+                if (byte != reject_guard_byte) reject_writes++;
+            }
+            for (uint8_t byte : rejected1) {
+                if (byte != reject_guard_byte) reject_writes++;
+            }
+        }
+    }
+    const bool rejection_ok =
+        rc_reject == DS4_MMQ_NOT_APPLICABLE &&
+        reject_sync == cudaSuccess && reject_writes == 0;
+    const bool ok = rc_quant == 0 && rc_ref0 == 0 && rc_ref1 == 0 &&
+        rc_got0 == 0 && rc_got1 == 0 && rc_pair == 0 &&
+        enqueue_err == cudaSuccess &&
+        sync_err == cudaSuccess && leg0.mismatches == 0 &&
+        leg1.mismatches == 0 && prod0.mismatches == 0 &&
+        prod1.mismatches == 0 && leg0.nonfinite_ref == 0 &&
+        leg0.nonfinite_got == 0 && leg1.nonfinite_ref == 0 &&
+        leg1.nonfinite_got == 0 && prod0.nonfinite_got == 0 &&
+        prod1.nonfinite_got == 0 && leg0.ref_canary == 0 &&
+        leg0.got_canary == 0 && leg1.ref_canary == 0 &&
+        leg1.got_canary == 0 && prod0.got_canary == 0 &&
+        prod1.got_canary == 0 && ref_scratch_canary == 0 &&
+        got_scratch_canary == 0 && rejection_ok;
+    fprintf(stderr,
+            "quant/ref0/ref1/16w0/16w1/pair=%d/%d/%d/%d/%d/%d "
+            "enqueue=%s sync=%s bits(raw/prod)=%zu/%zu,%zu/%zu "
+            "nonfinite(ref/raw/prod)=%zu/%zu/%zu,%zu/%zu/%zu "
+            "canary(ref/raw/prod)=%zu/%zu/%zu,%zu/%zu/%zu "
+            "scratch_canary(ref/16w)=%zu/%zu "
+            "reject=%d/%zu reject_sync=%s: %s\n",
+            rc_quant, rc_ref0, rc_ref1, rc_got0, rc_got1, rc_pair,
+            cudaGetErrorString(enqueue_err), cudaGetErrorString(sync_err),
+            leg0.mismatches, prod0.mismatches,
+            leg1.mismatches, prod1.mismatches,
+            leg0.nonfinite_ref, leg0.nonfinite_got, prod0.nonfinite_got,
+            leg1.nonfinite_ref, leg1.nonfinite_got, prod1.nonfinite_got,
+            leg0.ref_canary, leg0.got_canary, prod0.got_canary,
+            leg1.ref_canary, leg1.got_canary, prod1.got_canary,
+            ref_scratch_canary, got_scratch_canary,
+            rc_reject, reject_writes, cudaGetErrorString(reject_sync),
+            ok ? "PASS" : "FAIL");
+    if (leg0.first_bad != SIZE_MAX) {
+        fprintf(stderr,
+                "first pair leg0 mismatch at output[%zu]: "
+                "ref=0x%08x got=0x%08x\n",
+                leg0.first_bad, leg0.first_ref_bits, leg0.first_got_bits);
+    }
+    if (leg1.first_bad != SIZE_MAX) {
+        fprintf(stderr,
+                "first pair leg1 mismatch at output[%zu]: "
+                "ref=0x%08x got=0x%08x\n",
+                leg1.first_bad, leg1.first_ref_bits, leg1.first_got_bits);
+    }
+    if (prod0.first_bad != SIZE_MAX) {
+        fprintf(stderr,
+                "first pair production leg0 mismatch at output[%zu]: "
+                "ref=0x%08x got=0x%08x\n",
+                prod0.first_bad, prod0.first_ref_bits, prod0.first_got_bits);
+    }
+    if (prod1.first_bad != SIZE_MAX) {
+        fprintf(stderr,
+                "first pair production leg1 mismatch at output[%zu]: "
+                "ref=0x%08x got=0x%08x\n",
+                prod1.first_bad, prod1.first_ref_bits, prod1.first_got_bits);
+    }
+    fputc('\n', stderr);
+    cleanup();
+    return ok;
+}
+
+// Prefill dense-pair verifier.  The candidate shares only the canonical
+// token-tiled Q8_1 activation; both weight legs still run the ordinary Q4_K
+// MMQ kernel, so their outputs must match two independent dense calls bitwise.
+bool run_q4_K_dense_pair_parity(
+        int M0, int M1, int N, int K, uint32_t seed) {
+    fprintf(stderr,
+            "=== Q4_K/DENSE_PAIR  M0=%d M1=%d N=%d K=%d seed=%u ===\n",
+            M0, M1, N, K, seed);
+
+    std::mt19937 rng(seed);
+    std::normal_distribution<float> nd(0.0f, 1.0f);
+    const int blocks_per_row = K / QK_K_LOCAL;
+    std::vector<block_q4_K> W0((size_t)M0 * blocks_per_row);
+    std::vector<block_q4_K> W1((size_t)M1 * blocks_per_row);
+    for (auto &blk : W0) generate_random_block_q4_K(&blk, rng);
+    for (auto &blk : W1) generate_random_block_q4_K(&blk, rng);
+    std::vector<float> X((size_t)N * K);
+    for (auto &v : X) v = nd(rng);
+
+    cudaStream_t stream;
+    cudaStreamCreate(&stream);
+    void *dW0 = nullptr;
+    void *dW1 = nullptr;
+    float *dX = nullptr;
+    float *dRef0 = nullptr;
+    float *dRef1 = nullptr;
+    float *dGot0 = nullptr;
+    float *dGot1 = nullptr;
+    cudaMalloc(&dW0, W0.size() * sizeof(block_q4_K));
+    cudaMalloc(&dW1, W1.size() * sizeof(block_q4_K));
+    cudaMalloc(&dX, X.size() * sizeof(float));
+    cudaMalloc(&dRef0, (size_t)M0 * N * sizeof(float));
+    cudaMalloc(&dRef1, (size_t)M1 * N * sizeof(float));
+    cudaMalloc(&dGot0, (size_t)M0 * N * sizeof(float));
+    cudaMalloc(&dGot1, (size_t)M1 * N * sizeof(float));
+    cudaMemcpyAsync(dW0, W0.data(), W0.size() * sizeof(block_q4_K),
+                    cudaMemcpyHostToDevice, stream);
+    cudaMemcpyAsync(dW1, W1.data(), W1.size() * sizeof(block_q4_K),
+                    cudaMemcpyHostToDevice, stream);
+    cudaMemcpyAsync(dX, X.data(), X.size() * sizeof(float),
+                    cudaMemcpyHostToDevice, stream);
+    cudaMemsetAsync(dRef0, 0xa5, (size_t)M0 * N * sizeof(float), stream);
+    cudaMemsetAsync(dRef1, 0xa5, (size_t)M1 * N * sizeof(float), stream);
+    cudaMemsetAsync(dGot0, 0x5a, (size_t)M0 * N * sizeof(float), stream);
+    cudaMemsetAsync(dGot1, 0x5a, (size_t)M1 * N * sizeof(float), stream);
+
+    const int rc0 = ds4_mmq_q4_K_dense(
+        dW0, dX, dRef0, M0, N, K, stream);
+    const int rc1 = ds4_mmq_q4_K_dense(
+        dW1, dX, dRef1, M1, N, K, stream);
+    const int rcp = ds4_mmq_q4_K_dense_pair(
+        dW0, dW1, dX, dGot0, dGot1, M0, M1, N, K, stream);
+
+    std::vector<float> ref0((size_t)M0 * N);
+    std::vector<float> ref1((size_t)M1 * N);
+    std::vector<float> got0((size_t)M0 * N);
+    std::vector<float> got1((size_t)M1 * N);
+    cudaMemcpyAsync(ref0.data(), dRef0, ref0.size() * sizeof(float),
+                    cudaMemcpyDeviceToHost, stream);
+    cudaMemcpyAsync(ref1.data(), dRef1, ref1.size() * sizeof(float),
+                    cudaMemcpyDeviceToHost, stream);
+    cudaMemcpyAsync(got0.data(), dGot0, got0.size() * sizeof(float),
+                    cudaMemcpyDeviceToHost, stream);
+    cudaMemcpyAsync(got1.data(), dGot1, got1.size() * sizeof(float),
+                    cudaMemcpyDeviceToHost, stream);
+    const cudaError_t sync_err = cudaStreamSynchronize(stream);
+
+    size_t bad0 = 0;
+    size_t bad1 = 0;
+    for (size_t i = 0; i < ref0.size(); i++) {
+        if (std::memcmp(&ref0[i], &got0[i], sizeof(float)) != 0) bad0++;
+    }
+    for (size_t i = 0; i < ref1.size(); i++) {
+        if (std::memcmp(&ref1[i], &got1[i], sizeof(float)) != 0) bad1++;
+    }
+
+    cudaFree(dW0);
+    cudaFree(dW1);
+    cudaFree(dX);
+    cudaFree(dRef0);
+    cudaFree(dRef1);
+    cudaFree(dGot0);
+    cudaFree(dGot1);
+    cudaStreamDestroy(stream);
+
+    const bool ok = rc0 == 0 && rc1 == 0 && rcp == 0 &&
+                    sync_err == cudaSuccess && bad0 == 0 && bad1 == 0;
+    fprintf(stderr,
+            "pair rc=%d/%d/%d sync=%s mismatches=%zu/%zu: %s\n\n",
+            rc0, rc1, rcp, cudaGetErrorString(sync_err), bad0, bad1,
+            ok ? "PASS" : "FAIL");
+    return ok;
+}
+
+// Prefill attention output-A verifier.  The grouped entry consumes
+// X=[N][G][K] and writes out=[N][G][M].  Build the reference with exactly G
+// ordinary dense calls, using D2D 2D copies to pack/unpack each group.  This
+// keeps the quantizer and MMQ reduction tree identical while independently
+// checking the grouped entry's strided quantization and output pitch.
+bool run_q4_K_grouped_dense_parity(
+        int M, int N, int K, int n_groups, uint32_t seed,
+        bool inject_nonfinite = false) {
+    fprintf(stderr,
+            "=== Q4_K/GROUPED_DENSE  M=%d N=%d K=%d groups=%d seed=%u%s ===\n",
+            M, N, K, n_groups, seed,
+            inject_nonfinite ? " nonfinite" : "");
+
+    std::mt19937 rng(seed);
+    std::normal_distribution<float> nd(0.0f, 1.0f);
+    const int blocks_per_row = K / QK_K_LOCAL;
+    const size_t blocks_per_group = (size_t)M * blocks_per_row;
+    std::vector<block_q4_K> W((size_t)n_groups * blocks_per_group);
+    for (auto &blk : W) generate_random_block_q4_K(&blk, rng);
+    std::vector<float> X((size_t)N * n_groups * K);
+    for (float &v : X) v = nd(rng);
+
+    // One NaN scale makes this row's accumulator non-finite.  Both the
+    // ordinary dense reference and the grouped entry must apply Q4_K's
+    // fused sanitize contract and publish +0.0f for every token.
+    const int nonfinite_group = n_groups / 2;
+    const int nonfinite_row = M / 2;
+    if (inject_nonfinite) {
+        block_q4_K &blk =
+            W[((size_t)nonfinite_group * M + nonfinite_row) *
+              blocks_per_row];
+        set_half_from_u16(blk.data.d, (uint16_t)0x7e00u);
+    }
+
+    constexpr size_t guard_floats = 64;
+    constexpr uint8_t guard_byte = 0xa5;
+    const size_t output_count = (size_t)N * n_groups * M;
+    const size_t output_bytes = output_count * sizeof(float);
+    const size_t guarded_count = output_count + 2u * guard_floats;
+    const size_t guarded_bytes = guarded_count * sizeof(float);
+
+    cudaStream_t stream = nullptr;
+    void *dW = nullptr;
+    float *dX = nullptr;
+    float *dGroupX = nullptr;
+    float *dGroupOut = nullptr;
+    float *dRef = nullptr;
+    float *dGotStorage = nullptr;
+    bool allocated = cudaStreamCreate(&stream) == cudaSuccess &&
+        cudaMalloc(&dW, W.size() * sizeof(block_q4_K)) == cudaSuccess &&
+        cudaMalloc(&dX, X.size() * sizeof(float)) == cudaSuccess &&
+        cudaMalloc(&dGroupX, (size_t)N * K * sizeof(float)) == cudaSuccess &&
+        cudaMalloc(&dGroupOut, (size_t)N * M * sizeof(float)) == cudaSuccess &&
+        cudaMalloc(&dRef, output_bytes) == cudaSuccess &&
+        cudaMalloc(&dGotStorage, guarded_bytes) == cudaSuccess;
+    const auto cleanup = [&]() {
+        if (dGotStorage) cudaFree(dGotStorage);
+        if (dRef) cudaFree(dRef);
+        if (dGroupOut) cudaFree(dGroupOut);
+        if (dGroupX) cudaFree(dGroupX);
+        if (dX) cudaFree(dX);
+        if (dW) cudaFree(dW);
+        if (stream) cudaStreamDestroy(stream);
+    };
+    if (!allocated) {
+        fprintf(stderr, "Q4_K grouped dense parity allocation failed: %s\n\n",
+                cudaGetErrorString(cudaGetLastError()));
+        cleanup();
+        return false;
+    }
+
+    float *const dGot = dGotStorage + guard_floats;
+    cudaError_t enqueue_err = cudaMemcpyAsync(
+        dW, W.data(), W.size() * sizeof(block_q4_K),
+        cudaMemcpyHostToDevice, stream);
+    if (enqueue_err == cudaSuccess) {
+        enqueue_err = cudaMemcpyAsync(
+            dX, X.data(), X.size() * sizeof(float),
+            cudaMemcpyHostToDevice, stream);
+    }
+    if (enqueue_err == cudaSuccess) {
+        enqueue_err = cudaMemsetAsync(dRef, 0x5a, output_bytes, stream);
+    }
+    if (enqueue_err == cudaSuccess) {
+        enqueue_err = cudaMemsetAsync(
+            dGotStorage, guard_byte, guarded_bytes, stream);
+    }
+
+    int rc_ref = enqueue_err == cudaSuccess ? 0 : -100;
+    for (int g = 0; g < n_groups && rc_ref == 0; ++g) {
+        cudaError_t err = cudaMemcpy2DAsync(
+            dGroupX, (size_t)K * sizeof(float),
+            dX + (size_t)g * K,
+            (size_t)n_groups * K * sizeof(float),
+            (size_t)K * sizeof(float), (size_t)N,
+            cudaMemcpyDeviceToDevice, stream);
+        if (err != cudaSuccess) {
+            enqueue_err = err;
+            rc_ref = -101;
+            break;
+        }
+        rc_ref = ds4_mmq_q4_K_dense(
+            (const char *)dW + (size_t)g * blocks_per_group *
+                sizeof(block_q4_K),
+            dGroupX, dGroupOut, M, N, K, stream);
+        if (rc_ref != 0) break;
+        err = cudaMemcpy2DAsync(
+            dRef + (size_t)g * M,
+            (size_t)n_groups * M * sizeof(float),
+            dGroupOut, (size_t)M * sizeof(float),
+            (size_t)M * sizeof(float), (size_t)N,
+            cudaMemcpyDeviceToDevice, stream);
+        if (err != cudaSuccess) {
+            enqueue_err = err;
+            rc_ref = -102;
+        }
+    }
+
+    const int rc_got = enqueue_err == cudaSuccess
+        ? ds4_mmq_q4_K_grouped_dense(
+              dW, dX, dGot, M, N, K, n_groups, stream)
+        : -100;
+
+    std::vector<float> ref(output_count);
+    std::vector<float> got(output_count);
+    std::vector<uint8_t> guarded(guarded_bytes);
+    if (enqueue_err == cudaSuccess) {
+        enqueue_err = cudaMemcpyAsync(
+            ref.data(), dRef, output_bytes, cudaMemcpyDeviceToHost, stream);
+    }
+    if (enqueue_err == cudaSuccess) {
+        enqueue_err = cudaMemcpyAsync(
+            got.data(), dGot, output_bytes, cudaMemcpyDeviceToHost, stream);
+    }
+    if (enqueue_err == cudaSuccess) {
+        enqueue_err = cudaMemcpyAsync(
+            guarded.data(), dGotStorage, guarded_bytes,
+            cudaMemcpyDeviceToHost, stream);
+    }
+    const cudaError_t sync_err = cudaStreamSynchronize(stream);
+
+    size_t mismatches = 0;
+    size_t nonfinite_ref = 0;
+    size_t nonfinite_got = 0;
+    for (size_t i = 0; i < output_count; ++i) {
+        if (std::memcmp(&ref[i], &got[i], sizeof(float)) != 0) mismatches++;
+        if (!std::isfinite(ref[i])) nonfinite_ref++;
+        if (!std::isfinite(got[i])) nonfinite_got++;
+    }
+    const size_t guard_bytes = guard_floats * sizeof(float);
+    size_t canary_mismatches = 0;
+    for (size_t i = 0; i < guard_bytes; ++i) {
+        if (guarded[i] != guard_byte) canary_mismatches++;
+    }
+    for (size_t i = guard_bytes + output_bytes; i < guarded.size(); ++i) {
+        if (guarded[i] != guard_byte) canary_mismatches++;
+    }
+
+    size_t sanitize_mismatches = 0;
+    if (inject_nonfinite) {
+        const uint32_t positive_zero = 0;
+        for (int t = 0; t < N; ++t) {
+            const size_t i =
+                ((size_t)t * n_groups + nonfinite_group) * M +
+                nonfinite_row;
+            uint32_t ref_bits = 0;
+            uint32_t got_bits = 0;
+            std::memcpy(&ref_bits, &ref[i], sizeof(ref_bits));
+            std::memcpy(&got_bits, &got[i], sizeof(got_bits));
+            if (ref_bits != positive_zero || got_bits != positive_zero) {
+                sanitize_mismatches++;
+            }
+        }
+    }
+
+    const bool ok = rc_ref == 0 && rc_got == 0 &&
+        enqueue_err == cudaSuccess && sync_err == cudaSuccess &&
+        mismatches == 0 && nonfinite_ref == 0 && nonfinite_got == 0 &&
+        canary_mismatches == 0 && sanitize_mismatches == 0;
+    fprintf(stderr,
+            "rc_ref=%d rc_grouped=%d enqueue=%s sync=%s "
+            "mismatches=%zu nonfinite=%zu/%zu canary=%zu sanitize=%zu: %s\n\n",
+            rc_ref, rc_got, cudaGetErrorString(enqueue_err),
+            cudaGetErrorString(sync_err), mismatches, nonfinite_ref,
+            nonfinite_got, canary_mismatches, sanitize_mismatches,
+            ok ? "PASS" : "FAIL");
+    cleanup();
+    return ok;
+}
+
+#if !defined(GGML_USE_HIP)
+// The production selector is deliberately GB10-only, but the specialized
+// quantizer uses baseline CUDA operations.  Exercise it directly on every
+// CUDA test device and require the complete block_q8_1_mmq payload to match
+// the canonical strided producer byte-for-byte.
+bool run_q4_K_grouped_q8_1_kernel_parity(int N, uint32_t seed) {
+    fprintf(stderr,
+            "=== Q4_K/GROUPED_Q8_1_K4096_G8X2 N=%d seed=%u ===\n",
+            N, seed);
+
+    constexpr int K = 4096;
+    constexpr int groups = 8;
+    constexpr size_t guard_bytes = 256u;
+    constexpr uint8_t guard_byte = 0xa5u;
+    const size_t q8_bytes =
+        ds4_mmq_q4_K_grouped_q8_1_scratch_bytes_for_test(N);
+    const size_t x_count = (size_t)N * groups * K;
+    if (q8_bytes == 0u ||
+        q8_bytes > SIZE_MAX - 2u * guard_bytes) {
+        fprintf(stderr, "invalid grouped Q8_1 parity shape\n\n");
+        return false;
+    }
+
+    std::mt19937 rng(seed);
+    std::normal_distribution<float> nd(0.0f, 1.0f);
+    std::vector<float> X(x_count);
+    for (float &value : X) value = nd(rng);
+
+    cudaStream_t stream = nullptr;
+    float *dX = nullptr;
+    void *dReferenceStorage = nullptr;
+    void *dCandidateStorage = nullptr;
+    const bool allocated = cudaStreamCreate(&stream) == cudaSuccess &&
+        cudaMalloc(&dX, X.size() * sizeof(float)) == cudaSuccess &&
+        cudaMalloc(&dReferenceStorage, q8_bytes + 2u * guard_bytes) ==
+            cudaSuccess &&
+        cudaMalloc(&dCandidateStorage, q8_bytes + 2u * guard_bytes) ==
+            cudaSuccess;
+    const auto cleanup = [&]() {
+        if (dCandidateStorage) cudaFree(dCandidateStorage);
+        if (dReferenceStorage) cudaFree(dReferenceStorage);
+        if (dX) cudaFree(dX);
+        if (stream) cudaStreamDestroy(stream);
+    };
+    if (!allocated) {
+        fprintf(stderr,
+                "grouped Q8_1 parity allocation failed: %s\n\n",
+                cudaGetErrorString(cudaGetLastError()));
+        cleanup();
+        return false;
+    }
+
+    auto *dReference =
+        static_cast<uint8_t *>(dReferenceStorage) + guard_bytes;
+    auto *dCandidate =
+        static_cast<uint8_t *>(dCandidateStorage) + guard_bytes;
+    cudaError_t enqueue_err = cudaMemcpyAsync(
+        dX, X.data(), X.size() * sizeof(float),
+        cudaMemcpyHostToDevice, stream);
+    if (enqueue_err == cudaSuccess) {
+        enqueue_err = cudaMemsetAsync(
+            dReferenceStorage, guard_byte,
+            q8_bytes + 2u * guard_bytes, stream);
+    }
+    if (enqueue_err == cudaSuccess) {
+        enqueue_err = cudaMemsetAsync(
+            dCandidateStorage, guard_byte,
+            q8_bytes + 2u * guard_bytes, stream);
+    }
+
+    const int rc_reference = enqueue_err == cudaSuccess
+        ? ds4_mmq_q4_K_grouped_quantize_q8_1_for_test(
+              dX, dReference, q8_bytes, N, /*use_specialized=*/0, stream)
+        : -100;
+    const int rc_candidate = rc_reference == 0
+        ? ds4_mmq_q4_K_grouped_quantize_q8_1_for_test(
+              dX, dCandidate, q8_bytes, N, /*use_specialized=*/1, stream)
+        : -100;
+
+    std::vector<uint8_t> reference(q8_bytes + 2u * guard_bytes);
+    std::vector<uint8_t> candidate(q8_bytes + 2u * guard_bytes);
+    if (rc_candidate == 0) {
+        enqueue_err = cudaMemcpyAsync(
+            reference.data(), dReferenceStorage, reference.size(),
+            cudaMemcpyDeviceToHost, stream);
+    }
+    if (enqueue_err == cudaSuccess && rc_candidate == 0) {
+        enqueue_err = cudaMemcpyAsync(
+            candidate.data(), dCandidateStorage, candidate.size(),
+            cudaMemcpyDeviceToHost, stream);
+    }
+    const cudaError_t sync_err = cudaStreamSynchronize(stream);
+
+    size_t mismatches = 0;
+    size_t first_mismatch = SIZE_MAX;
+    for (size_t i = 0; i < q8_bytes; ++i) {
+        const size_t offset = guard_bytes + i;
+        if (reference[offset] != candidate[offset]) {
+            if (first_mismatch == SIZE_MAX) first_mismatch = i;
+            mismatches++;
+        }
+    }
+    size_t canary_mismatches = 0;
+    for (size_t i = 0; i < guard_bytes; ++i) {
+        if (reference[i] != guard_byte || candidate[i] != guard_byte) {
+            canary_mismatches++;
+        }
+        const size_t suffix = guard_bytes + q8_bytes + i;
+        if (reference[suffix] != guard_byte ||
+            candidate[suffix] != guard_byte) {
+            canary_mismatches++;
+        }
+    }
+
+    const bool ok = rc_reference == 0 && rc_candidate == 0 &&
+        enqueue_err == cudaSuccess && sync_err == cudaSuccess &&
+        mismatches == 0 && canary_mismatches == 0;
+    const std::string first = first_mismatch == SIZE_MAX
+        ? "none" : std::to_string(first_mismatch);
+    fprintf(stderr,
+            "rc=%d/%d enqueue=%s sync=%s bytes=%zu mismatches=%zu "
+            "first=%s canary=%zu: %s\n\n",
+            rc_reference, rc_candidate, cudaGetErrorString(enqueue_err),
+            cudaGetErrorString(sync_err), q8_bytes, mismatches,
+            first.c_str(), canary_mismatches, ok ? "PASS" : "FAIL");
+    cleanup();
+    return ok;
+}
+
+bool run_q4_K_grouped_q8_1_kernel_suite() {
+    bool ok = true;
+    ok &= run_q4_K_grouped_q8_1_kernel_parity(
+        /*N=*/9, 0xC4810009u);
+    ok &= run_q4_K_grouped_q8_1_kernel_parity(
+        /*N=*/127, 0xC481007Fu);
+    ok &= run_q4_K_grouped_q8_1_kernel_parity(
+        /*N=*/128, 0xC4810080u);
+    ok &= run_q4_K_grouped_q8_1_kernel_parity(
+        /*N=*/129, 0xC4810081u);
+    return ok;
+}
+#endif
+
 // IQ2_XXS internally accumulates in int8 via SIMD intrinsics
 // (__vsub4 / __vcmpne4 in vec_dot_iq2_xxs_q8_1) and applies the scale
 // post-accumulation, while the CPU reference does per-element float
@@ -843,6 +2094,652 @@ bool run_moe_pair_generic(
     return ok;
 }
 
+// Reference glue for the raw fused IQ2/Q2 pipeline.  Keep the expression and
+// non-finite handling identical to ds4_swiglu_weighted_f32 in ds4_mmq.cu so
+// parity below isolates routing-map reuse rather than activation math.
+__global__ void test_swiglu_weighted_f32(
+        const float * gate, const float * up, const float * router_weights,
+        float * mid, uint64_t n, int K, float clamp) {
+    const uint64_t i = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x;
+    if (i >= n) return;
+    const uint64_t pair = i / (uint64_t)K;
+    float g = isfinite(gate[i]) ? gate[i] : 0.0f;
+    float u = isfinite(up[i]) ? up[i] : 0.0f;
+    if (clamp > 1.0e-6f) {
+        g = fminf(g, clamp);
+        u = fminf(fmaxf(u, -clamp), clamp);
+    }
+    mid[i] = (g / (1.0f + expf(-g))) * u * router_weights[pair];
+}
+
+// Exercise the SSD-facing raw fused ABI with a non-identity compact expert
+// table.  The reference is the established materialized chain:
+//   raw pair -> weighted SwiGLU -> raw down.
+// Every token has six distinct experts.  Flattening ids to [assignments, 1]
+// for the reference down leg produces the same stable expert ordering as the
+// single routing map reused by the fused candidate.
+bool run_iq2_xxs_q2_K_fused_raw_parity(
+        int n_tokens, uint32_t seed, bool persistent_q81 = false) {
+    constexpr int global_experts = 13;
+    constexpr int compact_experts = 8;
+    constexpr int n_expert_used = 6;
+    // Deliberately asymmetric and multi-block in both legs.  A 256x256
+    // fixture degenerates every raw row to one GGUF block and cannot catch a
+    // bad row/expert stride in either the IQ2 gate/up or Q2 down tensor.
+    constexpr int expert_mid_dim = 512;
+    constexpr int expert_in_dim = 768;
+    constexpr int out_dim = 768;
+    constexpr float clamp = 6.0f;
+    const int compact_to_global[compact_experts] = {11, 2, 9, 0, 7, 12, 4, 6};
+    int persistent_device = 0;
+
+    if (persistent_q81) {
+        cudaDeviceProp prop = {};
+        if (cudaGetDevice(&persistent_device) != cudaSuccess ||
+            cudaGetDeviceProperties(&prop, persistent_device) != cudaSuccess ||
+            !prop.integrated || prop.major != 12 || prop.minor != 1) {
+            (void)cudaGetLastError();
+            fprintf(stderr,
+                    "=== IQ2_XXS+Q2_K/FUSED_RAW persistent Q8_1: "
+                    "SKIP (requires integrated sm_121) ===\n\n");
+            return true;
+        }
+    }
+
+    fprintf(stderr,
+            "=== IQ2_XXS+Q2_K/FUSED_RAW%s compact-remap ntok=%d "
+            "nexp=%d nused=%d seed=%u ===\n",
+            persistent_q81 ? "/PERSISTENT_Q81" : "",
+            n_tokens, compact_experts, n_expert_used, seed);
+
+    std::mt19937 rng(seed);
+    std::normal_distribution<float> activation(0.0f, 0.05f);
+    const size_t iq2_blocks_per_expert =
+        (size_t)expert_mid_dim * (expert_in_dim / QK_K_LOCAL);
+    const size_t q2_blocks_per_expert =
+        (size_t)out_dim * (expert_mid_dim / QK_K_LOCAL);
+
+    std::vector<block_iq2_xxs> gate_global(
+        (size_t)global_experts * iq2_blocks_per_expert);
+    std::vector<block_iq2_xxs> up_global(
+        (size_t)global_experts * iq2_blocks_per_expert);
+    std::vector<block_q2_K> down_global(
+        (size_t)global_experts * q2_blocks_per_expert);
+    for (auto & block : gate_global) generate_random_block_iq2_xxs(&block, rng);
+    for (auto & block : up_global) generate_random_block_iq2_xxs(&block, rng);
+    for (auto & block : down_global) generate_random_block_q2_K(&block, rng);
+
+    std::vector<block_iq2_xxs> gate_compact(
+        (size_t)compact_experts * iq2_blocks_per_expert);
+    std::vector<block_iq2_xxs> up_compact(
+        (size_t)compact_experts * iq2_blocks_per_expert);
+    std::vector<block_q2_K> down_compact(
+        (size_t)compact_experts * q2_blocks_per_expert);
+    int global_to_compact[global_experts];
+    std::fill(global_to_compact, global_to_compact + global_experts, -1);
+    for (int compact = 0; compact < compact_experts; compact++) {
+        const int global = compact_to_global[compact];
+        global_to_compact[global] = compact;
+        std::memcpy(gate_compact.data() + (size_t)compact * iq2_blocks_per_expert,
+                    gate_global.data() + (size_t)global * iq2_blocks_per_expert,
+                    iq2_blocks_per_expert * sizeof(block_iq2_xxs));
+        std::memcpy(up_compact.data() + (size_t)compact * iq2_blocks_per_expert,
+                    up_global.data() + (size_t)global * iq2_blocks_per_expert,
+                    iq2_blocks_per_expert * sizeof(block_iq2_xxs));
+        std::memcpy(down_compact.data() + (size_t)compact * q2_blocks_per_expert,
+                    down_global.data() + (size_t)global * q2_blocks_per_expert,
+                    q2_blocks_per_expert * sizeof(block_q2_K));
+    }
+
+    const size_t assignments = (size_t)n_tokens * n_expert_used;
+    std::vector<int32_t> global_ids(assignments);
+    std::vector<int32_t> remapped_ids(assignments);
+    std::vector<float> router_weights(assignments);
+    for (int token = 0; token < n_tokens; token++) {
+        bool seen[compact_experts] = {};
+        float router_sum = 0.0f;
+        for (int slot = 0; slot < n_expert_used; slot++) {
+            // Three is coprime with eight, so the six positions are unique;
+            // the global round-trip makes this an explicit compact-remap test.
+            const int compact = (token * 5 + slot * 3) % compact_experts;
+            const int global = compact_to_global[compact];
+            if (seen[compact] || global_to_compact[global] != compact) {
+                fprintf(stderr, "invalid compact routing fixture\n");
+                return false;
+            }
+            seen[compact] = true;
+            const size_t pair = (size_t)token * n_expert_used + slot;
+            global_ids[pair] = global;
+            remapped_ids[pair] = global_to_compact[global];
+            // Token- and slot-varying values make a pair-stride bug visible;
+            // normalize per token to retain the production router contract.
+            const float raw_weight =
+                (float)(1 + ((token * 11 + slot * 7) % 23));
+            router_weights[pair] = raw_weight;
+            router_sum += raw_weight;
+        }
+        for (int slot = 0; slot < n_expert_used; slot++) {
+            const size_t pair = (size_t)token * n_expert_used + slot;
+            router_weights[pair] /= router_sum;
+        }
+    }
+    std::vector<float> X((size_t)n_tokens * expert_in_dim);
+    for (float & value : X) value = activation(rng);
+
+    const size_t mid_count = assignments * expert_mid_dim;
+    const size_t down_count = assignments * out_dim;
+    cudaStream_t stream = nullptr;
+    void *d_gate_w = nullptr, *d_up_w = nullptr, *d_down_w = nullptr;
+    void *d_gate_global_w = nullptr, *d_up_global_w = nullptr,
+         *d_down_global_w = nullptr;
+    float *d_x = nullptr, *d_router = nullptr;
+    int32_t *d_ids = nullptr, *d_global_ids = nullptr;
+    float *d_gate_ref = nullptr, *d_up_ref = nullptr, *d_mid_ref = nullptr,
+          *d_down_ref = nullptr;
+    float *d_gate_got = nullptr, *d_up_got = nullptr, *d_mid_got = nullptr,
+          *d_down_got = nullptr;
+    float *d_gate_global = nullptr, *d_up_global = nullptr,
+          *d_mid_global = nullptr, *d_down_global = nullptr;
+
+    scoped_env_override q81_env("DS4_CUDA_MMQ_Q81_PERSISTENT");
+    int initial_arena_cleanup = 0;
+    int final_arena_cleanup = 0;
+    int q81_lazy_init_rc = 0;
+    uint64_t q81_init_allocations0 = 0, q81_init_resizes0 = 0;
+    uint64_t q81_init_allocations1 = 0, q81_init_resizes1 = 0;
+    size_t q81_init_arena0 = 0, q81_init_arena1 = 0;
+    bool q81_env_restore_ok = true;
+    bool persistent_active = false;
+
+    auto teardown_persistent = [&]() {
+        if (!persistent_active) return;
+        // Disable acquisition before retiring the owned arena, then restore
+        // the caller's exact environment (including an originally absent key).
+        q81_env_restore_ok = q81_env.set("0") && q81_env_restore_ok;
+        // Record the cleanup API result explicitly.  The setter repeats an
+        // idempotent cleanup while returning the runner-owned flag to false.
+        final_arena_cleanup = ds4_mmq_q81_persistent_cleanup();
+        ds4_mmq_set_gb10_optimizations(0);
+        q81_env_restore_ok = q81_env.restore() && q81_env_restore_ok;
+        persistent_active = false;
+    };
+
+    auto cleanup = [&]() {
+        teardown_persistent();
+        if (d_down_global) cudaFree(d_down_global);
+        if (d_mid_global) cudaFree(d_mid_global);
+        if (d_up_global) cudaFree(d_up_global);
+        if (d_gate_global) cudaFree(d_gate_global);
+        if (d_down_got) cudaFree(d_down_got);
+        if (d_mid_got) cudaFree(d_mid_got);
+        if (d_up_got) cudaFree(d_up_got);
+        if (d_gate_got) cudaFree(d_gate_got);
+        if (d_down_ref) cudaFree(d_down_ref);
+        if (d_mid_ref) cudaFree(d_mid_ref);
+        if (d_up_ref) cudaFree(d_up_ref);
+        if (d_gate_ref) cudaFree(d_gate_ref);
+        if (d_router) cudaFree(d_router);
+        if (d_global_ids) cudaFree(d_global_ids);
+        if (d_ids) cudaFree(d_ids);
+        if (d_x) cudaFree(d_x);
+        if (d_down_global_w) cudaFree(d_down_global_w);
+        if (d_up_global_w) cudaFree(d_up_global_w);
+        if (d_gate_global_w) cudaFree(d_gate_global_w);
+        if (d_down_w) cudaFree(d_down_w);
+        if (d_up_w) cudaFree(d_up_w);
+        if (d_gate_w) cudaFree(d_gate_w);
+        if (stream) cudaStreamDestroy(stream);
+    };
+
+    if (persistent_q81) {
+        if (!q81_env.set("1")) return false;
+        initial_arena_cleanup = ds4_mmq_q81_persistent_cleanup();
+        ds4_mmq_set_gb10_optimizations(1);
+        persistent_active = true;
+        ds4_mmq_q81_persistent_counters(
+            nullptr, nullptr, nullptr, nullptr,
+            &q81_init_allocations0, &q81_init_resizes0,
+            &q81_init_arena0, nullptr);
+        q81_lazy_init_rc = ds4_mmq_init(persistent_device);
+        ds4_mmq_q81_persistent_counters(
+            nullptr, nullptr, nullptr, nullptr,
+            &q81_init_allocations1, &q81_init_resizes1,
+            &q81_init_arena1, nullptr);
+        // A real `=0` dispatch below is the value-aware opt-out oracle.
+        if (!q81_env.set("0")) {
+            cleanup();
+            return false;
+        }
+    }
+
+    bool setup_ok = true;
+    cudaError_t setup_err = cudaSuccess;
+    const char *setup_step = nullptr;
+    size_t setup_bytes = 0;
+    bool setup_null_pointer = false;
+    if (!persistent_q81) {
+        setup_err = cudaStreamCreate(&stream);
+        if (setup_err != cudaSuccess) {
+            setup_ok = false;
+            setup_step = "cudaStreamCreate";
+        }
+    }
+    // Persistent Q8_1 admission deliberately requires the legacy default
+    // stream, represented by the null handle initialized above.
+    auto try_alloc = [&](void **ptr, size_t bytes, const char *label) {
+        if (!setup_ok) return;
+        setup_err = cudaMalloc(ptr, bytes);
+        if (setup_err != cudaSuccess || !*ptr) {
+            setup_ok = false;
+            setup_step = label;
+            setup_bytes = bytes;
+            setup_null_pointer = setup_err == cudaSuccess && !*ptr;
+        }
+    };
+    try_alloc(&d_gate_w,
+              gate_compact.size() * sizeof(block_iq2_xxs), "gate weights");
+    try_alloc(&d_up_w,
+              up_compact.size() * sizeof(block_iq2_xxs), "up weights");
+    try_alloc(&d_down_w,
+              down_compact.size() * sizeof(block_q2_K), "down weights");
+    try_alloc(&d_gate_global_w,
+              gate_global.size() * sizeof(block_iq2_xxs), "global gate weights");
+    try_alloc(&d_up_global_w,
+              up_global.size() * sizeof(block_iq2_xxs), "global up weights");
+    try_alloc(&d_down_global_w,
+              down_global.size() * sizeof(block_q2_K), "global down weights");
+    try_alloc((void **)&d_x, X.size() * sizeof(float), "activations");
+    try_alloc((void **)&d_ids,
+              remapped_ids.size() * sizeof(int32_t), "remapped ids");
+    try_alloc((void **)&d_global_ids,
+              global_ids.size() * sizeof(int32_t), "global ids");
+    try_alloc((void **)&d_router,
+              router_weights.size() * sizeof(float), "router weights");
+    try_alloc((void **)&d_gate_ref,
+              mid_count * sizeof(float), "reference gate output");
+    try_alloc((void **)&d_up_ref,
+              mid_count * sizeof(float), "reference up output");
+    try_alloc((void **)&d_mid_ref,
+              mid_count * sizeof(float), "reference mid output");
+    try_alloc((void **)&d_down_ref,
+              down_count * sizeof(float), "reference down output");
+    try_alloc((void **)&d_gate_got,
+              mid_count * sizeof(float), "candidate gate output");
+    try_alloc((void **)&d_up_got,
+              mid_count * sizeof(float), "candidate up output");
+    try_alloc((void **)&d_mid_got,
+              mid_count * sizeof(float), "candidate mid output");
+    try_alloc((void **)&d_down_got,
+              down_count * sizeof(float), "candidate down output");
+    try_alloc((void **)&d_gate_global,
+              mid_count * sizeof(float), "global gate output");
+    try_alloc((void **)&d_up_global,
+              mid_count * sizeof(float), "global up output");
+    try_alloc((void **)&d_mid_global,
+              mid_count * sizeof(float), "global mid output");
+    try_alloc((void **)&d_down_global,
+              down_count * sizeof(float), "global down output");
+
+    if (!setup_ok) {
+        if (setup_bytes != 0) {
+            fprintf(stderr,
+                    "fused raw parity %s allocation (%zu B) failed: %s%s\n",
+                    setup_step, setup_bytes, cudaGetErrorString(setup_err),
+                    setup_null_pointer ? " (null pointer)" : "");
+        } else {
+            fprintf(stderr, "fused raw parity %s failed: %s\n",
+                    setup_step, cudaGetErrorString(setup_err));
+        }
+        (void)cudaGetLastError();
+        fprintf(stderr, "FAIL\n\n");
+        cleanup();
+        return false;
+    }
+
+    cudaMemcpyAsync(d_gate_w, gate_compact.data(),
+                    gate_compact.size() * sizeof(block_iq2_xxs),
+                    cudaMemcpyHostToDevice, stream);
+    cudaMemcpyAsync(d_up_w, up_compact.data(),
+                    up_compact.size() * sizeof(block_iq2_xxs),
+                    cudaMemcpyHostToDevice, stream);
+    cudaMemcpyAsync(d_down_w, down_compact.data(),
+                    down_compact.size() * sizeof(block_q2_K),
+                    cudaMemcpyHostToDevice, stream);
+    cudaMemcpyAsync(d_gate_global_w, gate_global.data(),
+                    gate_global.size() * sizeof(block_iq2_xxs),
+                    cudaMemcpyHostToDevice, stream);
+    cudaMemcpyAsync(d_up_global_w, up_global.data(),
+                    up_global.size() * sizeof(block_iq2_xxs),
+                    cudaMemcpyHostToDevice, stream);
+    cudaMemcpyAsync(d_down_global_w, down_global.data(),
+                    down_global.size() * sizeof(block_q2_K),
+                    cudaMemcpyHostToDevice, stream);
+    cudaMemcpyAsync(d_x, X.data(), X.size() * sizeof(float),
+                    cudaMemcpyHostToDevice, stream);
+    cudaMemcpyAsync(d_ids, remapped_ids.data(),
+                    remapped_ids.size() * sizeof(int32_t),
+                    cudaMemcpyHostToDevice, stream);
+    cudaMemcpyAsync(d_global_ids, global_ids.data(),
+                    global_ids.size() * sizeof(int32_t),
+                    cudaMemcpyHostToDevice, stream);
+    cudaMemcpyAsync(d_router, router_weights.data(),
+                    router_weights.size() * sizeof(float),
+                    cudaMemcpyHostToDevice, stream);
+
+    // A rejected shape must be retryable and must not enqueue writes.  Check
+    // every materialized output, not just the final down buffer, with canaries.
+    cudaMemsetAsync(d_gate_got, 0xA5, mid_count * sizeof(float), stream);
+    cudaMemsetAsync(d_up_got, 0xA5, mid_count * sizeof(float), stream);
+    cudaMemsetAsync(d_mid_got, 0xA5, mid_count * sizeof(float), stream);
+    cudaMemsetAsync(d_down_got, 0xA5, down_count * sizeof(float), stream);
+    const int rc_na = ds4_mmq_iq2_xxs_q2_K_moe_fused_raw(
+        d_gate_w, d_up_w, d_down_w, d_x, d_ids, d_router,
+        d_gate_got, d_up_got, d_mid_got, d_down_got,
+        expert_mid_dim, expert_in_dim, /*out_dim=*/0,
+        n_tokens, compact_experts, n_expert_used, clamp, stream);
+    std::vector<uint8_t> gate_canary(mid_count * sizeof(float));
+    std::vector<uint8_t> up_canary(mid_count * sizeof(float));
+    std::vector<uint8_t> mid_canary(mid_count * sizeof(float));
+    std::vector<uint8_t> down_canary(down_count * sizeof(float));
+    cudaMemcpyAsync(gate_canary.data(), d_gate_got, gate_canary.size(),
+                    cudaMemcpyDeviceToHost, stream);
+    cudaMemcpyAsync(up_canary.data(), d_up_got, up_canary.size(),
+                    cudaMemcpyDeviceToHost, stream);
+    cudaMemcpyAsync(mid_canary.data(), d_mid_got, mid_canary.size(),
+                    cudaMemcpyDeviceToHost, stream);
+    cudaMemcpyAsync(down_canary.data(), d_down_got, down_canary.size(),
+                    cudaMemcpyDeviceToHost, stream);
+    cudaError_t sync_err = cudaStreamSynchronize(stream);
+    const auto is_canary_intact = [](const std::vector<uint8_t> & bytes) {
+        return std::all_of(bytes.begin(), bytes.end(),
+                           [](uint8_t value) { return value == 0xA5; });
+    };
+
+    const bool na_ok = rc_na == DS4_MMQ_NOT_APPLICABLE &&
+        sync_err == cudaSuccess && is_canary_intact(gate_canary) &&
+        is_canary_intact(up_canary) && is_canary_intact(mid_canary) &&
+        is_canary_intact(down_canary);
+
+    cudaMemsetAsync(d_gate_ref, 0, mid_count * sizeof(float), stream);
+    cudaMemsetAsync(d_up_ref, 0, mid_count * sizeof(float), stream);
+    cudaMemsetAsync(d_mid_ref, 0, mid_count * sizeof(float), stream);
+    cudaMemsetAsync(d_down_ref, 0, down_count * sizeof(float), stream);
+    cudaMemsetAsync(d_gate_got, 0, mid_count * sizeof(float), stream);
+    cudaMemsetAsync(d_up_got, 0, mid_count * sizeof(float), stream);
+    cudaMemsetAsync(d_mid_got, 0, mid_count * sizeof(float), stream);
+    cudaMemsetAsync(d_down_got, 0, down_count * sizeof(float), stream);
+    cudaMemsetAsync(d_gate_global, 0, mid_count * sizeof(float), stream);
+    cudaMemsetAsync(d_up_global, 0, mid_count * sizeof(float), stream);
+    cudaMemsetAsync(d_mid_global, 0, mid_count * sizeof(float), stream);
+    cudaMemsetAsync(d_down_global, 0, down_count * sizeof(float), stream);
+
+    const int rc_pair = ds4_mmq_iq2_xxs_moe_pair(
+        d_gate_w, d_up_w, d_x, d_ids, d_gate_ref, d_up_ref,
+        expert_mid_dim, expert_in_dim, n_tokens, compact_experts,
+        n_expert_used, stream);
+    test_swiglu_weighted_f32<<<
+        (unsigned)((mid_count + 255u) / 256u), 256, 0, stream>>>(
+            d_gate_ref, d_up_ref, d_router, d_mid_ref, mid_count,
+            expert_mid_dim, clamp);
+    const cudaError_t swiglu_err = cudaGetLastError();
+    const int rc_down = ds4_mmq_q2_K_moe(
+        d_down_w, d_mid_ref, d_ids, d_down_ref,
+        out_dim, expert_mid_dim, (int)assignments, compact_experts,
+        /*n_expert_used=*/1, stream);
+    uint64_t q81_candidates0 = 0, q81_uses0 = 0, q81_hits0 = 0;
+    uint64_t q81_fallbacks0 = 0, q81_allocations0 = 0, q81_resizes0 = 0;
+    size_t q81_arena0 = 0, q81_high_water0 = 0;
+    uint64_t q81_candidates_off = 0, q81_uses_off = 0, q81_hits_off = 0;
+    uint64_t q81_fallbacks_off = 0, q81_allocations_off = 0;
+    uint64_t q81_resizes_off = 0;
+    size_t q81_arena_off = 0, q81_high_water_off = 0;
+    if (persistent_q81) {
+        ds4_mmq_q81_persistent_counters(
+            &q81_candidates0, &q81_uses0, &q81_hits0,
+            &q81_fallbacks0, &q81_allocations0, &q81_resizes0,
+            &q81_arena0, &q81_high_water0);
+    }
+    const int rc_fused = ds4_mmq_iq2_xxs_q2_K_moe_fused_raw(
+        d_gate_w, d_up_w, d_down_w, d_x, d_ids, d_router,
+        d_gate_got, d_up_got, d_mid_got, d_down_got,
+        expert_mid_dim, expert_in_dim, out_dim,
+        n_tokens, compact_experts, n_expert_used, clamp, stream);
+    if (persistent_q81) {
+        ds4_mmq_q81_persistent_counters(
+            &q81_candidates_off, &q81_uses_off, &q81_hits_off,
+            &q81_fallbacks_off, &q81_allocations_off, &q81_resizes_off,
+            &q81_arena_off, &q81_high_water_off);
+        if (!q81_env.set("1")) {
+            cleanup();
+            return false;
+        }
+    }
+    // Run the same fused path against the original global expert table and
+    // unremapped ids.  Bitwise equality with the compact result validates the
+    // full-expert copies and, with the multi-block shape above, both raw
+    // channel strides independently of the materialized compact reference.
+    const int rc_global = ds4_mmq_iq2_xxs_q2_K_moe_fused_raw(
+        d_gate_global_w, d_up_global_w, d_down_global_w,
+        d_x, d_global_ids, d_router,
+        d_gate_global, d_up_global, d_mid_global, d_down_global,
+        expert_mid_dim, expert_in_dim, out_dim,
+        n_tokens, global_experts, n_expert_used, clamp, stream);
+    // Traverse the real acquire/grow path with a deterministic requirement
+    // just above the lazy minimum.  This drains the first dispatch before
+    // retiring its arena without making the parity fixture itself enormous.
+    constexpr size_t q81_growth_required =
+        4u * 1024u * 1024u + 257u;
+    const int rc_q81_grow = persistent_q81
+        ? ds4_mmq_q81_persistent_preflight_for_test(
+              persistent_device, q81_growth_required)
+        : 0;
+    // The second identical dispatch must reuse the same owned arena and raises
+    // the hit counter; default-stream order makes overwriting its Q8 input safe.
+    const int rc_global_reuse = persistent_q81
+        ? ds4_mmq_iq2_xxs_q2_K_moe_fused_raw(
+              d_gate_global_w, d_up_global_w, d_down_global_w,
+              d_x, d_global_ids, d_router,
+              d_gate_global, d_up_global, d_mid_global, d_down_global,
+              expert_mid_dim, expert_in_dim, out_dim,
+              n_tokens, global_experts, n_expert_used, clamp, stream)
+        : 0;
+
+    std::vector<float> gate_ref(mid_count), up_ref(mid_count),
+                       mid_ref(mid_count), down_ref(down_count);
+    std::vector<float> gate_got(mid_count), up_got(mid_count),
+                       mid_got(mid_count), down_got(down_count);
+    std::vector<float> gate_global_out(mid_count), up_global_out(mid_count),
+                       mid_global_out(mid_count), down_global_out(down_count);
+    cudaMemcpyAsync(gate_ref.data(), d_gate_ref, mid_count * sizeof(float),
+                    cudaMemcpyDeviceToHost, stream);
+    cudaMemcpyAsync(up_ref.data(), d_up_ref, mid_count * sizeof(float),
+                    cudaMemcpyDeviceToHost, stream);
+    cudaMemcpyAsync(mid_ref.data(), d_mid_ref, mid_count * sizeof(float),
+                    cudaMemcpyDeviceToHost, stream);
+    cudaMemcpyAsync(down_ref.data(), d_down_ref, down_count * sizeof(float),
+                    cudaMemcpyDeviceToHost, stream);
+    cudaMemcpyAsync(gate_got.data(), d_gate_got, mid_count * sizeof(float),
+                    cudaMemcpyDeviceToHost, stream);
+    cudaMemcpyAsync(up_got.data(), d_up_got, mid_count * sizeof(float),
+                    cudaMemcpyDeviceToHost, stream);
+    cudaMemcpyAsync(mid_got.data(), d_mid_got, mid_count * sizeof(float),
+                    cudaMemcpyDeviceToHost, stream);
+    cudaMemcpyAsync(down_got.data(), d_down_got, down_count * sizeof(float),
+                    cudaMemcpyDeviceToHost, stream);
+    cudaMemcpyAsync(gate_global_out.data(), d_gate_global,
+                    mid_count * sizeof(float), cudaMemcpyDeviceToHost, stream);
+    cudaMemcpyAsync(up_global_out.data(), d_up_global,
+                    mid_count * sizeof(float), cudaMemcpyDeviceToHost, stream);
+    cudaMemcpyAsync(mid_global_out.data(), d_mid_global,
+                    mid_count * sizeof(float), cudaMemcpyDeviceToHost, stream);
+    cudaMemcpyAsync(down_global_out.data(), d_down_global,
+                    down_count * sizeof(float), cudaMemcpyDeviceToHost, stream);
+    sync_err = cudaStreamSynchronize(stream);
+
+    uint64_t q81_candidates1 = 0, q81_uses1 = 0, q81_hits1 = 0;
+    uint64_t q81_fallbacks1 = 0, q81_allocations1 = 0, q81_resizes1 = 0;
+    size_t q81_arena1 = 0, q81_high_water1 = 0;
+    if (persistent_q81) {
+        ds4_mmq_q81_persistent_counters(
+            &q81_candidates1, &q81_uses1, &q81_hits1,
+            &q81_fallbacks1, &q81_allocations1, &q81_resizes1,
+            &q81_arena1, &q81_high_water1);
+        ds4_mmq_q81_persistent_report();
+    }
+    const size_t q81_growth_aligned =
+        (q81_growth_required + 255u) & ~(size_t)255u;
+    const bool q81_counters_ok = !persistent_q81 ||
+        (q81_lazy_init_rc == 0 &&
+         q81_init_allocations1 == q81_init_allocations0 &&
+         q81_init_resizes1 == q81_init_resizes0 &&
+         q81_init_arena0 == 0 && q81_init_arena1 == 0 &&
+         q81_candidates_off == q81_candidates0 &&
+         q81_uses_off == q81_uses0 && q81_hits_off == q81_hits0 &&
+         q81_fallbacks_off == q81_fallbacks0 &&
+         q81_allocations_off == q81_allocations0 &&
+         q81_resizes_off == q81_resizes0 &&
+         q81_arena_off == 0 &&
+         q81_candidates1 - q81_candidates_off == 3u &&
+         q81_uses1 - q81_uses_off == 3u &&
+         q81_hits1 - q81_hits_off == 1u &&
+         q81_fallbacks1 == q81_fallbacks_off &&
+         q81_allocations1 - q81_allocations_off == 2u &&
+         q81_resizes1 - q81_resizes_off == 1u &&
+         q81_arena1 == q81_growth_aligned &&
+         q81_high_water1 >= q81_growth_required &&
+         q81_high_water1 >= q81_high_water0);
+    if (persistent_q81) {
+        fprintf(stderr,
+                "q81 persistent lazy_init=%d arena=%zu->%zu alloc=%llu->%llu "
+                "resize=%llu->%llu "
+                "delta(off c/u/h/f/a/r)=%llu/%llu/%llu/%llu/%llu/%llu "
+                "delta(on c/u/h/f/a/r)=%llu/%llu/%llu/%llu/%llu/%llu "
+                "arena=%zu high=%zu\n",
+                q81_lazy_init_rc, q81_init_arena0, q81_init_arena1,
+                (unsigned long long)q81_init_allocations0,
+                (unsigned long long)q81_init_allocations1,
+                (unsigned long long)q81_init_resizes0,
+                (unsigned long long)q81_init_resizes1,
+                (unsigned long long)(q81_candidates_off - q81_candidates0),
+                (unsigned long long)(q81_uses_off - q81_uses0),
+                (unsigned long long)(q81_hits_off - q81_hits0),
+                (unsigned long long)(q81_fallbacks_off - q81_fallbacks0),
+                (unsigned long long)(q81_allocations_off - q81_allocations0),
+                (unsigned long long)(q81_resizes_off - q81_resizes0),
+                (unsigned long long)(q81_candidates1 - q81_candidates_off),
+                (unsigned long long)(q81_uses1 - q81_uses_off),
+                (unsigned long long)(q81_hits1 - q81_hits_off),
+                (unsigned long long)(q81_fallbacks1 - q81_fallbacks_off),
+                (unsigned long long)(q81_allocations1 - q81_allocations_off),
+                (unsigned long long)(q81_resizes1 - q81_resizes_off),
+                q81_arena1, q81_high_water1);
+    }
+    if (persistent_q81) {
+        teardown_persistent();
+    }
+
+    const auto mismatches = [](const std::vector<float> & a,
+                               const std::vector<float> & b) {
+        size_t bad = 0;
+        for (size_t i = 0; i < a.size(); i++) {
+            if (std::memcmp(&a[i], &b[i], sizeof(float)) != 0) bad++;
+        }
+        return bad;
+    };
+    const size_t gate_bad = mismatches(gate_got, gate_ref);
+    const size_t up_bad = mismatches(up_got, up_ref);
+    const size_t mid_bad = mismatches(mid_got, mid_ref);
+    const size_t down_bad = mismatches(down_got, down_ref);
+    const size_t gate_remap_bad = mismatches(gate_got, gate_global_out);
+    const size_t up_remap_bad = mismatches(up_got, up_global_out);
+    const size_t mid_remap_bad = mismatches(mid_got, mid_global_out);
+    const size_t down_remap_bad = mismatches(down_got, down_global_out);
+
+    // The comparisons above use memcmp, i.e. bit-exact equality with no
+    // tolerance.  When a case fails, the counts alone cannot tell a last-bit
+    // rounding difference from a real divergence, which makes triage on a new
+    // architecture guesswork.  Report the magnitude next to the count: worst
+    // absolute and relative difference, ULP distance, how many differences are
+    // within a single ULP, how many exceed 1e-3, and the value pair at the
+    // worst point.  Diagnostics only - the pass/fail criterion is unchanged.
+    const auto diagnose = [](const char *tag, const std::vector<float> & a,
+                             const std::vector<float> & b) {
+        double max_abs = 0.0, max_rel = 0.0;
+        float at_max_got = 0.0f, at_max_ref = 0.0f;
+        size_t bad = 0, first = (size_t)-1, ulp1 = 0, nonfinite = 0, big = 0;
+        long long max_ulp = 0;
+        for (size_t i = 0; i < a.size(); i++) {
+            if (std::memcmp(&a[i], &b[i], sizeof(float)) == 0) continue;
+            bad++;
+            if (first == (size_t)-1) first = i;
+            if (!std::isfinite(a[i]) || !std::isfinite(b[i])) nonfinite++;
+            const double d = std::fabs((double)a[i] - (double)b[i]);
+            const double den = std::fabs((double)b[i]);
+            if (d > max_abs) { max_abs = d; at_max_got = a[i]; at_max_ref = b[i]; }
+            if (den > 0.0 && d / den > max_rel) max_rel = d / den;
+            if (d > 1e-3) big++;
+            // Monotonic ordering of the float bit patterns, so the subtraction
+            // is a real ULP distance across the sign boundary as well.
+            int32_t ia = 0, ib = 0;
+            std::memcpy(&ia, &a[i], sizeof(int32_t));
+            std::memcpy(&ib, &b[i], sizeof(int32_t));
+            if (ia < 0) ia = (int32_t)0x80000000 - ia;
+            if (ib < 0) ib = (int32_t)0x80000000 - ib;
+            const long long u = std::llabs((long long)ia - (long long)ib);
+            if (u <= 1) ulp1++;
+            if (u > max_ulp) max_ulp = u;
+        }
+        if (bad == 0) {
+            fprintf(stderr, "  diag %-11s clean\n", tag);
+            return;
+        }
+        fprintf(stderr,
+                "  diag %-11s bad=%zu/%zu (%.1f%%) max_abs=%.4g max_rel=%.4g "
+                "max_ulp=%lld within_1ulp=%zu nonfinite=%zu abs_gt_1e-3=%zu "
+                "at_max(got/ref)=%.9g/%.9g first=%zu got=%.9g ref=%.9g\n",
+                tag, bad, a.size(), 100.0 * (double)bad / (double)a.size(),
+                max_abs, max_rel, max_ulp, ulp1, nonfinite, big,
+                (double)at_max_got, (double)at_max_ref,
+                first, (double)a[first], (double)b[first]);
+    };
+    diagnose("gate", gate_got, gate_ref);
+    diagnose("up", up_got, up_ref);
+    diagnose("mid", mid_got, mid_ref);
+    diagnose("down", down_got, down_ref);
+    diagnose("gate/remap", gate_got, gate_global_out);
+    diagnose("up/remap", up_got, up_global_out);
+    diagnose("mid/remap", mid_got, mid_global_out);
+    diagnose("down/remap", down_got, down_global_out);
+    const bool ok = na_ok && rc_pair == 0 && swiglu_err == cudaSuccess &&
+        rc_down == 0 && rc_fused == 0 && rc_global == 0 &&
+        rc_global_reuse == 0 && rc_q81_grow == 0 &&
+        initial_arena_cleanup == 0 &&
+        final_arena_cleanup == 0 && q81_env_restore_ok && q81_counters_ok &&
+        sync_err == cudaSuccess && gate_bad == 0 && up_bad == 0 &&
+        mid_bad == 0 && down_bad == 0 && gate_remap_bad == 0 &&
+        up_remap_bad == 0 && mid_remap_bad == 0 && down_remap_bad == 0;
+    fprintf(stderr,
+            "rc_na=%d canary=%s rc_pair=%d swiglu=%s rc_down=%d "
+            "rc_fused=%d rc_global=%d rc_grow=%d rc_reuse=%d "
+            "mismatches(g/u/m/d)=%zu/%zu/%zu/%zu "
+            "remap_mismatches(g/u/m/d)=%zu/%zu/%zu/%zu sync=%s\n%s\n\n",
+        rc_na, na_ok ? "intact" : "FAILED", rc_pair,
+            cudaGetErrorString(swiglu_err), rc_down, rc_fused, rc_global,
+            rc_q81_grow, rc_global_reuse,
+            gate_bad, up_bad, mid_bad, down_bad,
+            gate_remap_bad, up_remap_bad, mid_remap_bad, down_remap_bad,
+            cudaGetErrorString(sync_err), ok ? "PASS" : "FAIL");
+
+    cleanup();
+    return ok;
+}
+
+bool run_iq2_xxs_q2_K_fused_raw_persistent_gb10_parity(
+        int n_tokens, uint32_t seed) {
+    return run_iq2_xxs_q2_K_fused_raw_parity(
+        n_tokens, seed, /*persistent_q81=*/true);
+}
+
 bool run_q4_K_moe(int M, int K, int nt, int ne, int nu, uint32_t seed) {
     auto fn = [](block_q4_K * blk, float * out,
                  int n_experts, int M, int K, int blocks_per_expert,
@@ -1142,15 +3039,553 @@ bool run_q8_0_dense_vec(int M, int N, int K, uint32_t seed) {
     return ok;
 }
 
+bool run_q4_K_dense_vec_gb10_parity(
+        int M, int N, int K, uint32_t seed, bool persistent_k1024) {
+    fprintf(stderr,
+            "=== Q4_K/DENSE_VEC_%s  M=%d N=%d K=%d  seed=%u ===\n",
+            persistent_k1024 ? "PERSISTENT" : "SCRATCH",
+            M, N, K, seed);
+
+    std::mt19937 rng(seed);
+    std::normal_distribution<float> nd(0.0f, 1.0f);
+    const int blocks_per_row = K / QK_K_LOCAL;
+    std::vector<block_q4_K> W((size_t)M * blocks_per_row);
+    for (auto &blk : W) generate_random_block_q4_K(&blk, rng);
+    std::vector<float> X((size_t)N * K);
+    for (float &v : X) v = nd(rng);
+
+    cudaStream_t stream = nullptr;
+    void *dW = nullptr;
+    void *scratch = nullptr;
+    float *dX = nullptr;
+    float *dRef = nullptr;
+    float *dGot = nullptr;
+    bool ok = cudaStreamCreate(&stream) == cudaSuccess &&
+              cudaMalloc(&dW, W.size() * sizeof(block_q4_K)) == cudaSuccess &&
+              cudaMalloc(&dX, X.size() * sizeof(float)) == cudaSuccess &&
+              cudaMalloc(&dRef, (size_t)M * N * sizeof(float)) == cudaSuccess &&
+              cudaMalloc(&dGot, (size_t)M * N * sizeof(float)) == cudaSuccess &&
+              cudaMalloc(&scratch, 256u * 1024u) == cudaSuccess;
+    if (!ok) {
+        fprintf(stderr, "Q4_K dense vec parity allocation failed\n");
+        if (scratch) cudaFree(scratch);
+        if (dGot) cudaFree(dGot);
+        if (dRef) cudaFree(dRef);
+        if (dX) cudaFree(dX);
+        if (dW) cudaFree(dW);
+        if (stream) cudaStreamDestroy(stream);
+        return false;
+    }
+    cudaMemcpyAsync(dW, W.data(), W.size() * sizeof(block_q4_K),
+                    cudaMemcpyHostToDevice, stream);
+    cudaMemcpyAsync(dX, X.data(), X.size() * sizeof(float),
+                    cudaMemcpyHostToDevice, stream);
+
+    uint64_t candidates0 = 0, uses0 = 0, fallbacks0 = 0;
+    uint64_t require_failures0 = 0, oracle_calls0 = 0;
+    uint64_t oracle_mismatches0 = 0, oracle_skips0 = 0;
+    ds4_mmq_q4_K_k1024_persistent_counters(
+        &candidates0, &uses0, &fallbacks0, &require_failures0,
+        &oracle_calls0, &oracle_mismatches0, &oracle_skips0);
+
+    unsetenv("DS4_CUDA_NO_Q4_GB10_FAST");
+    unsetenv("DS4_CUDA_NO_Q4_DENSE_SCRATCH");
+    unsetenv("DS4_CUDA_NO_Q4_K1024_PERSISTENT");
+    unsetenv("DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT");
+    unsetenv("DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT");
+    unsetenv("DS4_CUDA_Q4_K1024_PERSISTENT_ORACLE");
+    ds4_mmq_set_gb10_optimizations(persistent_k1024 ? 1 : 0);
+    ds4_mmq_set_aligned_q81_scratch(
+        persistent_k1024 ? scratch : nullptr,
+        persistent_k1024 ? 256u * 1024u : 0u);
+    /* The reference must remain canonical even after a future default-on
+     * promotion.  The authoritative kill switch makes this a real
+     * reference-vs-candidate comparison rather than candidate-vs-candidate. */
+    if (persistent_k1024) {
+        setenv("DS4_CUDA_NO_Q4_K1024_PERSISTENT", "1", 1);
+    }
+    const int rc_ref = ds4_mmq_q4_K_dense_vec(
+        dW, dX, dRef, M, N, K, stream);
+    unsetenv("DS4_CUDA_NO_Q4_K1024_PERSISTENT");
+
+    ds4_mmq_set_gb10_optimizations(1);
+    ds4_mmq_set_aligned_q81_scratch(scratch, 256u * 1024u);
+    if (persistent_k1024) {
+        setenv("DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT", "1", 1);
+        setenv("DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT", "1", 1);
+    }
+    const int rc_got = ds4_mmq_q4_K_dense_vec(
+        dW, dX, dGot, M, N, K, stream);
+
+    std::vector<float> ref((size_t)M * N);
+    std::vector<float> got((size_t)M * N);
+    cudaMemcpyAsync(ref.data(), dRef, ref.size() * sizeof(float),
+                    cudaMemcpyDeviceToHost, stream);
+    cudaMemcpyAsync(got.data(), dGot, got.size() * sizeof(float),
+                    cudaMemcpyDeviceToHost, stream);
+    int rc_required_disabled = 0;
+    int rc_oracle = 0;
+    int rc_nonresident_fallback = 0;
+    int rc_nonresident_required = 0;
+    cudaError_t nonresident_guard_setup_err = cudaSuccess;
+    std::vector<float> oracle((size_t)M * N);
+    std::vector<float> nonresident_fallback((size_t)M * N);
+    std::vector<unsigned char> nonresident_guard(
+        (size_t)M * N * sizeof(float));
+    if (persistent_k1024) {
+        setenv("DS4_CUDA_NO_Q4_K1024_PERSISTENT", "1", 1);
+        rc_required_disabled = ds4_mmq_q4_K_dense_vec(
+            dW, dX, dGot, M, N, K, stream);
+        unsetenv("DS4_CUDA_NO_Q4_K1024_PERSISTENT");
+        setenv("DS4_CUDA_Q4_K1024_PERSISTENT_ORACLE", "1", 1);
+        rc_oracle = ds4_mmq_q4_K_dense_vec(
+            dW, dX, dGot, M, N, K, stream);
+        unsetenv("DS4_CUDA_Q4_K1024_PERSISTENT_ORACLE");
+        cudaMemcpyAsync(oracle.data(), dGot, oracle.size() * sizeof(float),
+                        cudaMemcpyDeviceToHost, stream);
+
+        /* Simulate the full runtime resolving W from mapped host/HMM rather
+         * than a cudaMalloc cache. ENABLE must fall back to canonical MMVQ;
+         * REQUIRE must reject before enqueue and leave the sentinel intact. */
+        unsetenv("DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT");
+        rc_nonresident_fallback =
+            ds4_mmq_q4_K_dense_vec_with_weight_residency(
+                dW, dX, dGot, M, N, K,
+                /*weight_device_resident=*/0, stream);
+        cudaMemcpyAsync(nonresident_fallback.data(), dGot,
+                        nonresident_fallback.size() * sizeof(float),
+                        cudaMemcpyDeviceToHost, stream);
+        cudaMemsetAsync(dGot, 0xa5, (size_t)M * N * sizeof(float), stream);
+        nonresident_guard_setup_err = cudaStreamSynchronize(stream);
+
+        setenv("DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT", "1", 1);
+        rc_nonresident_required =
+            ds4_mmq_q4_K_dense_vec_with_weight_residency(
+                dW, dX, dGot, M, N, K,
+                /*weight_device_resident=*/0, stream);
+        cudaMemcpyAsync(nonresident_guard.data(), dGot,
+                        nonresident_guard.size(), cudaMemcpyDeviceToHost,
+                        stream);
+    }
+    const cudaError_t sync_err = cudaStreamSynchronize(stream);
+    size_t mismatches = 0;
+    size_t oracle_output_mismatches = 0;
+    size_t nonresident_fallback_mismatches = 0;
+    size_t nonresident_guard_mismatches = 0;
+    for (size_t i = 0; i < ref.size(); i++) {
+        if (std::memcmp(&ref[i], &got[i], sizeof(float)) != 0) mismatches++;
+        if (persistent_k1024 &&
+            std::memcmp(&ref[i], &oracle[i], sizeof(float)) != 0) {
+            oracle_output_mismatches++;
+        }
+        if (persistent_k1024 &&
+            std::memcmp(&ref[i], &nonresident_fallback[i],
+                        sizeof(float)) != 0) {
+            nonresident_fallback_mismatches++;
+        }
+    }
+    if (persistent_k1024) {
+        for (unsigned char value : nonresident_guard) {
+            if (value != 0xa5u) nonresident_guard_mismatches++;
+        }
+    }
+
+    uint64_t candidates1 = 0, uses1 = 0, fallbacks1 = 0;
+    uint64_t require_failures1 = 0, oracle_calls1 = 0;
+    uint64_t oracle_mismatches1 = 0, oracle_skips1 = 0;
+    ds4_mmq_q4_K_k1024_persistent_counters(
+        &candidates1, &uses1, &fallbacks1, &require_failures1,
+        &oracle_calls1, &oracle_mismatches1, &oracle_skips1);
+    const bool counter_ok = !persistent_k1024 ||
+        (candidates1 - candidates0 >= 6u &&
+         uses1 - uses0 >= 2u &&
+         fallbacks1 - fallbacks0 >= 4u &&
+         require_failures1 - require_failures0 >= 2u &&
+         oracle_calls1 - oracle_calls0 >= 1u &&
+         oracle_mismatches1 == oracle_mismatches0 &&
+         oracle_skips1 == oracle_skips0);
+    ok = rc_ref == 0 && rc_got == 0 &&
+         (!persistent_k1024 || rc_required_disabled != 0) &&
+         (!persistent_k1024 || rc_oracle == 0) &&
+         (!persistent_k1024 || rc_nonresident_fallback == 0) &&
+         (!persistent_k1024 || rc_nonresident_required != 0) &&
+         nonresident_guard_setup_err == cudaSuccess &&
+         sync_err == cudaSuccess &&
+         mismatches == 0 && oracle_output_mismatches == 0 &&
+         nonresident_fallback_mismatches == 0 &&
+         nonresident_guard_mismatches == 0 && counter_ok;
+    fprintf(stderr,
+            "rc_ref=%d rc_candidate=%d rc_required_disabled=%d "
+            "rc_oracle=%d rc_nonresident_fallback=%d "
+            "rc_nonresident_required=%d mismatches=%zu "
+            "oracle_output_mismatches=%zu "
+            "nonresident_fallback_mismatches=%zu "
+            "nonresident_guard_mismatches=%zu "
+            "counter_delta=%llu/%llu/%llu/%llu/%llu/%llu/%llu "
+            "guard_setup=%s sync=%s\n%s\n\n",
+            rc_ref, rc_got, rc_required_disabled, rc_oracle,
+            rc_nonresident_fallback, rc_nonresident_required, mismatches,
+            oracle_output_mismatches, nonresident_fallback_mismatches,
+            nonresident_guard_mismatches,
+            (unsigned long long)(candidates1 - candidates0),
+            (unsigned long long)(uses1 - uses0),
+            (unsigned long long)(fallbacks1 - fallbacks0),
+            (unsigned long long)(require_failures1 - require_failures0),
+            (unsigned long long)(oracle_calls1 - oracle_calls0),
+            (unsigned long long)(oracle_mismatches1 - oracle_mismatches0),
+            (unsigned long long)(oracle_skips1 - oracle_skips0),
+            cudaGetErrorString(nonresident_guard_setup_err),
+            cudaGetErrorString(sync_err),
+            ok ? "PASS" : "FAIL");
+
+    unsetenv("DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT");
+    unsetenv("DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT");
+    unsetenv("DS4_CUDA_Q4_K1024_PERSISTENT_ORACLE");
+    ds4_mmq_set_aligned_q81_scratch(nullptr, 0u);
+    ds4_mmq_set_gb10_optimizations(0);
+    cudaFree(scratch);
+    cudaFree(dGot);
+    cudaFree(dRef);
+    cudaFree(dX);
+    cudaFree(dW);
+    cudaStreamDestroy(stream);
+    return ok;
+}
+
+bool run_q4_K_grouped_vec_parity(
+        int M, int N, int K, int n_groups, uint32_t seed) {
+    fprintf(stderr,
+            "=== Q4_K/GROUPED_VEC  M=%d N=%d K=%d groups=%d seed=%u ===\n",
+            M, N, K, n_groups, seed);
+    std::mt19937 rng(seed);
+    std::normal_distribution<float> nd(0.0f, 1.0f);
+    const int blocks_per_row = K / QK_K_LOCAL;
+    const size_t blocks_per_group = (size_t)M * blocks_per_row;
+    std::vector<block_q4_K> W((size_t)n_groups * blocks_per_group);
+    for (auto &blk : W) generate_random_block_q4_K(&blk, rng);
+    std::vector<float> X((size_t)N * n_groups * K);
+    for (float &v : X) v = nd(rng);
+
+    cudaStream_t stream = nullptr;
+    void *dW = nullptr;
+    void *scratch = nullptr;
+    float *dX = nullptr;
+    float *dRef = nullptr;
+    float *dGot = nullptr;
+    const size_t output_count = (size_t)N * n_groups * M;
+    /* Covers the N=8, G=16, K=4096 parity envelope with room for ids. */
+    const size_t scratch_bytes = 1024u * 1024u;
+    bool ok = cudaStreamCreate(&stream) == cudaSuccess &&
+              cudaMalloc(&dW, W.size() * sizeof(block_q4_K)) == cudaSuccess &&
+              cudaMalloc(&dX, X.size() * sizeof(float)) == cudaSuccess &&
+              cudaMalloc(&dRef, output_count * sizeof(float)) == cudaSuccess &&
+              cudaMalloc(&dGot, output_count * sizeof(float)) == cudaSuccess &&
+              cudaMalloc(&scratch, scratch_bytes) == cudaSuccess;
+    if (!ok) {
+        fprintf(stderr, "Q4_K grouped vec parity allocation failed\n");
+        if (scratch) cudaFree(scratch);
+        if (dGot) cudaFree(dGot);
+        if (dRef) cudaFree(dRef);
+        if (dX) cudaFree(dX);
+        if (dW) cudaFree(dW);
+        if (stream) cudaStreamDestroy(stream);
+        return false;
+    }
+    cudaMemcpyAsync(dW, W.data(), W.size() * sizeof(block_q4_K),
+                    cudaMemcpyHostToDevice, stream);
+    cudaMemcpyAsync(dX, X.data(), X.size() * sizeof(float),
+                    cudaMemcpyHostToDevice, stream);
+    unsetenv("DS4_CUDA_NO_Q4_GB10_FAST");
+    unsetenv("DS4_CUDA_NO_Q4_GROUPED_ATTN_A");
+    unsetenv("DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH");
+    unsetenv("DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_BATCH");
+    ds4_mmq_set_gb10_optimizations(1);
+    ds4_mmq_set_aligned_q81_scratch(scratch, scratch_bytes);
+
+    int rc_ref = 0;
+    for (int t = 0; t < N && rc_ref == 0; t++) {
+        for (int g = 0; g < n_groups && rc_ref == 0; g++) {
+            const size_t channel = (size_t)t * n_groups + g;
+            rc_ref = ds4_mmq_q4_K_dense_vec(
+                (const char *)dW + (size_t)g * blocks_per_group *
+                    sizeof(block_q4_K),
+                dX + channel * K,
+                dRef + channel * M,
+                M, 1, K, stream);
+        }
+    }
+
+    int rc_opt_out = DS4_MMQ_NOT_APPLICABLE;
+    int rc_short_scratch = DS4_MMQ_NOT_APPLICABLE;
+    if (N > 1) {
+        rc_opt_out = ds4_mmq_q4_K_grouped_batch_vec(
+            dW, dX, dGot, M, K, N, n_groups, stream);
+        setenv("DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_BATCH", "1", 1);
+        const size_t padded_k = ((size_t)K + 511u) & ~(size_t)511u;
+        const size_t q8_row_bytes =
+            padded_k * sizeof(block_q8_1) / QK8_1;
+        const size_t q8_bytes = (size_t)N * n_groups * q8_row_bytes;
+        const size_t ids_offset = (q8_bytes + 15u) & ~(size_t)15u;
+        const size_t required_bytes =
+            ids_offset + (size_t)N * n_groups * sizeof(int32_t);
+        if (required_bytes > 0u) {
+            ds4_mmq_set_aligned_q81_scratch(scratch, required_bytes - 1u);
+            rc_short_scratch = ds4_mmq_q4_K_grouped_batch_vec(
+                dW, dX, dGot, M, K, N, n_groups, stream);
+            ds4_mmq_set_aligned_q81_scratch(scratch, scratch_bytes);
+        }
+    }
+    const int rc_got = N == 1
+        ? ds4_mmq_q4_K_grouped_vec(
+              dW, dX, dGot, M, K, n_groups, stream)
+        : ds4_mmq_q4_K_grouped_batch_vec(
+              dW, dX, dGot, M, K, N, n_groups, stream);
+    if (N == 1) {
+        setenv("DS4_CUDA_NO_Q4_GB10_FAST", "1", 1);
+    } else {
+        setenv("DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH", "1", 1);
+    }
+    const int rc_disabled = N == 1
+        ? ds4_mmq_q4_K_grouped_vec(
+              dW, dX, dGot, M, K, n_groups, stream)
+        : ds4_mmq_q4_K_grouped_batch_vec(
+              dW, dX, dGot, M, K, N, n_groups, stream);
+    unsetenv("DS4_CUDA_NO_Q4_GB10_FAST");
+    unsetenv("DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH");
+
+    std::vector<float> ref(output_count);
+    std::vector<float> got(output_count);
+    cudaMemcpyAsync(ref.data(), dRef, ref.size() * sizeof(float),
+                    cudaMemcpyDeviceToHost, stream);
+    cudaMemcpyAsync(got.data(), dGot, got.size() * sizeof(float),
+                    cudaMemcpyDeviceToHost, stream);
+    const cudaError_t sync_err = cudaStreamSynchronize(stream);
+    size_t mismatches = 0;
+    for (size_t i = 0; i < ref.size(); i++) {
+        if (std::memcmp(&ref[i], &got[i], sizeof(float)) != 0) mismatches++;
+    }
+    ok = rc_ref == 0 && rc_got == 0 &&
+         rc_opt_out == DS4_MMQ_NOT_APPLICABLE &&
+         rc_short_scratch == DS4_MMQ_NOT_APPLICABLE &&
+         rc_disabled == DS4_MMQ_NOT_APPLICABLE &&
+         sync_err == cudaSuccess &&
+         mismatches == 0;
+    fprintf(stderr,
+            "rc_ref=%d rc_grouped=%d rc_opt_out=%d rc_short_scratch=%d "
+            "rc_disabled=%d "
+            "mismatches=%zu sync=%s\n%s\n\n",
+            rc_ref, rc_got, rc_opt_out, rc_short_scratch, rc_disabled, mismatches,
+            cudaGetErrorString(sync_err),
+            ok ? "PASS" : "FAIL");
+
+    unsetenv("DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_BATCH");
+    ds4_mmq_set_aligned_q81_scratch(nullptr, 0u);
+    ds4_mmq_set_gb10_optimizations(0);
+    cudaFree(scratch);
+    cudaFree(dGot);
+    cudaFree(dRef);
+    cudaFree(dX);
+    cudaFree(dW);
+    cudaStreamDestroy(stream);
+    return ok;
+}
+
 } // namespace
 
 int main(int argc, char ** argv) {
-    (void)argc; (void)argv;
+    const bool q4_16warp_oracle =
+        argc == 2 && std::strcmp(argv[1], "--q4-16warp") == 0;
+#if !defined(GGML_USE_HIP)
+    const bool q4_grouped_q81_oracle =
+        argc == 2 && std::strcmp(argv[1], "--q4-grouped-q81") == 0;
+#endif
+    scoped_env_override require_16warp(
+        "DS4_CUDA_REQUIRE_Q4_MMQ_16WARP");
+    scoped_env_override disable_16warp(
+        "DS4_CUDA_NO_Q4_MMQ_16WARP");
+    // Production dense/pair oracles must fail closed if any leg falls back.
+    // Set the required mode before initialization can observe its process-wide
+    // cache, and neutralize an inherited rollback request.
+    if (q4_16warp_oracle &&
+        (!require_16warp.set("1") || !disable_16warp.set("0"))) {
+        fprintf(stderr, "Q4 16-warp oracle environment setup failed\n");
+        return 1;
+    }
+
     int rc = ds4_mmq_init(0);
     if (rc != 0) { fprintf(stderr, "ds4_mmq_init failed: %d\n", rc); return 1; }
 
     bool all_ok = true;
 
+#if !defined(GGML_USE_HIP)
+    if (q4_grouped_q81_oracle) {
+        all_ok &= run_q4_K_grouped_q8_1_kernel_suite();
+        fprintf(stderr, "===================\n");
+        fprintf(stderr, "Q4 GROUPED Q8_1 %s\n",
+                all_ok ? "PASS" : "FAILED");
+        return all_ok ? 0 : 1;
+    }
+#endif
+
+    if (q4_16warp_oracle) {
+        // The canonical production baseline must select mmq_x=128. Mirror
+        // get_mmq_x_max_host's numeric-prefix parsing and reject a narrower
+        // experiment override before MMQ caches it.
+        const char *const mmq_x_env = std::getenv("DS4_CUDA_MMQ_X_MAX");
+        if (mmq_x_env && mmq_x_env[0]) {
+            char *end = nullptr;
+            const long value = std::strtol(mmq_x_env, &end, 10);
+            if (end != mmq_x_env && value >= 8 &&
+                (value > INT32_MAX || (value / 8) * 8 < 128)) {
+                fprintf(stderr,
+                        "Q4 16-warp oracle requires DS4_CUDA_MMQ_X_MAX>=128 "
+                        "(got %s)\n",
+                        mmq_x_env);
+                return 1;
+            }
+        }
+
+        int device = -1;
+        cudaDeviceProp prop = {};
+        const cudaError_t device_err = cudaGetDevice(&device);
+        const cudaError_t prop_err = device_err == cudaSuccess
+            ? cudaGetDeviceProperties(&prop, device) : device_err;
+        if (prop_err != cudaSuccess || prop.multiProcessorCount <= 0) {
+            fprintf(stderr, "Q4 16-warp geometry query failed: %s\n",
+                    cudaGetErrorString(prop_err));
+            return 1;
+        }
+        const int cc = prop.major * 100 + prop.minor * 10;
+        if (!ds4_mmq_q4_K_dense_16warp_available(cc)) {
+            fprintf(stderr,
+                    "Q4 16-warp oracle unsupported on cc=%d.%d\n",
+                    prop.major, prop.minor);
+            return 77;
+        }
+        if (!ds4_mmq_q4_K_dense_16warp_supported(
+                cc, /*M=*/4096, /*N=*/768, /*K=*/8192) ||
+            ds4_mmq_q4_K_dense_16warp_supported(
+                cc, /*M=*/4096, /*N=*/768, /*K=*/8448)) {
+            fprintf(stderr,
+                    "Q4 16-warp K=8192 admission envelope is invalid\n");
+            return 1;
+        }
+        const int prepare_rc = ds4_mmq_q4_K_dense_16warp_prepare();
+        if (prepare_rc != 0) {
+            fprintf(stderr,
+                    "Q4 16-warp oracle prepare failed: %d\n", prepare_rc);
+            return 1;
+        }
+
+        const auto grid_efficiency = [&](int M, int N) {
+            const int64_t tiles_m = ((int64_t)M + 127) / 128;
+            const int64_t tiles_n = ((int64_t)N + 127) / 128;
+            const int64_t tiles = tiles_m * tiles_n;
+            const int64_t waves =
+                (tiles + prop.multiProcessorCount - 1) /
+                prop.multiProcessorCount;
+            return (int)(100 * tiles /
+                         ((int64_t)prop.multiProcessorCount * waves));
+        };
+        const int dense_4096_eff = grid_efficiency(1024, 4096);
+        const int kv_4096_eff = grid_efficiency(512, 4096);
+        const int output_b_8192_eff = grid_efficiency(4096, 768);
+        const bool public_dense_4096 = dense_4096_eff >= 80;
+        const bool public_pair_4096 =
+            public_dense_4096 && kv_4096_eff >= 80;
+        const bool public_output_b_8192 = output_b_8192_eff >= 80;
+
+        // N=512 and mmq_x=128 give four N tiles. Choose the smallest M-tile
+        // count >=16 for which 4*tiles_m is divisible by nSM. Stream-K then
+        // launches one block per tile (100% efficiency) and needs no fixup.
+        int gcd = prop.multiProcessorCount;
+        int remainder = 4;
+        while (remainder != 0) {
+            const int next = gcd % remainder;
+            gcd = remainder;
+            remainder = next;
+        }
+        const int tiles_m_step = prop.multiProcessorCount / gcd;
+        int64_t tiles_m = tiles_m_step;
+        while (tiles_m < 16) tiles_m += tiles_m_step;
+        if (tiles_m > INT32_MAX / 128) {
+            fprintf(stderr, "Q4 16-warp geometry overflow\n");
+            return 1;
+        }
+        const int no_fixup_M = (int)(128 * tiles_m);
+
+        fprintf(stderr,
+                "Q4 16-warp no-fixup geometry: nsm=%d tiles_m=%lld "
+                "tiles=%lld M=%d\n",
+                prop.multiProcessorCount, (long long)tiles_m,
+                (long long)(4 * tiles_m), no_fixup_M);
+
+        // Cover a device-dependent no-fixup baseline, an N tail, and the real
+        // 4096-token production dense/pair envelope.
+        all_ok &= run_q4_K_dense_16warp_parity(
+            /*M=*/no_fixup_M, /*N=*/512, /*K=*/1024,
+            prop.multiProcessorCount, 0xC4160001u,
+            /*check_public_dense=*/false,
+            /*check_rejection=*/true);
+        all_ok &= run_q4_K_dense_16warp_parity(
+            // N=601 keeps the canonical selector on m128n128 while retaining
+            // an N-tail, so external-scratch Stream-K remains bit-comparable.
+            /*M=*/2176, /*N=*/601, /*K=*/4096,
+            prop.multiProcessorCount, 0xC4160002u,
+            /*check_public_dense=*/false,
+            /*check_rejection=*/false);
+        // Production Q-A and Q-A/KV-pair envelope: exercise the newly admitted
+        // M=1024 dense leg at a full 4096-token context, then validate the
+        // asymmetric 1024+512 pair with one shared Q8_1 activation.
+        all_ok &= run_q4_K_dense_16warp_parity(
+            /*M=*/1024, /*N=*/4096, /*K=*/4096,
+            prop.multiProcessorCount, 0xC4160003u,
+            /*check_public_dense=*/public_dense_4096,
+            /*check_rejection=*/false);
+        if (!public_dense_4096) {
+            fprintf(stderr,
+                    "Q4 16-warp public dense N=4096 SKIP: grid efficiency "
+                    "%d%% < 80%% (nsm=%d)\n",
+                    dense_4096_eff, prop.multiProcessorCount);
+        }
+        // Production output-B has M=4096,K=8192. N=768 is a smaller,
+        // independent set of complete N128 tiles that selects the same
+        // scratch-free direct kernel as N=2048 on GB10, while the dedicated
+        // speed benchmark covers the real N=2048 timing geometry.
+        all_ok &= run_q4_K_dense_16warp_parity(
+            /*M=*/4096, /*N=*/768, /*K=*/8192,
+            prop.multiProcessorCount, 0xC4160006u,
+            /*check_public_dense=*/public_output_b_8192,
+            /*check_rejection=*/false);
+        if (!public_output_b_8192) {
+            fprintf(stderr,
+                    "Q4 16-warp public output-B K=8192 SKIP: grid "
+                    "efficiency %d%% < 80%% (nsm=%d)\n",
+                    output_b_8192_eff, prop.multiProcessorCount);
+        }
+        if (public_pair_4096) {
+            all_ok &= run_q4_K_dense_pair_16warp_parity(
+                /*M0=*/1024, /*M1=*/512, /*N=*/4096, /*K=*/4096,
+                prop.multiProcessorCount, 0xC4160004u);
+        } else {
+            // Keep raw coverage of the pair-only 512-row leg even when this
+            // device's SM geometry makes the required public pair ineligible.
+            all_ok &= run_q4_K_dense_16warp_parity(
+                /*M=*/512, /*N=*/4096, /*K=*/4096,
+                prop.multiProcessorCount, 0xC4160005u,
+                /*check_public_dense=*/false,
+                /*check_rejection=*/false);
+            fprintf(stderr,
+                    "Q4 16-warp public pair N=4096 SKIP: grid efficiency "
+                    "dense=%d%% kv=%d%% (need both >=80%%, nsm=%d)\n",
+                    dense_4096_eff, kv_4096_eff,
+                    prop.multiProcessorCount);
+        }
+        fprintf(stderr, "===================\n");
+        fprintf(stderr, "Q4 16-WARP %s\n", all_ok ? "PASS" : "FAILED");
+        return all_ok ? 0 : 1;
+    }
+
     // Q8_0
     all_ok &= run_q8_0(/*M=*/64,   /*N=*/4,   /*K=*/256,  0xC0FFEE);
     all_ok &= run_q8_0(/*M=*/128,  /*N=*/8,   /*K=*/512,  0xDEADBEE);
@@ -1174,6 +3609,37 @@ int main(int argc, char ** argv) {
     all_ok &= run_q4_K(/*M=*/128,  /*N=*/8,   /*K=*/512,  0xC4FE2);
     all_ok &= run_q4_K(/*M=*/256,  /*N=*/1,   /*K=*/2048, 0xC4FE3);
     all_ok &= run_q4_K(/*M=*/2048, /*N=*/16,  /*K=*/4096, 0xC4FE4);
+    // Prefill Q-A/KV pair: cover the MMVQ/MMQ boundary, token-tile tails,
+    // asymmetric output dimensions, and a full-width token tile.
+    all_ok &= run_q4_K_dense_pair_parity(
+        /*M0=*/257, /*M1=*/65,  /*N=*/9,   /*K=*/768,  0xC4FE50);
+    all_ok &= run_q4_K_dense_pair_parity(
+        /*M0=*/128, /*M1=*/73,  /*N=*/32,  /*K=*/4096, 0xC4FE51);
+    all_ok &= run_q4_K_dense_pair_parity(
+        /*M0=*/65,  /*M1=*/129, /*N=*/129, /*K=*/1024, 0xC4FE52);
+    all_ok &= run_q4_K_dense_pair_parity(
+        /*M0=*/96,  /*M1=*/33,  /*N=*/128, /*K=*/4096, 0xC4FE53);
+    // Grouped attention output-A prefill: exercise each token-tile tail
+    // around 8/16/32/128, with at least three groups in every case.  The
+    // final case also injects a NaN Q4 scale to verify fused sanitization.
+    all_ok &= run_q4_K_grouped_dense_parity(
+        /*M=*/33, /*N=*/9,   /*K=*/256, /*groups=*/3, 0xC4D009);
+    all_ok &= run_q4_K_grouped_dense_parity(
+        /*M=*/65, /*N=*/17,  /*K=*/512, /*groups=*/4, 0xC4D011);
+    all_ok &= run_q4_K_grouped_dense_parity(
+        /*M=*/47, /*N=*/33,  /*K=*/768, /*groups=*/5, 0xC4D021);
+    all_ok &= run_q4_K_grouped_dense_parity(
+        /*M=*/31, /*N=*/127, /*K=*/256, /*groups=*/3, 0xC4D07F);
+    all_ok &= run_q4_K_grouped_dense_parity(
+        /*M=*/31, /*N=*/128, /*K=*/256, /*groups=*/3, 0xC4D080);
+    all_ok &= run_q4_K_grouped_dense_parity(
+        /*M=*/31, /*N=*/129, /*K=*/512, /*groups=*/3, 0xC4D081,
+        /*inject_nonfinite=*/true);
+#if !defined(GGML_USE_HIP)
+    // Fixed production Q8_1 front-end: cover the first eligible width and
+    // both sides of the canonical 128-token tile boundary.
+    all_ok &= run_q4_K_grouped_q8_1_kernel_suite();
+#endif
 
     // MoE (_id) path.  Small expert counts + small shapes for fast verification.
     // Per-token-distinct routing with top_k=2 or 6.
@@ -1247,6 +3713,15 @@ int main(int argc, char ** argv) {
         /*ne=*/16, /*nu=*/6, 0xC4FE10, gen_q4k,
         ds4_mmq_q4_K_moe_pair, ds4_mmq_q4_K_moe);
 
+    // SSD compact-table raw fusion: one expert map/activation quantize for
+    // IQ2 gate+up and Q2 down.  Cover the production top-6 routing shape at
+    // each target prefill width.
+    all_ok &= run_iq2_xxs_q2_K_fused_raw_parity(/*nt=*/8,   0xC2F008);
+    all_ok &= run_iq2_xxs_q2_K_fused_raw_parity(/*nt=*/32,  0xC2F020);
+    all_ok &= run_iq2_xxs_q2_K_fused_raw_parity(/*nt=*/128, 0xC2F080);
+    all_ok &= run_iq2_xxs_q2_K_fused_raw_persistent_gb10_parity(
+        /*nt=*/32, 0xC2F021);
+
     // Step 6 - mmvq vector matmul tests.
     //
     // Single-W _moe_vec tests. Two shape classes per type:
@@ -1283,6 +3758,32 @@ int main(int argc, char ** argv) {
     all_ok &= run_q8_0_dense_vec(/*M=*/64,   /*N=*/1, /*K=*/256,  0xC0FE40);
     all_ok &= run_q8_0_dense_vec(/*M=*/256,  /*N=*/1, /*K=*/512,  0xC0FE41);
     all_ok &= run_q8_0_dense_vec(/*M=*/1024, /*N=*/1, /*K=*/4096, 0xC0FE42);
+    all_ok &= run_q4_K_dense_vec_gb10_parity(
+        /*M=*/1024, /*N=*/5, /*K=*/4096, 0xC4FE40, false);
+    all_ok &= run_q4_K_dense_vec_gb10_parity(
+        /*M=*/32768, /*N=*/1, /*K=*/1024, 0xC4FE41, true);
+    // Preserve coverage of the original one-token grouped ABI.
+    all_ok &= run_q4_K_grouped_vec_parity(
+        /*M=*/64, /*N=*/1, /*K=*/512, /*groups=*/4, 0xC4FE45);
+    all_ok &= run_q4_K_grouped_vec_parity(
+        /*M=*/256, /*N=*/2, /*K=*/8192, /*groups=*/4, 0xC4FE42);
+    // Small synthetic Flash-like case for quick token-aware coverage.
+    all_ok &= run_q4_K_grouped_vec_parity(
+        /*M=*/128, /*N=*/5, /*K=*/4096, /*groups=*/8, 0xC4FE43);
+    // DeepSeek-V4 Flash AProjQ4 attention-A production shape: each of the
+    // eight output groups owns a [rank=1024, group_dim=4096] Q4_K matrix.
+    // Cover both ordinary one-token decode and the maximum DSpark proposal
+    // width used by the verifier.
+    all_ok &= run_q4_K_grouped_vec_parity(
+        /*M=*/1024, /*N=*/1, /*K=*/4096, /*groups=*/8, 0xC4FE46);
+    all_ok &= run_q4_K_grouped_vec_parity(
+        /*M=*/1024, /*N=*/5, /*K=*/4096, /*groups=*/8, 0xC4FE47);
+    // Pro-style maximum group count and the N=8 API ceiling.
+    all_ok &= run_q4_K_grouped_vec_parity(
+        /*M=*/64, /*N=*/8, /*K=*/4096, /*groups=*/16, 0xC4FE44);
+    // DeepSeek-V4 Pro has the same rank/group_dim as Flash and 16 groups.
+    all_ok &= run_q4_K_grouped_vec_parity(
+        /*M=*/1024, /*N=*/1, /*K=*/4096, /*groups=*/16, 0xC4FE48);
 
     fprintf(stderr, "===================\n");
     fprintf(stderr, "%s\n", all_ok ? "ALL PASS" : "SOME FAILED");
diff --git a/cuda/mmq/test/test_mmq_soa_tiles.cu b/cuda/mmq/test/test_mmq_soa_tiles.cu
index 3f5f1d1e38..36ba8f3dc4 100644
--- a/cuda/mmq/test/test_mmq_soa_tiles.cu
+++ b/cuda/mmq/test/test_mmq_soa_tiles.cu
@@ -36,14 +36,6 @@
 #include <cuda_runtime.h>
 #endif
 
-// libds4mmq.a references this ds4_cuda.cu symbol from the q8-fold vec paths
-// (C3 Inc4); the entries under test never reach it, so a "no fold available"
-// stub satisfies the link.
-extern "C" int ds4_cuda_q8_fold_take_q81(const void *src, uint64_t in_dim, void *out) {
-    (void)src; (void)in_dim; (void)out;
-    return 0;
-}
-
 #include <cstdint>
 #include <cstdio>
 #include <cstdlib>
diff --git a/cuda/mmq/vendors/hip.h b/cuda/mmq/vendors/hip.h
index 27885edc00..5099a4437d 100644
--- a/cuda/mmq/vendors/hip.h
+++ b/cuda/mmq/vendors/hip.h
@@ -58,6 +58,7 @@
 #define cudaDeviceProp hipDeviceProp_t
 #define cudaDeviceSynchronize hipDeviceSynchronize
 #define cudaError_t hipError_t
+#define cudaErrorInvalidValue hipErrorInvalidValue
 #define cudaErrorMemoryAllocation hipErrorOutOfMemory
 #define cudaErrorPeerAccessAlreadyEnabled hipErrorPeerAccessAlreadyEnabled
 #define cudaErrorPeerAccessNotEnabled hipErrorPeerAccessNotEnabled
@@ -97,7 +98,10 @@
 #define cudaMemset hipMemset
 #define cudaMemsetAsync hipMemsetAsync
 #define cudaMemGetInfo hipMemGetInfo
+#define cudaMemoryTypeDevice hipMemoryTypeDevice
 #define cudaOccupancyMaxPotentialBlockSize hipOccupancyMaxPotentialBlockSize
+#define cudaPointerAttributes hipPointerAttribute_t
+#define cudaPointerGetAttributes hipPointerGetAttributes
 #define cudaSetDevice hipSetDevice
 #define cuDeviceGet hipDeviceGet
 #define CUdevice hipDevice_t
diff --git a/ds4.c b/ds4.c
index 91ab214ab7..75548fb89a 100644
--- a/ds4.c
+++ b/ds4.c
@@ -46,7 +46,7 @@
 #include "ds4_tp.h"
 
 /* TP context for the verify-block RDMA window (set with the gate callbacks). */
-#if !defined(DS4_NO_GPU) && defined(__APPLE__)
+#if !defined(DS4_NO_GPU)
 static ds4_tp *g_tp_block_ctx;
 #endif
 
@@ -402,6 +402,24 @@ static bool ds4_backend_uses_graph(ds4_backend backend) {
     return backend == DS4_BACKEND_METAL || backend == DS4_BACKEND_CUDA;
 }
 
+typedef enum {
+    DS4_DSPARK_RUNTIME_SUPPORTED = 0,
+    DS4_DSPARK_RUNTIME_UNSUPPORTED_CPU,
+    DS4_DSPARK_RUNTIME_UNSUPPORTED_DISTRIBUTED,
+} ds4_dspark_runtime_policy;
+
+static ds4_dspark_runtime_policy ds4_dspark_runtime_policy_for(
+        ds4_backend backend,
+        ds4_distributed_role distributed_role) {
+    if (backend == DS4_BACKEND_CPU) {
+        return DS4_DSPARK_RUNTIME_UNSUPPORTED_CPU;
+    }
+    if (distributed_role != DS4_DISTRIBUTED_NONE) {
+        return DS4_DSPARK_RUNTIME_UNSUPPORTED_DISTRIBUTED;
+    }
+    return DS4_DSPARK_RUNTIME_SUPPORTED;
+}
+
 static bool ds4_backend_supports_ssd_streaming(ds4_backend backend) {
     if (backend == DS4_BACKEND_METAL) return true;
     if (backend == DS4_BACKEND_CUDA) {
@@ -939,6 +957,7 @@ typedef struct {
     float *routed_mid_all;
     block_q8_K *routed_xq;
     block_q8_K *routed_midq;
+    block_q8_K *dense_xq;
     int8_t *routed_q8_xq;
     float *routed_q8_xscale;
     int8_t *routed_q8_midq;
@@ -3729,6 +3748,129 @@ static void ds4_vec_dot_q4_K_q8_K(int n, float *s, const block_q4_K *x, const bl
 #endif
 }
 
+/* Evaluate two activation rows against one Q4_K weight row.  Each token keeps
+ * its own integer and floating-point accumulation order, while the packed
+ * Q4 nibbles and scale/min metadata are decoded only once. */
+static void ds4_vec_dot_q4_K_q8_K_2(
+        int n,
+        float *s0,
+        float *s1,
+        const block_q4_K *x,
+        const block_q8_K *y0,
+        const block_q8_K *y1) {
+    const int nb = n / QK_K;
+
+#if defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)
+    const int32x4_t zero = vdupq_n_s32(0);
+    float sumf0 = 0.0f;
+    float sumf1 = 0.0f;
+
+    for (int i = 0; i < nb; i++) {
+        const float xd = f16_to_f32(x[i].d);
+        const float xmin = f16_to_f32(x[i].dmin);
+        const float d0 = y0[i].d * xd;
+        const float d1 = y1[i].d * xd;
+        const float dm0 = -y0[i].d * xmin;
+        const float dm1 = -y1[i].d * xmin;
+        const uint8_t *qs = x[i].qs;
+        const uint8_t *sc = x[i].scales;
+        const int8_t *q80 = y0[i].qs;
+        const int8_t *q81 = y1[i].qs;
+
+        int32_t summs0 = 0;
+        int32_t summs1 = 0;
+        for (int j = 0; j < QK_K / 32; j++) {
+            uint8_t sc_val, m_val;
+            q4_k_get_scale_min(j, sc, &sc_val, &m_val);
+            const int32_t gsum0 = (int32_t)y0[i].bsums[j * 2] +
+                                  (int32_t)y0[i].bsums[j * 2 + 1];
+            const int32_t gsum1 = (int32_t)y1[i].bsums[j * 2] +
+                                  (int32_t)y1[i].bsums[j * 2 + 1];
+            summs0 += m_val * gsum0;
+            summs1 += m_val * gsum1;
+        }
+
+        int isum0 = 0;
+        int isum1 = 0;
+        for (int j = 0; j < QK_K / 32; j++) {
+            uint8_t sc_val, m_val;
+            q4_k_get_scale_min(j, sc, &sc_val, &m_val);
+            const int byte_off = (j >> 1) * 32;
+            const int shift = (j & 1) * 4;
+            const int8x16x2_t q8v0 = vld1q_s8_x2(q80 + j * 32);
+            const int8x16x2_t q8v1 = vld1q_s8_x2(q81 + j * 32);
+            uint8_t q4_u[32];
+            if (shift == 0) {
+                for (int l = 0; l < 32; l++) q4_u[l] = qs[byte_off + l] & 0xF;
+            } else {
+                for (int l = 0; l < 32; l++) q4_u[l] = qs[byte_off + l] >> 4;
+            }
+            const int8x16_t q4a = vreinterpretq_s8_u8(vld1q_u8(q4_u));
+            const int8x16_t q4b = vreinterpretq_s8_u8(vld1q_u8(q4_u + 16));
+            isum0 += vaddvq_s32(vdotq_s32(zero, q4a, q8v0.val[0])) * sc_val;
+            isum0 += vaddvq_s32(vdotq_s32(zero, q4b, q8v0.val[1])) * sc_val;
+            isum1 += vaddvq_s32(vdotq_s32(zero, q4a, q8v1.val[0])) * sc_val;
+            isum1 += vaddvq_s32(vdotq_s32(zero, q4b, q8v1.val[1])) * sc_val;
+        }
+
+        sumf0 += d0 * (float)isum0 + dm0 * (float)summs0;
+        sumf1 += d1 * (float)isum1 + dm1 * (float)summs1;
+    }
+
+    *s0 = sumf0;
+    *s1 = sumf1;
+#else
+    float sumf0 = 0.0f;
+    float sumf1 = 0.0f;
+
+    for (int i = 0; i < nb; i++) {
+        const float xd = f16_to_f32(x[i].d);
+        const float xmin = f16_to_f32(x[i].dmin);
+        const float d0 = y0[i].d * xd;
+        const float d1 = y1[i].d * xd;
+        const float dm0 = -y0[i].d * xmin;
+        const float dm1 = -y1[i].d * xmin;
+        const uint8_t *qs = x[i].qs;
+        const uint8_t *sc = x[i].scales;
+        const int8_t *q80 = y0[i].qs;
+        const int8_t *q81 = y1[i].qs;
+
+        int summs0 = 0;
+        int summs1 = 0;
+        for (int j = 0; j < QK_K / 32; j++) {
+            uint8_t sc_val, m_val;
+            q4_k_get_scale_min(j, sc, &sc_val, &m_val);
+            const int32_t gsum0 = (int32_t)y0[i].bsums[j * 2] +
+                                  (int32_t)y0[i].bsums[j * 2 + 1];
+            const int32_t gsum1 = (int32_t)y1[i].bsums[j * 2] +
+                                  (int32_t)y1[i].bsums[j * 2 + 1];
+            summs0 += m_val * gsum0;
+            summs1 += m_val * gsum1;
+        }
+
+        int isum0 = 0;
+        int isum1 = 0;
+        for (int j = 0; j < QK_K / 32; j++) {
+            uint8_t sc_val, m_val;
+            q4_k_get_scale_min(j, sc, &sc_val, &m_val);
+            const int byte_off = (j >> 1) * 32;
+            const int shift = (j & 1) * 4;
+            for (int l = 0; l < 32; l++) {
+                const int q4 = (qs[byte_off + l] >> shift) & 0xF;
+                isum0 += q4 * (int)q80[j * 32 + l] * sc_val;
+                isum1 += q4 * (int)q81[j * 32 + l] * sc_val;
+            }
+        }
+
+        sumf0 += d0 * (float)isum0 + dm0 * (float)summs0;
+        sumf1 += d1 * (float)isum1 + dm1 * (float)summs1;
+    }
+
+    *s0 = sumf0;
+    *s1 = sumf1;
+#endif
+}
+
 static void ds4_vec_dot_q5_K_q8_K(int n, float *s, const block_q5_K *x, const block_q8_K *y) {
     const int nb = n / QK_K;
     float sumf = 0.0f;
@@ -4529,20 +4671,28 @@ static void tensor_expect_plain_layout(
     tensor_expect_layout(t, t->type, ndim, d0, d1, d2);
 }
 
-static bool tensor_type_is_f16_or_q8_0(uint32_t type) {
-    return type == DS4_TENSOR_F16 || type == DS4_TENSOR_Q8_0;
+static bool tensor_type_is_indexer_q(uint32_t type) {
+    return type == DS4_TENSOR_F16 ||
+           type == DS4_TENSOR_Q8_0 ||
+           type == DS4_TENSOR_Q4_K;
 }
 
-static void tensor_expect_f16_or_q8_0_layout(
+#ifdef DS4_TEST_HOOKS
+int ds4_test_indexer_q_type_supported(uint32_t type) {
+    return tensor_type_is_indexer_q(type) ? 1 : 0;
+}
+#endif
+
+static void tensor_expect_indexer_q_layout(
         const ds4_tensor *t,
         uint32_t          ndim,
         uint64_t          d0,
         uint64_t          d1,
         uint64_t          d2) {
     if (!t) ds4_die("internal error: missing tensor while validating layout");
-    if (!tensor_type_is_f16_or_q8_0(t->type)) {
+    if (!tensor_type_is_indexer_q(t->type)) {
         fprintf(stderr,
-                "ds4: tensor %.*s has type %s, expected f16 or q8_0\n",
+                "ds4: tensor %.*s has type %s, expected f16, q8_0, or q4_K\n",
                 (int)t->name.len,
                 t->name.ptr,
                 tensor_type_name(t->type));
@@ -4813,44 +4963,140 @@ static ds4_gpu_stream_expert_table graph_stream_expert_table_make(
 }
 #endif
 
-static uint64_t ds4_streaming_manual_cache_safe_bytes(
+static bool ds4_streaming_manual_cache_safe_bytes(
         ds4_backend backend,
         int         ctx_size,
         uint32_t    prefill_chunk,
-        bool        ssd_streaming) {
+        bool        ssd_streaming,
+        uint64_t    fixed_model_bytes,
+        uint64_t   *safe_bytes_out) {
+    if (safe_bytes_out) *safe_bytes_out = 0;
 #ifdef DS4_NO_GPU
     (void)backend;
     (void)ctx_size;
     (void)prefill_chunk;
     (void)ssd_streaming;
-    return 0;
+    (void)fixed_model_bytes;
+    return false;
 #else
-    const uint64_t gib = 1024ull * 1024ull * 1024ull;
     const uint64_t recommended = ds4_gpu_recommended_working_set_size();
-    if (recommended == 0) return 0;
+    if (recommended == 0 || !safe_bytes_out) return false;
 
     /*
-     * Explicit NGB budgets name only the routed expert cache. Keep that cache
-     * below the graph backend's working-set recommendation after accounting for
-     * the graph context/KV buffers. This is intentionally not an mlock-derived
-     * cap: crossing too close to the recommended working set makes short
-     * token-major prefill spend most of its time in VM/driver synchronization.
+     * Explicit NGB budgets name the total routed-expert budget (prefill
+     * headroom plus the decode cache). The mmap-backed Metal views and the
+     * CUDA/ROCm device caches all share the accelerator memory target with
+     * non-routed weights and an active support model. Failing to include those
+     * fixed weights lets a seemingly safe expert cache evict the dense working
+     * set or OOM while loading DSpark. Keep the total below the backend's
+     * recommended working set after the context/KV estimate and fixed-model
+     * accounting. This estimate deliberately does not claim the complete
+     * per-tier batch-prefill workspace or routed-prefill transient reserve.
+     *
+     * If less than the minimum usable cache remains, the normal planner emits
+     * its existing "budget too small" error instead of silently exceeding the
+     * memory-pressure limit.
      */
-    uint64_t target = recommended > UINT64_MAX / 7ull ?
-        UINT64_MAX : (recommended * 7ull) / 8ull;
+    const uint64_t target = recommended - recommended / 8ull;
     const ds4_context_memory ctx_mem =
         ds4_context_memory_estimate_with_prefill_mode(backend,
                                                       ctx_size,
                                                       prefill_chunk,
                                                       ssd_streaming);
-    uint64_t safe = 0;
-    if (target > ctx_mem.total_bytes) safe = target - ctx_mem.total_bytes;
-    safe = (safe / gib) * gib;
-    if (safe == 0) safe = gib;
-    return safe;
+    uint64_t fixed = ctx_mem.total_bytes;
+    if (ds4_backend_uses_graph(backend)) {
+        fixed = fixed > UINT64_MAX - fixed_model_bytes ?
+            UINT64_MAX : fixed + fixed_model_bytes;
+    }
+    *safe_bytes_out = target > fixed ? target - fixed : 0;
+    return true;
 #endif
 }
 
+#define DS4_METAL_DSPARK_SAFE_EXPERT_COUNT_ENV \
+    "DS4_METAL_DSPARK_SAFE_EXPERT_COUNT"
+
+/*
+ * A plain --ssd-streaming-cache-experts N value names dynamic cache slots,
+ * unlike an NGB value.  Convert it to bytes before comparing it with the same
+ * working-set limit used by manual NGB budgets, then convert the safe result
+ * back to whole slots.  Keeping this arithmetic in one checked helper avoids
+ * wrapping a large count into a deceptively small byte request.
+ */
+static bool ds4_streaming_manual_cache_cap_count(
+        uint32_t  requested_count,
+        uint64_t  per_expert_bytes,
+        uint64_t  safe_cache_bytes,
+        uint32_t *effective_count_out,
+        uint64_t *requested_bytes_out,
+        uint64_t *effective_bytes_out) {
+    if (effective_count_out) *effective_count_out = 0;
+    if (requested_bytes_out) *requested_bytes_out = 0;
+    if (effective_bytes_out) *effective_bytes_out = 0;
+    if (requested_count == 0 || per_expert_bytes == 0 ||
+        !effective_count_out || !requested_bytes_out ||
+        !effective_bytes_out) {
+        return false;
+    }
+    if ((uint64_t)requested_count > UINT64_MAX / per_expert_bytes) {
+        return false;
+    }
+
+    const uint64_t requested_bytes =
+        (uint64_t)requested_count * per_expert_bytes;
+    uint64_t safe_count = safe_cache_bytes / per_expert_bytes;
+    if (safe_count > UINT32_MAX) safe_count = UINT32_MAX;
+    const uint32_t effective_count =
+        safe_count < requested_count ? (uint32_t)safe_count : requested_count;
+    /* effective_count <= requested_count, whose product was checked above. */
+    const uint64_t effective_bytes =
+        (uint64_t)effective_count * per_expert_bytes;
+
+    *effective_count_out = effective_count;
+    *requested_bytes_out = requested_bytes;
+    *effective_bytes_out = effective_bytes;
+    return true;
+}
+
+static bool ds4_streaming_manual_cache_count_cap_enabled(void) {
+    const char *env = getenv(DS4_METAL_DSPARK_SAFE_EXPERT_COUNT_ENV);
+    /* Explicit opt-in while the policy is being validated on real hardware. */
+    return env && strcmp(env, "1") == 0;
+}
+
+static bool ds4_streaming_manual_cache_count_cap_eligible(
+        ds4_backend       backend,
+        bool              ssd_streaming,
+        bool              dspark_enabled,
+        ds4_support_kind  support_kind,
+        uint32_t          cache_experts,
+        uint64_t          cache_bytes) {
+    return backend == DS4_BACKEND_METAL &&
+           ssd_streaming &&
+           dspark_enabled &&
+           support_kind == DS4_SUPPORT_DSPARK &&
+           cache_experts != 0 &&
+           cache_bytes == 0;
+}
+
+static uint64_t ds4_streaming_dspark_active_support_reserve_bytes(
+        uint64_t support_model_bytes) {
+    const uint64_t max_active_support_reserve =
+        2ull * 1024ull * 1024ull * 1024ull;
+    return support_model_bytes < max_active_support_reserve ?
+        support_model_bytes : max_active_support_reserve;
+}
+
+/* Unknown or sub-slot room must not make an existing explicit-count command
+ * fail. A measured non-zero candidate may only reduce the requested count. */
+static uint32_t ds4_streaming_manual_cache_nonfatal_effective_count(
+        uint32_t requested_count,
+        bool     safe_cache_known,
+        uint32_t candidate_count) {
+    if (!safe_cache_known || candidate_count == 0) return requested_count;
+    return candidate_count < requested_count ? candidate_count : requested_count;
+}
+
 static uint64_t ds4_add_sat_u64(uint64_t a, uint64_t b) {
     return a > UINT64_MAX - b ? UINT64_MAX : a + b;
 }
@@ -5301,7 +5547,7 @@ static void weights_validate_layout(
         if (ratio == 4) {
             const uint64_t index_q_dim = (uint64_t)DS4_N_INDEXER_HEAD * DS4_N_INDEXER_HEAD_DIM;
             const uint64_t index_width = 2u * DS4_N_INDEXER_HEAD_DIM;
-            tensor_expect_f16_or_q8_0_layout(l->indexer_attn_q_b, 2, DS4_N_LORA_Q, index_q_dim, 0);
+            tensor_expect_indexer_q_layout(l->indexer_attn_q_b, 2, DS4_N_LORA_Q, index_q_dim, 0);
             tensor_expect_layout(l->indexer_proj,              DS4_TENSOR_F16, 2, DS4_N_EMBD, DS4_N_INDEXER_HEAD, 0);
             tensor_expect_layout(l->indexer_compressor_ape,    DS4_TENSOR_F16, 2, index_width, ratio, 0);
             tensor_expect_layout(l->indexer_compressor_kv,     DS4_TENSOR_F16, 2, DS4_N_EMBD, index_width, 0);
@@ -8231,26 +8477,6 @@ static void matvec_q8_0_3d_slice_prequant(
     ds4_parallel_for(out_dim, matvec_q8_0_worker, &ctx);
 }
 
-static DS4_MAYBE_UNUSED void matvec_q8_0_3d_slice(
-        float           * out,
-        const ds4_model * m,
-        const ds4_tensor * w,
-        const float     * x,
-        uint64_t          slice) {
-    if (w->type != DS4_TENSOR_Q8_0 || w->ndim != 3) ds4_die("expected a 3D Q8_0 tensor");
-
-    const uint64_t in_dim = w->dim[0];
-    const uint64_t blocks = (in_dim + 31) / 32;
-    int8_t *xq = xmalloc((size_t)blocks * 32);
-    float *xscale = xmalloc((size_t)blocks * sizeof(xscale[0]));
-
-    quantize_q8_0_activation(x, xq, xscale, in_dim);
-    matvec_q8_0_3d_slice_prequant(out, m, w, xq, xscale, slice);
-
-    free(xscale);
-    free(xq);
-}
-
 /* Compute two Q8_0 projections from the same input, used by gate/up and
  * compressor kv/score pairs. */
 static void matvec_q8_0_pair_prequant(
@@ -8468,6 +8694,12 @@ static void matvec_q8_0_f32_ref(
 }
 
 static void matvec_any(float *out, const ds4_model *m, const ds4_tensor *w, const float *x);
+static void matvec_q4_K_decode_scratch(
+        float                  *out,
+        const ds4_model        *m,
+        const ds4_tensor       *w,
+        const float            *x,
+        ds4_cpu_decode_scratch *scratch);
 
 /* Decode scratch owns this temporary activation quantization so generation
  * can assert that the hot path performs no malloc. */
@@ -8509,6 +8741,8 @@ static void matvec_any_decode_scratch(
         ds4_cpu_decode_scratch * scratch) {
     if (w->type == 8) {
         matvec_q8_0_decode_scratch(out, m, w, x, scratch);
+    } else if (w->type == DS4_TENSOR_Q4_K) {
+        matvec_q4_K_decode_scratch(out, m, w, x, scratch);
     } else {
         matvec_any(out, m, w, x);
     }
@@ -8635,6 +8869,393 @@ static void matmul_q8_0_grouped_batch(
     free(xq);
 }
 
+/* =========================================================================
+ * Dense Q4_K matvec/matmul.
+ * =========================================================================
+ *
+ * The AProjQ4 GGUFs store the dense attention projections (q_a, q_b, kv,
+ * output_a, output_b) as Q4_K instead of Q8_0.  This family mirrors the
+ * Q8_0 dense functions above: the activation is quantized to Q8_K once and
+ * each Q4_K weight row is reduced with the shared ds4_vec_dot_q4_K_q8_K
+ * kernel, so the same call sites can dispatch on the tensor type.
+ */
+
+typedef struct {
+    float *out;
+    const uint8_t *data;
+    const block_q8_K *xq;
+    uint64_t in_dim;
+    uint64_t row_bytes;
+} matvec_q4_K_dense_ctx;
+
+static void matvec_q4_K_dense_worker(void *vctx, uint64_t row0, uint64_t row1) {
+    matvec_q4_K_dense_ctx *ctx = vctx;
+    for (uint64_t r = row0; r < row1; r++) {
+        const block_q4_K *row = (const block_q4_K *)(ctx->data + r * ctx->row_bytes);
+        ds4_vec_dot_q4_K_q8_K((int)ctx->in_dim, &ctx->out[r], row, ctx->xq);
+    }
+}
+
+static void dense_q4_K_expect(const ds4_tensor *w, uint64_t in_dim) {
+    if (w->type != DS4_TENSOR_Q4_K || w->ndim != 2) ds4_die("expected a 2D Q4_K tensor");
+    if (w->dim[0] != in_dim) ds4_die("Q4_K dense tensor has an unexpected width");
+    if ((in_dim % QK_K) != 0) ds4_die("Q4_K dense row is not QK_K aligned");
+}
+
+static void matvec_q4_K_prequant(
+        float            *out,
+        const ds4_model  *m,
+        const ds4_tensor *w,
+        const block_q8_K *xq) {
+    matvec_q4_K_dense_ctx ctx = {
+        .out = out,
+        .data = tensor_data(m, w),
+        .xq = xq,
+        .in_dim = w->dim[0],
+        .row_bytes = (w->dim[0] / QK_K) * sizeof(block_q4_K),
+    };
+    ds4_parallel_for(w->dim[1], matvec_q4_K_dense_worker, &ctx);
+}
+
+static void matvec_q4_K(float *out, const ds4_model *m, const ds4_tensor *w, const float *x) {
+    dense_q4_K_expect(w, w->dim[0]);
+    const uint64_t blocks = w->dim[0] / QK_K;
+    block_q8_K *xq = xmalloc((size_t)blocks * sizeof(block_q8_K));
+    ds4_quantize_row_q8_K(x, xq, (int64_t)w->dim[0]);
+    matvec_q4_K_prequant(out, m, w, xq);
+    free(xq);
+}
+
+static void matvec_q4_K_decode_scratch(
+        float                  *out,
+        const ds4_model        *m,
+        const ds4_tensor       *w,
+        const float            *x,
+        ds4_cpu_decode_scratch *scratch) {
+    dense_q4_K_expect(w, w->dim[0]);
+    if (w->dim[0] > scratch->q8_cap) ds4_die("CPU decode Q4_K scratch buffer is too small");
+    ds4_quantize_row_q8_K(x, scratch->dense_xq, (int64_t)w->dim[0]);
+    matvec_q4_K_prequant(out, m, w, scratch->dense_xq);
+}
+
+typedef struct {
+    float *out;
+    const uint8_t *data;
+    const block_q8_K *xq;
+    uint64_t in_dim;
+    uint64_t blocks;
+    uint64_t rank;
+} matvec_q4_K_grouped_ctx;
+
+static void matvec_q4_K_grouped_worker(void *vctx, uint64_t r0, uint64_t r1) {
+    matvec_q4_K_grouped_ctx *ctx = vctx;
+    for (uint64_t idx = r0; idx < r1; idx++) {
+        const uint64_t group = idx / ctx->rank;
+        const block_q4_K *row = (const block_q4_K *)
+            (ctx->data + idx * ctx->blocks * sizeof(block_q4_K));
+        ds4_vec_dot_q4_K_q8_K((int)ctx->in_dim, &ctx->out[idx], row,
+                              ctx->xq + group * ctx->blocks);
+    }
+}
+
+static void matvec_q4_K_grouped_expect(
+        const ds4_tensor *w,
+        uint32_t          n_groups,
+        uint64_t          group_dim,
+        uint64_t          rank) {
+    dense_q4_K_expect(w, group_dim);
+    if (w->dim[1] < (uint64_t)n_groups * rank) {
+        ds4_die("grouped Q4_K tensor has an unexpected layout");
+    }
+}
+
+static void matvec_q4_K_grouped_rows_prequant(
+        float            *out,
+        const ds4_model  *m,
+        const ds4_tensor *w,
+        const block_q8_K *xq,
+        uint32_t          n_groups,
+        uint64_t          group_dim,
+        uint64_t          rank) {
+    matvec_q4_K_grouped_ctx ctx = {
+        .out = out,
+        .data = tensor_data(m, w),
+        .xq = xq,
+        .in_dim = group_dim,
+        .blocks = group_dim / QK_K,
+        .rank = rank,
+    };
+    ds4_parallel_for((uint64_t)n_groups * rank, matvec_q4_K_grouped_worker, &ctx);
+}
+
+static void matvec_q4_K_grouped_rows(
+        float            *out,
+        const ds4_model  *m,
+        const ds4_tensor *w,
+        const float      *x,
+        uint32_t          n_groups,
+        uint64_t          group_dim,
+        uint64_t          rank) {
+    matvec_q4_K_grouped_expect(w, n_groups, group_dim, rank);
+    const uint64_t blocks = group_dim / QK_K;
+    block_q8_K *xq = xmalloc((size_t)n_groups * blocks * sizeof(block_q8_K));
+    for (uint32_t g = 0; g < n_groups; g++) {
+        ds4_quantize_row_q8_K(x + (uint64_t)g * group_dim,
+                              xq + (uint64_t)g * blocks,
+                              (int64_t)group_dim);
+    }
+    matvec_q4_K_grouped_rows_prequant(out, m, w, xq, n_groups, group_dim, rank);
+    free(xq);
+}
+
+static void matvec_q4_K_grouped_rows_decode_scratch(
+        float                  *out,
+        const ds4_model        *m,
+        const ds4_tensor       *w,
+        const float            *x,
+        uint32_t                n_groups,
+        uint64_t                group_dim,
+        uint64_t                rank,
+        ds4_cpu_decode_scratch *scratch) {
+    matvec_q4_K_grouped_expect(w, n_groups, group_dim, rank);
+    if ((uint64_t)n_groups * group_dim > scratch->q8_cap) {
+        ds4_die("CPU decode grouped Q4_K scratch buffer is too small");
+    }
+    const uint64_t blocks = group_dim / QK_K;
+    for (uint32_t g = 0; g < n_groups; g++) {
+        ds4_quantize_row_q8_K(x + (uint64_t)g * group_dim,
+                              scratch->dense_xq + (uint64_t)g * blocks,
+                              (int64_t)group_dim);
+    }
+    matvec_q4_K_grouped_rows_prequant(out, m, w, scratch->dense_xq,
+                                      n_groups, group_dim, rank);
+}
+
+typedef struct {
+    float *out;
+    const uint8_t *data;
+    const block_q8_K *xq;
+    uint64_t n_tok;
+    uint64_t in_dim;
+    uint64_t out_dim;
+    uint64_t blocks;
+} matmul_q4_K_batch_ctx;
+
+typedef struct {
+    const float *x;
+    block_q8_K *xq;
+    uint64_t row_dim;
+    uint64_t blocks;
+} quantize_q8_K_rows_ctx;
+
+static void quantize_q8_K_rows_worker(
+        void *vctx, uint64_t row0, uint64_t row1) {
+    quantize_q8_K_rows_ctx *ctx = vctx;
+    for (uint64_t row = row0; row < row1; row++) {
+        ds4_quantize_row_q8_K(ctx->x + row * ctx->row_dim,
+                              ctx->xq + row * ctx->blocks,
+                              (int64_t)ctx->row_dim);
+    }
+}
+
+static void quantize_q8_K_rows(
+        const float *x,
+        block_q8_K *xq,
+        uint64_t n_rows,
+        uint64_t row_dim) {
+    quantize_q8_K_rows_ctx ctx = {
+        .x = x,
+        .xq = xq,
+        .row_dim = row_dim,
+        .blocks = row_dim / QK_K,
+    };
+    /* A pool round-trip costs more than quantizing a handful of rows.  Match
+     * the existing F16 matvec crossover and parallelize only once the batch
+     * contains at least 256K activation elements. */
+    const bool work_overflow = row_dim != 0 && n_rows > UINT64_MAX / row_dim;
+    const uint64_t work = work_overflow ? UINT64_MAX : n_rows * row_dim;
+    const uint64_t min_parallel_rows = work >= 262144u ? 1u : 512u;
+    ds4_parallel_for_min_rows(n_rows, quantize_q8_K_rows_worker, &ctx,
+                              min_parallel_rows);
+}
+
+static void matmul_q4_K_batch_worker(void *vctx, uint64_t r0, uint64_t r1) {
+    matmul_q4_K_batch_ctx *ctx = vctx;
+    for (uint64_t r = r0; r < r1; r++) {
+        const block_q4_K *row = (const block_q4_K *)
+            (ctx->data + r * ctx->blocks * sizeof(block_q4_K));
+        uint64_t t = 0;
+        for (; t + 1 < ctx->n_tok; t += 2) {
+            ds4_vec_dot_q4_K_q8_K_2(
+                    (int)ctx->in_dim,
+                    &ctx->out[t * ctx->out_dim + r],
+                    &ctx->out[(t + 1) * ctx->out_dim + r],
+                    row,
+                    ctx->xq + t * ctx->blocks,
+                    ctx->xq + (t + 1) * ctx->blocks);
+        }
+        for (; t < ctx->n_tok; t++) {
+            ds4_vec_dot_q4_K_q8_K((int)ctx->in_dim,
+                                  &ctx->out[t * ctx->out_dim + r],
+                                  row,
+                                  ctx->xq + t * ctx->blocks);
+        }
+    }
+}
+
+static void matmul_q4_K_batch(
+        float            *out,
+        const ds4_model  *m,
+        const ds4_tensor *w,
+        const float      *x,
+        uint64_t          n_tok) {
+    dense_q4_K_expect(w, w->dim[0]);
+    const uint64_t in_dim = w->dim[0];
+    const uint64_t blocks = in_dim / QK_K;
+    block_q8_K *xq = xmalloc((size_t)n_tok * blocks * sizeof(block_q8_K));
+    quantize_q8_K_rows(x, xq, n_tok, in_dim);
+    matmul_q4_K_batch_ctx ctx = {
+        .out = out,
+        .data = tensor_data(m, w),
+        .xq = xq,
+        .n_tok = n_tok,
+        .in_dim = in_dim,
+        .out_dim = w->dim[1],
+        .blocks = blocks,
+    };
+    ds4_parallel_for(w->dim[1], matmul_q4_K_batch_worker, &ctx);
+    free(xq);
+}
+
+typedef struct {
+    float *out;
+    const uint8_t *data;
+    const block_q8_K *xq;
+    uint64_t n_tok;
+    uint64_t n_groups;
+    uint64_t group_dim;
+    uint64_t blocks;
+    uint64_t rank;
+} matmul_q4_K_grouped_batch_ctx;
+
+static void matmul_q4_K_grouped_batch_worker(void *vctx, uint64_t r0, uint64_t r1) {
+    matmul_q4_K_grouped_batch_ctx *ctx = vctx;
+    for (uint64_t idx = r0; idx < r1; idx++) {
+        const uint64_t group = idx / ctx->rank;
+        const block_q4_K *row = (const block_q4_K *)
+            (ctx->data + idx * ctx->blocks * sizeof(block_q4_K));
+        uint64_t t = 0;
+        for (; t + 1 < ctx->n_tok; t += 2) {
+            ds4_vec_dot_q4_K_q8_K_2(
+                    (int)ctx->group_dim,
+                    &ctx->out[t * ctx->n_groups * ctx->rank + idx],
+                    &ctx->out[(t + 1) * ctx->n_groups * ctx->rank + idx],
+                    row,
+                    ctx->xq + (t * ctx->n_groups + group) * ctx->blocks,
+                    ctx->xq + ((t + 1) * ctx->n_groups + group) * ctx->blocks);
+        }
+        for (; t < ctx->n_tok; t++) {
+            ds4_vec_dot_q4_K_q8_K((int)ctx->group_dim,
+                                  &ctx->out[t * ctx->n_groups * ctx->rank + idx],
+                                  row,
+                                  ctx->xq + (t * ctx->n_groups + group) * ctx->blocks);
+        }
+    }
+}
+
+static void matmul_q4_K_grouped_batch(
+        float            *out,
+        const ds4_model  *m,
+        const ds4_tensor *w,
+        const float      *x,
+        uint64_t          n_tok,
+        uint32_t          n_groups,
+        uint64_t          group_dim,
+        uint64_t          rank) {
+    matvec_q4_K_grouped_expect(w, n_groups, group_dim, rank);
+    const uint64_t blocks = group_dim / QK_K;
+    block_q8_K *xq = xmalloc((size_t)n_tok * n_groups * blocks * sizeof(block_q8_K));
+    quantize_q8_K_rows(x, xq, n_tok * n_groups, group_dim);
+    matmul_q4_K_grouped_batch_ctx ctx = {
+        .out = out,
+        .data = tensor_data(m, w),
+        .xq = xq,
+        .n_tok = n_tok,
+        .n_groups = n_groups,
+        .group_dim = group_dim,
+        .blocks = blocks,
+        .rank = rank,
+    };
+    ds4_parallel_for((uint64_t)n_groups * rank, matmul_q4_K_grouped_batch_worker, &ctx);
+    free(xq);
+}
+
+/* Type dispatch for the dense attention projections: the AProjQ8 GGUFs keep
+ * them Q8_0, the AProjQ4 ones use Q4_K.  Q8_0 stays on the exact functions
+ * it always used so existing models remain bit-identical. */
+
+static void matvec_dense_grouped_rows(
+        float            *out,
+        const ds4_model  *m,
+        const ds4_tensor *w,
+        const float      *x,
+        uint32_t          n_groups,
+        uint64_t          group_dim,
+        uint64_t          rank) {
+    if (w->type == DS4_TENSOR_Q4_K) {
+        matvec_q4_K_grouped_rows(out, m, w, x, n_groups, group_dim, rank);
+    } else {
+        matvec_q8_0_grouped_rows(out, m, w, x, n_groups, group_dim, rank);
+    }
+}
+
+static void matvec_dense_grouped_rows_decode_scratch(
+        float                  *out,
+        const ds4_model        *m,
+        const ds4_tensor       *w,
+        const float            *x,
+        uint32_t                n_groups,
+        uint64_t                group_dim,
+        uint64_t                rank,
+        ds4_cpu_decode_scratch *scratch) {
+    if (w->type == DS4_TENSOR_Q4_K) {
+        matvec_q4_K_grouped_rows_decode_scratch(out, m, w, x, n_groups,
+                                                group_dim, rank, scratch);
+    } else {
+        matvec_q8_0_grouped_rows_decode_scratch(out, m, w, x, n_groups,
+                                                group_dim, rank, scratch);
+    }
+}
+
+static void matmul_dense_grouped_batch(
+        float            *out,
+        const ds4_model  *m,
+        const ds4_tensor *w,
+        const float      *x,
+        uint64_t          n_tok,
+        uint32_t          n_groups,
+        uint64_t          group_dim,
+        uint64_t          rank) {
+    if (w->type == DS4_TENSOR_Q4_K) {
+        matmul_q4_K_grouped_batch(out, m, w, x, n_tok, n_groups, group_dim, rank);
+    } else {
+        matmul_q8_0_grouped_batch(out, m, w, x, n_tok, n_groups, group_dim, rank);
+    }
+}
+
+static void matmul_dense_batch(
+        float            *out,
+        const ds4_model  *m,
+        const ds4_tensor *w,
+        const float      *x,
+        uint64_t          n_tok) {
+    if (w->type == DS4_TENSOR_Q4_K) {
+        matmul_q4_K_batch(out, m, w, x, n_tok);
+    } else {
+        matmul_q8_0_batch(out, m, w, x, n_tok);
+    }
+}
+
 typedef struct {
     float *out;
     const float *data;
@@ -8667,12 +9288,14 @@ static void matvec_f32(float *out, const ds4_model *m, const ds4_tensor *w, cons
     ds4_parallel_for(w->dim[1], matvec_f32_worker, &ctx);
 }
 
-/* Dispatch for dense F32/F16/Q8_0 tensors used by auxiliary projections. */
+/* Dispatch for dense F32/F16/Q8_0/Q4_K tensors used by the attention and
+ * auxiliary projections. */
 static void matvec_any(float *out, const ds4_model *m, const ds4_tensor *w, const float *x) {
     switch (w->type) {
     case 0: matvec_f32(out, m, w, x); break;
     case 1: matvec_f16(out, m, w, x); break;
     case 8: matvec_q8_0(out, m, w, x); break;
+    case DS4_TENSOR_Q4_K: matvec_q4_K(out, m, w, x); break;
     default:
         ds4_die("unsupported tensor type for dense matvec");
     }
@@ -9661,40 +10284,6 @@ static void quantize_mid_pairs_worker(void *vctx, uint64_t p0, uint64_t p1) {
     }
 }
 
-typedef struct {
-    float *down_pair;
-    const uint8_t *base[DS4_MAX_EXPERT];
-    const block_q8_K *midq;
-    const uint32_t *pair_ids;
-    const uint32_t *expert_offset;
-    const uint32_t *active_expert;
-    uint64_t in_dim;
-    uint64_t out_dim;
-    uint64_t row_bytes[DS4_MAX_EXPERT];
-    uint64_t midq_blocks;
-} matvec_q2_k_batch_down_ctx;
-
-static DS4_MAYBE_UNUSED void matvec_q2_k_batch_down_worker(void *vctx, uint64_t task0, uint64_t task1) {
-    matvec_q2_k_batch_down_ctx *ctx = vctx;
-
-    for (uint64_t task = task0; task < task1; task++) {
-        const uint32_t active_idx = (uint32_t)(task / ctx->out_dim);
-        const uint64_t row = task - (uint64_t)active_idx * ctx->out_dim;
-        const uint32_t expert = ctx->active_expert[active_idx];
-        const uint32_t begin = ctx->expert_offset[expert];
-        const uint32_t end = ctx->expert_offset[expert + 1];
-        const block_q2_K *br = (const block_q2_K *)(ctx->base[expert] + row * ctx->row_bytes[expert]);
-
-        for (uint32_t i = begin; i < end; i++) {
-            const uint32_t pair_id = ctx->pair_ids[i];
-            const block_q8_K *xq = ctx->midq + (uint64_t)pair_id * ctx->midq_blocks;
-            ds4_vec_dot_q2_K_q8_K((int)ctx->in_dim,
-                                  ctx->down_pair + (uint64_t)pair_id * ctx->out_dim + row,
-                                  br, xq);
-        }
-    }
-}
-
 typedef struct {
     float *moe;
     const uint8_t *base[DS4_MAX_EXPERT];
@@ -10495,27 +11084,6 @@ static void matvec_q8_0_batch_accum_rows_worker(void *vctx, uint64_t row0, uint6
     }
 }
 
-typedef struct {
-    float *moe;
-    const float *down_pair;
-    uint32_t n_tok;
-    uint64_t out_dim;
-} sum_down_pairs_ctx;
-
-static DS4_MAYBE_UNUSED void sum_down_pairs_worker(void *vctx, uint64_t row0, uint64_t row1) {
-    sum_down_pairs_ctx *ctx = vctx;
-    for (uint64_t idx = row0; idx < row1; idx++) {
-        const uint32_t token = (uint32_t)(idx / ctx->out_dim);
-        const uint64_t row = idx - (uint64_t)token * ctx->out_dim;
-        float acc = 0.0f;
-        for (uint32_t slot = 0; slot < DS4_N_EXPERT_USED; slot++) {
-            const uint64_t pair_id = (uint64_t)token * DS4_N_EXPERT_USED + slot;
-            acc += ctx->down_pair[pair_id * ctx->out_dim + row];
-        }
-        ctx->moe[idx] = acc;
-    }
-}
-
 /* =========================================================================
  * Hyper-Connection Transforms.
  * =========================================================================
@@ -10949,9 +11517,9 @@ static void layer_q_projection_normed_one(
 
     const float *q_a_norm = tensor_data(model, layer->attn_q_a_norm);
 
-    matvec_q8_0(qr, model, layer->attn_q_a, norm);
+    matvec_any(qr, model, layer->attn_q_a, norm);
     rms_norm_weight(qr_norm, qr, q_a_norm, q_rank, DS4_RMS_EPS);
-    matvec_q8_0(q, model, layer->attn_q_b, qr_norm);
+    matvec_any(q, model, layer->attn_q_b, qr_norm);
     head_rms_norm_inplace(q, DS4_N_HEAD, DS4_N_HEAD_DIM, DS4_RMS_EPS);
 
     free(qr_norm);
@@ -10968,9 +11536,9 @@ static void layer_q_projection_with_lora_one(
     float *qr = xmalloc((size_t)q_rank * sizeof(qr[0]));
     const float *q_a_norm = tensor_data(model, layer->attn_q_a_norm);
 
-    matvec_q8_0(qr, model, layer->attn_q_a, norm);
+    matvec_any(qr, model, layer->attn_q_a, norm);
     rms_norm_weight(qr_norm, qr, q_a_norm, q_rank, DS4_RMS_EPS);
-    matvec_q8_0(q, model, layer->attn_q_b, qr_norm);
+    matvec_any(q, model, layer->attn_q_b, qr_norm);
     head_rms_norm_inplace(q, DS4_N_HEAD, DS4_N_HEAD_DIM, DS4_RMS_EPS);
 
     free(qr);
@@ -10986,7 +11554,7 @@ static void layer_kv_projection_normed_one(
 
     const float *kv_norm = tensor_data(model, layer->attn_kv_a_norm);
 
-    matvec_q8_0(raw, model, layer->attn_kv, normed);
+    matvec_any(raw, model, layer->attn_kv, normed);
     rms_norm_weight(kv, raw, kv_norm, DS4_N_HEAD_DIM, DS4_RMS_EPS);
 
     free(raw);
@@ -11001,9 +11569,9 @@ static void layer_q_projection_with_lora_one_decode_scratch(
         ds4_cpu_decode_scratch  * scratch) {
     const float *q_a_norm = tensor_data(model, layer->attn_q_a_norm);
 
-    matvec_q8_0_decode_scratch(scratch->qr, model, layer->attn_q_a, norm, scratch);
+    matvec_any_decode_scratch(scratch->qr, model, layer->attn_q_a, norm, scratch);
     rms_norm_weight(qr_norm, scratch->qr, q_a_norm, DS4_N_LORA_Q, DS4_RMS_EPS);
-    matvec_q8_0_decode_scratch(q, model, layer->attn_q_b, qr_norm, scratch);
+    matvec_any_decode_scratch(q, model, layer->attn_q_b, qr_norm, scratch);
     head_rms_norm_inplace(q, DS4_N_HEAD, DS4_N_HEAD_DIM, DS4_RMS_EPS);
 }
 
@@ -11015,7 +11583,7 @@ static void layer_kv_projection_normed_one_decode_scratch(
         ds4_cpu_decode_scratch  * scratch) {
     const float *kv_norm = tensor_data(model, layer->attn_kv_a_norm);
 
-    matvec_q8_0_decode_scratch(scratch->kv_raw, model, layer->attn_kv, normed, scratch);
+    matvec_any_decode_scratch(scratch->kv_raw, model, layer->attn_kv, normed, scratch);
     rms_norm_weight(kv, scratch->kv_raw, kv_norm, DS4_N_HEAD_DIM, DS4_RMS_EPS);
 }
 
@@ -11304,9 +11872,9 @@ static void layer_grouped_out_one(
 
     float *low = xcalloc((size_t)n_groups * rank, sizeof(low[0]));
 
-    matvec_q8_0_grouped_rows(low, model, layer->attn_output_a, heads, n_groups, group_dim, rank);
+    matvec_dense_grouped_rows(low, model, layer->attn_output_a, heads, n_groups, group_dim, rank);
 
-    matvec_q8_0(out, model, layer->attn_output_b, low);
+    matvec_any(out, model, layer->attn_output_b, low);
     free(low);
 }
 
@@ -11322,9 +11890,9 @@ static void layer_grouped_out_one_decode_scratch(
     const uint32_t rank = 1024;
 
     memset(scratch->attn_low, 0, (size_t)n_groups * rank * sizeof(scratch->attn_low[0]));
-    matvec_q8_0_grouped_rows_decode_scratch(scratch->attn_low, model, layer->attn_output_a,
-                                            heads, n_groups, group_dim, rank, scratch);
-    matvec_q8_0_decode_scratch(out, model, layer->attn_output_b, scratch->attn_low, scratch);
+    matvec_dense_grouped_rows_decode_scratch(scratch->attn_low, model, layer->attn_output_a,
+                                             heads, n_groups, group_dim, rank, scratch);
+    matvec_any_decode_scratch(out, model, layer->attn_output_b, scratch->attn_low, scratch);
 }
 
 static void layer_grouped_out_batch(
@@ -11340,9 +11908,9 @@ static void layer_grouped_out_batch(
 
     float *low = xcalloc((size_t)n_tok * n_groups * rank, sizeof(low[0]));
 
-    matmul_q8_0_grouped_batch(low, model, layer->attn_output_a, heads,
-                              n_tok, n_groups, group_dim, rank);
-    matmul_q8_0_batch(out, model, layer->attn_output_b, low, n_tok);
+    matmul_dense_grouped_batch(low, model, layer->attn_output_a, heads,
+                               n_tok, n_groups, group_dim, rank);
+    matmul_dense_batch(out, model, layer->attn_output_b, low, n_tok);
 
     free(low);
 }
@@ -13065,6 +13633,7 @@ static void cpu_decode_scratch_init(ds4_cpu_decode_scratch *scratch, uint32_t ct
     scratch->routed_mid_all = xmalloc((size_t)DS4_N_EXPERT_USED * DS4_N_FF_EXP * sizeof(float));
     scratch->routed_xq = xmalloc((size_t)(DS4_N_EMBD / QK_K) * sizeof(block_q8_K));
     scratch->routed_midq = xmalloc((size_t)DS4_N_EXPERT_USED * (DS4_N_FF_EXP / QK_K) * sizeof(block_q8_K));
+    scratch->dense_xq = xmalloc((size_t)((q8_cap + QK_K - 1u) / QK_K) * sizeof(block_q8_K));
     scratch->routed_q8_xq = xmalloc((size_t)routed_q8_x_blocks * 32u);
     scratch->routed_q8_xscale = xmalloc((size_t)routed_q8_x_blocks * sizeof(float));
     scratch->routed_q8_midq = xmalloc((size_t)DS4_N_EXPERT_USED * routed_q8_mid_blocks * 32u);
@@ -13095,6 +13664,7 @@ static void cpu_decode_scratch_free(ds4_cpu_decode_scratch *scratch) {
     free(scratch->routed_q8_midq);
     free(scratch->routed_q8_xscale);
     free(scratch->routed_q8_xq);
+    free(scratch->dense_xq);
     free(scratch->routed_midq);
     free(scratch->routed_xq);
     free(scratch->routed_mid_all);
@@ -14044,7 +14614,7 @@ static void layer_attention_raw_swa_batch(
     if (profile) t_hc_norm = now_sec() - t0;
 
     t0 = profile ? now_sec() : 0.0;
-    matmul_q8_0_batch(qr, model, layer->attn_q_a, attn_norm, n_tok);
+    matmul_dense_batch(qr, model, layer->attn_q_a, attn_norm, n_tok);
     for (uint32_t t = 0; t < n_tok; t++) {
         rms_norm_weight(qr_norm + (uint64_t)t * q_rank,
                         qr + (uint64_t)t * q_rank,
@@ -14052,7 +14622,7 @@ static void layer_attention_raw_swa_batch(
                         q_rank,
                         DS4_RMS_EPS);
     }
-    matmul_q8_0_batch(q, model, layer->attn_q_b, qr_norm, n_tok);
+    matmul_dense_batch(q, model, layer->attn_q_b, qr_norm, n_tok);
     for (uint32_t t = 0; t < n_tok; t++) {
         head_rms_norm_inplace(q + (uint64_t)t * q_dim,
                               DS4_N_HEAD,
@@ -14062,7 +14632,7 @@ static void layer_attention_raw_swa_batch(
     if (profile) t_q = now_sec() - t0;
 
     t0 = profile ? now_sec() : 0.0;
-    matmul_q8_0_batch(kv_raw, model, layer->attn_kv, attn_norm, n_tok);
+    matmul_dense_batch(kv_raw, model, layer->attn_kv, attn_norm, n_tok);
     for (uint32_t t = 0; t < n_tok; t++) {
         rms_norm_weight(kv + (uint64_t)t * DS4_N_HEAD_DIM,
                         kv_raw + (uint64_t)t * DS4_N_HEAD_DIM,
@@ -15867,6 +16437,23 @@ typedef struct {
     uint32_t spec_prefix_n_comp[DS4_SPEC_PREFIX_SLOTS][DS4_MAX_LAYER];
     uint32_t spec_prefix_n_index_comp[DS4_SPEC_PREFIX_SLOTS][DS4_MAX_LAYER];
     bool spec_capture_prefixes;
+    /* While exact-N collects GPU router rows, hash routing must not start the
+     * ordinary per-token SSD early load or publish a process-global selected
+     * override.  The routed tails consume the GPU-selected matrix through the
+     * exact-row union scope instead. */
+    bool spec_exactn_union_collect_routes;
+    /* Emergency/tape-local CUDA graph gate.  Graph keys now use stable device
+     * storage identities rather than short-lived wrapper handles, so the
+     * resident exact-N tape may opt back in.  Other experimental tapes remain
+     * eager until they receive the same lifetime and oracle coverage. */
+    bool spec_disable_decode_graphs;
+    /* Zero is ordinary decode. Exact-N assigns a stable domain-tagged row id
+     * while encoding each speculative row, then restores this field. */
+    uint32_t decode_graph_variant;
+    /* Batch verification normally takes the per-row compressor path when it
+     * captures intermediate frontiers.  Rollback+replay DSpark verification
+     * needs the same arithmetic path, but not the frontier copies themselves. */
+    bool spec_force_sequential_compressor;
     uint32_t raw_cap;
     /* Maximum compressed-row capacity across layers.  Shared work buffers use
      * this worst-case size because ratio-4 indexer layers can still reach it. */
@@ -15883,6 +16470,8 @@ typedef struct {
      * predictable: each pointer names an actual DS4 stage. */
     ds4_gpu_tensor *comp_kv_cur_by_tier[DS4_MAX_GPUS];
     ds4_gpu_tensor *comp_sc_cur_by_tier[DS4_MAX_GPUS];
+    /* The Q4 Q-A/KV compound projects both compressor pairs in one dispatch.
+     * Attention and indexer outputs must therefore have disjoint scratch. */
     ds4_gpu_tensor *index_comp_kv_cur_by_tier[DS4_MAX_GPUS];
     ds4_gpu_tensor *index_comp_sc_cur_by_tier[DS4_MAX_GPUS];
     ds4_gpu_tensor *attn_comp_stage_by_tier[DS4_MAX_GPUS];
@@ -15935,11 +16524,14 @@ typedef struct {
     ds4_gpu_tensor *dspark_stage0_proj;
     ds4_gpu_tensor *dspark_main_x;
     ds4_gpu_tensor *dspark_draft_tokens;
+    /* Device-resident confidence/Markov returns one compact result for
+     * the whole speculative block.  Keep it separate from draft_tokens: that
+     * buffer is only block_size * i32 and is too small for the 64-byte ABI. */
+    ds4_gpu_tensor *dspark_device_proposal;
     ds4_gpu_tensor *dspark_draft_hc;
     ds4_gpu_tensor *dspark_target_hc;
     ds4_gpu_tensor *dspark_stage_input_hc;
     ds4_gpu_tensor *dspark_stage_output_hc;
-    ds4_gpu_tensor *dspark_position_ids;
     ds4_gpu_tensor *dspark_raw_cache[DS4_DSPARK_MAX_STAGES];
     uint32_t dspark_cache_cap;
     uint32_t dspark_cache_start;
@@ -16650,9 +17242,9 @@ static void metal_graph_free(ds4_gpu_graph *g) {
         ds4_gpu_tensor_free(g->attn_out_by_tier[t]);
         ds4_gpu_tensor_free(g->attn_low_by_tier[t]);
         ds4_gpu_tensor_free(g->heads_by_tier[t]);
-        ds4_gpu_tensor_free(g->comp_sc_cur_by_tier[t]);
-        ds4_gpu_tensor_free(g->index_comp_kv_cur_by_tier[t]);
         ds4_gpu_tensor_free(g->index_comp_sc_cur_by_tier[t]);
+        ds4_gpu_tensor_free(g->index_comp_kv_cur_by_tier[t]);
+        ds4_gpu_tensor_free(g->comp_sc_cur_by_tier[t]);
         ds4_gpu_tensor_free(g->comp_kv_cur_by_tier[t]);
         ds4_gpu_tensor_free(g->attn_comp_stage_by_tier[t]);
         ds4_gpu_tensor_free(g->comp_mask_by_tier[t]);
@@ -16718,11 +17310,11 @@ static void metal_graph_free(ds4_gpu_graph *g) {
         ds4_gpu_tensor_free(g->flat_hc_by_tier[t]);
         ds4_gpu_tensor_free(g->cur_hc_by_tier[t]);
     }
-    ds4_gpu_tensor_free(g->dspark_position_ids);
     ds4_gpu_tensor_free(g->dspark_stage_output_hc);
     ds4_gpu_tensor_free(g->dspark_stage_input_hc);
     ds4_gpu_tensor_free(g->dspark_target_hc);
     ds4_gpu_tensor_free(g->dspark_draft_hc);
+    ds4_gpu_tensor_free(g->dspark_device_proposal);
     ds4_gpu_tensor_free(g->dspark_draft_tokens);
     for (uint32_t stage = 0; stage < DS4_DSPARK_MAX_STAGES; stage++) {
         ds4_gpu_tensor_free(g->dspark_raw_cache[stage]);
@@ -16895,6 +17487,12 @@ static bool metal_graph_configure_dspark_capture(
         const uint64_t hc_dim = (uint64_t)DS4_N_HC * DS4_N_EMBD;
         g->dspark_draft_tokens =
             ds4_gpu_tensor_alloc((uint64_t)dw->block_size * sizeof(int32_t));
+#if !defined(DS4_ROCM_BUILD)
+        /* Opportunistic CUDA/Metal accelerator scratch.  Allocation failure
+         * must not make the ordinary DSpark graph unavailable. */
+        g->dspark_device_proposal = ds4_gpu_tensor_alloc(
+                DS4_GPU_DSPARK_DEVICE_PROPOSAL_BYTES);
+#endif
         g->dspark_draft_hc =
             ds4_gpu_tensor_alloc((uint64_t)dw->block_size * hc_dim * sizeof(float));
         g->dspark_target_hc =
@@ -16905,12 +17503,9 @@ static bool metal_graph_configure_dspark_capture(
         g->dspark_stage_output_hc =
             ds4_gpu_tensor_alloc((uint64_t)dw->block_size *
                                  hc_dim * sizeof(float));
-        g->dspark_position_ids =
-            ds4_gpu_tensor_alloc((uint64_t)(dw->block_size + 1u) *
-                                 sizeof(int32_t));
         if (!g->dspark_draft_tokens || !g->dspark_draft_hc ||
             !g->dspark_target_hc || !g->dspark_stage_input_hc ||
-            !g->dspark_stage_output_hc || !g->dspark_position_ids) {
+            !g->dspark_stage_output_hc) {
             return false;
         }
         if (dw->n_stages != 0 && g->raw_cap != 0) {
@@ -20604,6 +21199,25 @@ static bool metal_graph_use_reference_hc_decode(void) {
     return metal_graph_env_flag("DS4_METAL_DISABLE_HC_FUSION", &cache);
 }
 
+static bool metal_graph_use_hc_norm_mix_f16(
+        const ds4_tensor *weight,
+        uint64_t          in_dim,
+        uint64_t          out_dim) {
+#if !defined(DS4_NO_GPU) && !defined(DS4_ROCM_BUILD)
+    return weight != NULL &&
+           in_dim == 16384u &&
+           out_dim == 24u &&
+           weight->type == DS4_TENSOR_F16 &&
+           !metal_graph_use_reference_hc_decode() &&
+           ds4_gpu_hc_rms_norm_mix_f16_available() != 0;
+#else
+    (void)weight;
+    (void)in_dim;
+    (void)out_dim;
+    return false;
+#endif
+}
+
 static bool metal_graph_use_reference_kv_decode(void) {
     static int cache = -1;
     return metal_graph_env_flag("DS4_METAL_DISABLE_KV_FUSION", &cache);
@@ -21079,7 +21693,8 @@ static bool metal_graph_attention_output_dense_quant_low(
         uint64_t                rank,
         uint32_t                group0,
         uint32_t                group_cnt,
-        const ds4_gpu_tensor *heads);
+        const ds4_gpu_tensor *heads,
+        bool                   resident_decode);
 static bool metal_graph_attention_output_dense_quant_tp(
         ds4_gpu_tensor       *out,
         ds4_gpu_tensor       *low,
@@ -21187,6 +21802,25 @@ static bool metal_graph_use_iq2_selected_async_load(const ds4_gpu_graph *g) {
 
 static bool metal_graph_use_iq2_selected_async_early_commit(
         const ds4_gpu_graph *g) {
+    /* The selected-id event already lets the service thread start the IQ2
+     * expert load while router and shared-expert work remain in one command
+     * buffer.  Keep the historical extra frontier as an explicit rollback
+     * for isolated A/Bs instead of paying it by default. */
+    return g &&
+           g->ssd_streaming &&
+#ifndef DS4_ROCM_BUILD
+           getenv("DS4_METAL_ENABLE_IQ2_SELECTED_ASYNC_EARLY_COMMIT") != NULL &&
+           getenv("DS4_METAL_DISABLE_STREAMING_SELECTED_ASYNC_EARLY_COMMIT") == NULL;
+#else
+           false;
+#endif
+}
+
+static bool metal_graph_use_selected_async_early_commit_legacy(
+        const ds4_gpu_graph *g) {
+    /* MXFP4 and native CUDA were not part of the IQ2 Metal scheduling A/B.
+     * Preserve their established default while the IQ2-only rollback above
+     * selects the merged Metal boundary. */
     return g &&
            g->ssd_streaming &&
 #ifndef DS4_ROCM_BUILD
@@ -21754,6 +22388,13 @@ static bool metal_graph_decode_set_hash_selected_override(
         return true;
     }
 
+    /* ds4_gpu_router_select_tensor() has already encoded the hash lookup into
+     * this row's router_selected/router_weights views.  Exact-N reads those
+     * GPU-produced rows together after the prefix batch, so publishing the
+     * ordinary host override here would overwrite each preceding row and its
+     * early-load request would serialize/load the same layer once per token. */
+    if (g && g->spec_exactn_union_collect_routes) return true;
+
     int selected[DS4_MAX_EXPERT_USED];
     int32_t selected_i32[DS4_MAX_EXPERT_USED];
     layer_hash_selected_experts(selected, model, layer, (int)token);
@@ -22140,11 +22781,15 @@ typedef struct metal_graph_selected_async_load {
     /* Selected ids remain usable for a synchronous retry if the service
      * thread cannot stage the cache load without waiting on GPU work. */
     bool                      ids_ok;
+    bool                      cuda_event_pipeline;
+    bool                      cuda_event_required;
+    bool                      fail_closed;
     ds4_gpu_tensor           *router_selected;
     const ds4_model          *model;
     const ds4_layer_weights  *layer;
     uint32_t                  il;
     uint64_t                  event_value;
+    uint64_t                  upload_event_value;
     uint64_t                  gate_expert_bytes;
     uint64_t                  down_expert_bytes;
     int32_t                   selected_ids[DS4_MAX_EXPERT_USED];
@@ -22171,8 +22816,54 @@ static void metal_graph_selected_async_load_run(
         DS4_N_EXPERT_USED == 0 || DS4_N_EXPERT_USED > DS4_MAX_EXPERT_USED) {
         return;
     }
+#if !defined(__APPLE__) && !defined(DS4_ROCM_BUILD) && !defined(DS4_NO_GPU)
+    if (job->cuda_event_pipeline &&
+        ds4_gpu_cuda_stream_selected_set_owner_device() == 0) {
+        job->fail_closed = job->cuda_event_required;
+        return;
+    }
+#endif
     if (job->event_value != 0) {
-#ifdef DS4_ROCM_BUILD
+#if !defined(__APPLE__) && !defined(DS4_ROCM_BUILD) && !defined(DS4_NO_GPU)
+        if (job->cuda_event_pipeline) {
+            if (ds4_gpu_tensor_read_after_selected_event(
+                        job->router_selected,
+                        0,
+                        job->selected_ids,
+                        (uint64_t)DS4_N_EXPERT_USED *
+                            sizeof(job->selected_ids[0]),
+                        job->event_value,
+                        "selected-id async expert load") == 0) {
+                if (job->cuda_event_required) {
+                    job->fail_closed = true;
+                    return;
+                }
+                ds4_gpu_cuda_stream_selected_event_note_fallback();
+                if (ds4_gpu_synchronize() == 0 ||
+                    ds4_gpu_tensor_read(
+                        job->router_selected,
+                        0,
+                        job->selected_ids,
+                        (uint64_t)DS4_N_EXPERT_USED *
+                            sizeof(job->selected_ids[0])) == 0) {
+                    return;
+                }
+                job->cuda_event_pipeline = false;
+            }
+        } else {
+            if (ds4_gpu_wait_selected_readback_ready(
+                        job->event_value,
+                        "selected-id async expert load") == 0 ||
+                ds4_gpu_tensor_read(
+                        job->router_selected,
+                        0,
+                        job->selected_ids,
+                        (uint64_t)DS4_N_EXPERT_USED *
+                            sizeof(job->selected_ids[0])) == 0) {
+                return;
+            }
+        }
+#elif defined(DS4_ROCM_BUILD)
         if (ds4_gpu_tensor_read_after_selected_event(
                     job->router_selected,
                     0,
@@ -22184,15 +22875,15 @@ static void metal_graph_selected_async_load_run(
             return;
         }
 #else
-        if (ds4_gpu_wait_selected_readback_ready(job->event_value,
-                                                 "selected-id async expert load") == 0) {
-            return;
-        }
-        if (ds4_gpu_tensor_read(job->router_selected,
-                                0,
-                                job->selected_ids,
-                                (uint64_t)DS4_N_EXPERT_USED *
-                                    sizeof(job->selected_ids[0])) == 0) {
+        if (ds4_gpu_wait_selected_readback_ready(
+                    job->event_value,
+                    "selected-id async expert load") == 0 ||
+            ds4_gpu_tensor_read(
+                    job->router_selected,
+                    0,
+                    job->selected_ids,
+                    (uint64_t)DS4_N_EXPERT_USED *
+                        sizeof(job->selected_ids[0])) == 0) {
             return;
         }
 #endif
@@ -22215,6 +22906,32 @@ static void metal_graph_selected_async_load_run(
                                        job->il,
                                        job->gate_expert_bytes,
                                        job->down_expert_bytes);
+#if !defined(__APPLE__) && !defined(DS4_ROCM_BUILD) && !defined(DS4_NO_GPU)
+    if (job->cuda_event_pipeline) {
+        const int load_rc =
+            ds4_gpu_stream_expert_cache_begin_selected_load_async(
+                &table,
+                job->selected_ids,
+                DS4_N_EXPERT_USED,
+                &job->upload_event_value);
+        if (load_rc <= 0) {
+            if (load_rc < 0 || job->cuda_event_required) {
+                job->fail_closed = true;
+                return;
+            }
+            /* A zero result guarantees that no H2D operation was submitted,
+             * so the established synchronous loader is a safe rollback. */
+            ds4_gpu_cuda_stream_selected_event_note_fallback();
+            job->cuda_event_pipeline = false;
+            if (ds4_gpu_stream_expert_cache_begin_selected_load(
+                        &table,
+                        job->selected_ids,
+                        DS4_N_EXPERT_USED) == 0) {
+                return;
+            }
+        }
+    } else
+#endif
     if (ds4_gpu_stream_expert_cache_begin_selected_load(
                 &table,
                 job->selected_ids,
@@ -22284,6 +23001,8 @@ static DS4_MAYBE_UNUSED bool metal_graph_selected_async_load_start_tensor(
         const ds4_layer_weights         *layer,
         uint32_t                         il,
         uint64_t                         event_value,
+        bool                             cuda_event_pipeline,
+        bool                             cuda_event_required,
         uint64_t                         gate_expert_bytes,
         uint64_t                         down_expert_bytes) {
     if (!job || !router_selected || event_value == 0) return false;
@@ -22294,9 +23013,10 @@ static DS4_MAYBE_UNUSED bool metal_graph_selected_async_load_start_tensor(
     job->layer = layer;
     job->il = il;
     job->event_value = event_value;
+    job->cuda_event_pipeline = cuda_event_pipeline;
+    job->cuda_event_required = cuda_event_required;
     job->gate_expert_bytes = gate_expert_bytes;
     job->down_expert_bytes = down_expert_bytes;
-
     pthread_mutex_lock(&g_metal_graph_selected_async_load_mutex);
     if (g_metal_graph_selected_async_load_has_job ||
         g_metal_graph_selected_async_load_done) {
@@ -22319,6 +23039,8 @@ static DS4_MAYBE_UNUSED bool metal_graph_selected_async_load_start(
         const ds4_layer_weights         *layer,
         uint32_t                         il,
         uint64_t                         event_value,
+        bool                             cuda_event_pipeline,
+        bool                             cuda_event_required,
         uint64_t                         gate_expert_bytes,
         uint64_t                         down_expert_bytes) {
     return metal_graph_selected_async_load_start_tensor(
@@ -22328,6 +23050,8 @@ static DS4_MAYBE_UNUSED bool metal_graph_selected_async_load_start(
             layer,
             il,
             event_value,
+            cuda_event_pipeline,
+            cuda_event_required,
             gate_expert_bytes,
             down_expert_bytes);
 }
@@ -22867,11 +23591,16 @@ static bool metal_graph_encode_decode_layer_phase(
      * eagerly, byte-identical to before.  A replayed graph re-runs the
      * captured kernels byte-for-byte, so replay output is bit-identical
      * to the eager encode it recorded. */
+    /* Equivalent tensor views are frequently recreated by speculative tapes.
+     * The CUDA graph bakes the device address, not the host-side wrapper, into
+     * its nodes; key the cache by that same stable address.  Metal/ROCm/CPU do
+     * not execute these graphs and retain the historical handle identity. */
     const bool decode_graphs_common_ok =
         !g->placement &&
         g->tp_world <= 1 &&
         !g->cuda_tp_decode &&
         !g->ssd_streaming &&
+        !g->spec_disable_decode_graphs &&
         !g->materialize_ffn_out &&
         !decode_stage_profile &&
         !g_expert_profile.active &&
@@ -22890,10 +23619,20 @@ static bool metal_graph_encode_decode_layer_phase(
         memset(&isla_key, 0, sizeof(isla_key));
         isla_key.il = il;
         isla_key.island = 0u;
-        isla_key.cur_hc = (void *)metal_graph_cur_hc(g);
-        isla_key.after_attn_hc = (void *)metal_graph_after_attn_hc(g);
-        isla_key.after_ffn_hc = (void *)metal_graph_after_ffn_hc(g);
-        isla_key.attn_norm = (void *)metal_graph_attn_norm(g);
+        isla_key.variant = g->decode_graph_variant;
+#if !defined(DS4_NO_GPU) && !defined(__APPLE__) && !defined(DS4_ROCM_BUILD)
+#define DS4_DECODE_GRAPH_TENSOR_KEY(t) \
+        ((void *)(uintptr_t)ds4_gpu_tensor_storage_key((t)))
+#else
+#define DS4_DECODE_GRAPH_TENSOR_KEY(t) ((void *)(t))
+#endif
+        isla_key.cur_hc = DS4_DECODE_GRAPH_TENSOR_KEY(metal_graph_cur_hc(g));
+        isla_key.after_attn_hc =
+            DS4_DECODE_GRAPH_TENSOR_KEY(metal_graph_after_attn_hc(g));
+        isla_key.after_ffn_hc =
+            DS4_DECODE_GRAPH_TENSOR_KEY(metal_graph_after_ffn_hc(g));
+        isla_key.attn_norm =
+            DS4_DECODE_GRAPH_TENSOR_KEY(metal_graph_attn_norm(g));
         for (;;) {
             const int isla_state = ds4_gpu_decode_graph_begin(&isla_key);
             if (isla_state == 1) {
@@ -22958,13 +23697,12 @@ static bool metal_graph_encode_decode_layer_phase(
         /* Fused norm+mix removes one decode dispatch per layer; the kernel
          * reproduces both reduction trees bit-exactly (see dsv4_hc.metal). */
         const bool fuse_norm_mix =
-            hc_dim == 16384u && mix_hc == 24u &&
-            layer->hc_attn_fn->type == DS4_TENSOR_F16 &&
-            !metal_graph_use_reference_hc_decode() &&
+            metal_graph_use_hc_norm_mix_f16(layer->hc_attn_fn,
+                                             hc_dim,
+                                             mix_hc) &&
             getenv("DS4_METAL_DISABLE_PRE_M5_HC_NORM_MIX_FUSE") == NULL &&
             (ds4_gpu_device_is_pre_m5_apple_silicon() ||
-             ds4_gpu_device_is_m5_apple_silicon()) &&
-            ds4_gpu_hc_rms_norm_mix_f16_available() != 0;
+             ds4_gpu_device_is_m5_apple_silicon());
 #if defined(__APPLE__)
         const bool fuse_producer_pre_norm =
             fuse_norm_mix && fuse_hc_norm &&
@@ -23123,19 +23861,239 @@ static bool metal_graph_encode_decode_layer_phase(
     bool fuse_kv_rope_store = false;
     bool qkv_pair_quad_fused = false;
     if (!resume_after_qkv) {
+    /* AProjQ4 GGUFs carry q_a/kv (and the output pair) as Q4_K; the fused
+     * Q8_0 pair kernel and the plain Q8_0 matvec would read those blocks as
+     * Q8_0 and produce garbage, so both stay gated on the actual type and
+     * everything else goes through the type-dispatching dense-quant path. */
+    const bool qkv_proj_q8 =
+        layer->attn_q_a->type == DS4_TENSOR_Q8_0 &&
+        layer->attn_kv->type == DS4_TENSOR_Q8_0;
+    const bool qkv_proj_q4 =
+        layer->attn_q_a->type == DS4_TENSOR_Q4_K &&
+        layer->attn_kv->type == DS4_TENSOR_Q4_K;
     bool qkv_pair_projected = resume_after_qa_kv_raw;
+#if defined(__APPLE__) && !defined(DS4_NO_GPU)
+    /* M1-M5 decode fusion: the q_a/kv Q8 pair and the eligible F16
+     * compressor projections all read the same normalized attention row and
+     * write disjoint outputs.  Resident FULL decode retains its established
+     * default.  SSD FULL decode and exact-union's TO_ROUTER prefix stay
+     * opt-in because one compound grid can change model-page scheduling even
+     * though it reads exactly the same bytes as the separate dispatches. */
+    const uint32_t q8_compound_ratio =
+        compressed ? ds4_layer_compress_ratio(il) : 0u;
+    const uint32_t q8_compound_width0 =
+        q8_compound_ratio == 4u ? 2u * DS4_N_HEAD_DIM : DS4_N_HEAD_DIM;
+    const uint32_t q8_compound_width1 =
+        q8_compound_ratio == 4u ? 2u * DS4_N_INDEXER_HEAD_DIM : 0u;
+    const bool q8_compound_resident_scope =
+        phase == METAL_DECODE_LAYER_FULL &&
+        !g->ssd_streaming && !g->ssd_streaming_cold;
+    const bool q8_compound_required =
+        metal_graph_tp_env_flag(
+            "DS4_METAL_REQUIRE_Q8_QKV_COMPRESSOR_FUSE", false);
+    const bool q8_compound_stream_enabled =
+        q8_compound_required ||
+        metal_graph_tp_env_flag(
+            "DS4_METAL_ENABLE_Q8_QKV_COMPRESSOR_FUSE", false);
+    const bool q8_compound_stream_scope =
+        q8_compound_stream_enabled &&
+        ((phase == METAL_DECODE_LAYER_FULL &&
+          (g->ssd_streaming || g->ssd_streaming_cold)) ||
+         (phase == METAL_DECODE_LAYER_TO_ROUTER &&
+          g->spec_exactn_union_collect_routes));
+    const bool q8_compound_scope =
+        q8_compound_resident_scope || q8_compound_stream_scope;
+    const bool q8_compound_device = q8_compound_ratio == 4u
+        ? metal_graph_ported_m5_decode_feature_enabled(
+              "DS4_METAL_DISABLE_PRE_M5_QKV_PAIR_QUAD_FUSE",
+              "DS4_METAL_DISABLE_M5_QKV_PAIR_QUAD_FUSE")
+        : (q8_compound_ratio == 128u &&
+           metal_graph_ported_m5_decode_feature_enabled(
+              "DS4_METAL_DISABLE_PRE_M5_QKV_PAIR_COMPRESSOR_FUSE",
+              "DS4_METAL_DISABLE_M5_QKV_PAIR_COMPRESSOR_FUSE"));
+    const bool q8_compound_index_ok =
+        q8_compound_ratio != 4u ||
+        (layer->indexer_compressor_kv && layer->indexer_compressor_gate &&
+         layer->indexer_compressor_ape &&
+         layer->indexer_compressor_kv->type == DS4_TENSOR_F16 &&
+         layer->indexer_compressor_gate->type == DS4_TENSOR_F16 &&
+         layer->indexer_compressor_kv->dim[0] == DS4_N_EMBD &&
+         layer->indexer_compressor_gate->dim[0] == DS4_N_EMBD &&
+         layer->indexer_compressor_kv->dim[1] == q8_compound_width1 &&
+         layer->indexer_compressor_gate->dim[1] == q8_compound_width1);
+    const bool q8_compound_eligible =
+        !resume_after_qa_kv_raw && ok && qkv_rms_fused && qkv_proj_q8 &&
+        compressed && q8_compound_scope && q8_compound_device &&
+        (q8_compound_ratio == 4u || q8_compound_ratio == 128u) &&
+        g->cuda_qkv_pair && !metal_graph_use_reference_qkv_pair_proj() &&
+        !metal_graph_use_reference_compressor_pair_proj() &&
+        layer->attn_compressor_kv && layer->attn_compressor_gate &&
+        layer->attn_compressor_ape &&
+        layer->attn_compressor_kv->type == DS4_TENSOR_F16 &&
+        layer->attn_compressor_gate->type == DS4_TENSOR_F16 &&
+        layer->attn_compressor_kv->dim[0] == DS4_N_EMBD &&
+        layer->attn_compressor_gate->dim[0] == DS4_N_EMBD &&
+        layer->attn_compressor_kv->dim[1] == q8_compound_width0 &&
+        layer->attn_compressor_gate->dim[1] == q8_compound_width0 &&
+        q8_compound_index_ok;
+    if (q8_compound_eligible) {
+        ds4_gpu_tensor *out1_kv = q8_compound_width1
+            ? metal_graph_index_comp_kv_cur(g) : metal_graph_comp_kv_cur(g);
+        ds4_gpu_tensor *out1_sc = q8_compound_width1
+            ? metal_graph_index_comp_sc_cur(g) : metal_graph_comp_sc_cur(g);
+        ds4_gpu_tensor *state1_kv = q8_compound_width1
+            ? g->layer_index_state_kv[il] : g->layer_attn_state_kv[il];
+        ds4_gpu_tensor *state1_sc = q8_compound_width1
+            ? g->layer_index_state_score[il] : g->layer_attn_state_score[il];
+        const uint64_t weight1_kv = q8_compound_width1
+            ? layer->indexer_compressor_kv->abs_offset
+            : layer->attn_compressor_kv->abs_offset;
+        const uint64_t weight1_sc = q8_compound_width1
+            ? layer->indexer_compressor_gate->abs_offset
+            : layer->attn_compressor_gate->abs_offset;
+        const uint64_t ape1 = q8_compound_width1
+            ? layer->indexer_compressor_ape->abs_offset
+            : layer->attn_compressor_ape->abs_offset;
+        const uint32_t ape1_type = q8_compound_width1
+            ? layer->indexer_compressor_ape->type
+            : layer->attn_compressor_ape->type;
+        const int fused = ds4_gpu_qkv_pair_quad_compressor_store_tensor(
+                metal_graph_qr(g), metal_graph_kv_raw(g),
+                metal_graph_comp_kv_cur(g), metal_graph_comp_sc_cur(g),
+                out1_kv, out1_sc,
+                g->layer_attn_state_kv[il],
+                g->layer_attn_state_score[il],
+                state1_kv, state1_sc,
+                model->map, model->size,
+                layer->attn_q_a->abs_offset,
+                layer->attn_kv->abs_offset,
+                layer->attn_compressor_kv->abs_offset,
+                layer->attn_compressor_gate->abs_offset,
+                weight1_kv, weight1_sc,
+                layer->attn_compressor_ape->abs_offset,
+                layer->attn_compressor_ape->type,
+                ape1, ape1_type,
+                DS4_N_EMBD, (uint32_t)q_rank, DS4_N_HEAD_DIM,
+                q8_compound_width0, q8_compound_width1,
+                metal_graph_attn_norm(g), q8_compound_ratio, pos);
+        if (fused < 0) {
+            ok = false;
+        } else if (fused > 0) {
+            qkv_pair_projected = true;
+            qkv_pair_quad_fused = true;
+        }
+    }
+    if (ok && !qkv_pair_quad_fused &&
+        q8_compound_required &&
+        !resume_after_qa_kv_raw && qkv_proj_q8 && compressed &&
+        q8_compound_scope &&
+        (q8_compound_ratio == 4u || q8_compound_ratio == 128u)) {
+        fprintf(stderr,
+                "ds4: required Metal Q8 QKV/compressor compound was not selected "
+                "at layer %u ratio=%u phase=%u\n",
+                il, q8_compound_ratio, (unsigned)phase);
+        ok = false;
+    }
+#endif
+#if defined(__APPLE__) && !defined(DS4_NO_GPU)
+    /* Fuse the Q4 Q-A/KV pair with the compressor projections that consume
+     * the same normalized row.  Keep the larger compound dispatch opt-in in
+     * both exact-union and ordinary FULL decode: on the measured M1 Pro SSD
+     * path it reduced dispatch count but did not improve verifier time. */
+    const uint32_t q4_compound_ratio =
+        compressed ? ds4_layer_compress_ratio(il) : 0u;
+    const uint32_t q4_compound_width0 =
+        q4_compound_ratio == 4u ? 2u * DS4_N_HEAD_DIM : DS4_N_HEAD_DIM;
+    const uint32_t q4_compound_width1 =
+        q4_compound_ratio == 4u ? 2u * DS4_N_INDEXER_HEAD_DIM : 0u;
+    const bool q4_compound_scope =
+        getenv("DS4_METAL_ENABLE_Q4_QKV_COMPRESSOR_FUSE") != NULL &&
+        ((phase == METAL_DECODE_LAYER_TO_ROUTER &&
+          g->spec_exactn_union_collect_routes) ||
+         phase == METAL_DECODE_LAYER_FULL);
+    const bool q4_compound_index_ok =
+        q4_compound_ratio != 4u ||
+        (layer->indexer_compressor_kv && layer->indexer_compressor_gate &&
+         layer->indexer_compressor_ape &&
+         layer->indexer_compressor_kv->type == DS4_TENSOR_F16 &&
+         layer->indexer_compressor_gate->type == DS4_TENSOR_F16 &&
+         layer->indexer_compressor_kv->dim[0] == DS4_N_EMBD &&
+         layer->indexer_compressor_gate->dim[0] == DS4_N_EMBD &&
+         layer->indexer_compressor_kv->dim[1] == q4_compound_width1 &&
+         layer->indexer_compressor_gate->dim[1] == q4_compound_width1);
+    if (!resume_after_qa_kv_raw && ok && qkv_rms_fused && qkv_proj_q4 &&
+        compressed && q4_compound_scope &&
+        (q4_compound_ratio == 4u || q4_compound_ratio == 128u) &&
+        !metal_graph_use_reference_qkv_pair_proj() &&
+        !metal_graph_use_reference_compressor_pair_proj() &&
+        layer->attn_compressor_kv && layer->attn_compressor_gate &&
+        layer->attn_compressor_ape &&
+        layer->attn_compressor_kv->type == DS4_TENSOR_F16 &&
+        layer->attn_compressor_gate->type == DS4_TENSOR_F16 &&
+        layer->attn_compressor_kv->dim[0] == DS4_N_EMBD &&
+        layer->attn_compressor_gate->dim[0] == DS4_N_EMBD &&
+        layer->attn_compressor_kv->dim[1] == q4_compound_width0 &&
+        layer->attn_compressor_gate->dim[1] == q4_compound_width0 &&
+        q4_compound_index_ok) {
+        ds4_gpu_tensor *out1_kv = q4_compound_width1
+            ? metal_graph_index_comp_kv_cur(g) : metal_graph_comp_kv_cur(g);
+        ds4_gpu_tensor *out1_sc = q4_compound_width1
+            ? metal_graph_index_comp_sc_cur(g) : metal_graph_comp_sc_cur(g);
+        ds4_gpu_tensor *state1_kv = q4_compound_width1
+            ? g->layer_index_state_kv[il] : g->layer_attn_state_kv[il];
+        ds4_gpu_tensor *state1_sc = q4_compound_width1
+            ? g->layer_index_state_score[il] : g->layer_attn_state_score[il];
+        const uint64_t weight1_kv = q4_compound_width1
+            ? layer->indexer_compressor_kv->abs_offset
+            : layer->attn_compressor_kv->abs_offset;
+        const uint64_t weight1_sc = q4_compound_width1
+            ? layer->indexer_compressor_gate->abs_offset
+            : layer->attn_compressor_gate->abs_offset;
+        const uint64_t ape1 = q4_compound_width1
+            ? layer->indexer_compressor_ape->abs_offset
+            : layer->attn_compressor_ape->abs_offset;
+        const uint32_t ape1_type = q4_compound_width1
+            ? layer->indexer_compressor_ape->type
+            : layer->attn_compressor_ape->type;
+        const int fused = ds4_gpu_q4_K_pair_quad_compressor_store_tensor(
+                metal_graph_qr(g), metal_graph_kv_raw(g),
+                metal_graph_comp_kv_cur(g), metal_graph_comp_sc_cur(g),
+                out1_kv, out1_sc,
+                g->layer_attn_state_kv[il],
+                g->layer_attn_state_score[il],
+                state1_kv, state1_sc,
+                model->map, model->size,
+                layer->attn_q_a->abs_offset,
+                layer->attn_kv->abs_offset,
+                layer->attn_compressor_kv->abs_offset,
+                layer->attn_compressor_gate->abs_offset,
+                weight1_kv, weight1_sc,
+                layer->attn_compressor_ape->abs_offset,
+                layer->attn_compressor_ape->type,
+                ape1, ape1_type,
+                DS4_N_EMBD, (uint32_t)q_rank, DS4_N_HEAD_DIM,
+                q4_compound_width0, q4_compound_width1,
+                metal_graph_attn_norm(g), q4_compound_ratio, pos);
+        if (fused < 0) {
+            ok = false;
+        } else if (fused > 0) {
+            qkv_pair_projected = true;
+            qkv_pair_quad_fused = true;
+        }
+    }
+#endif
     /* M1-M5 decode fusion: the q_a/kv Q8 pair and the four F16 compressor
      * projections all read the same normalized attention input and write
      * disjoint outputs, so one dispatch covers both stages with unchanged
      * per-row reduction trees (see the kernel comment).  Restricted to the
      * resident FULL phase so split-phase / CUDA / SSD flows keep their
      * original ordering. */
-    if (!resume_after_qa_kv_raw && ok && qkv_rms_fused && compressed &&
+    if (!resume_after_qa_kv_raw && ok && !qkv_pair_quad_fused &&
+        qkv_rms_fused && compressed &&
         phase == METAL_DECODE_LAYER_FULL &&
         !g->ssd_streaming && !g->ssd_streaming_cold &&
         ds4_layer_compress_ratio(il) == 4u &&
-        layer->attn_q_a->type == DS4_TENSOR_Q8_0 &&
-        layer->attn_kv->type == DS4_TENSOR_Q8_0 &&
+        qkv_proj_q8 &&
         g->cuda_qkv_pair && !metal_graph_use_reference_qkv_pair_proj() &&
         !metal_graph_use_reference_compressor_pair_proj() &&
         layer->attn_compressor_kv && layer->attn_compressor_gate &&
@@ -23200,8 +24158,7 @@ static bool metal_graph_encode_decode_layer_phase(
         phase == METAL_DECODE_LAYER_FULL &&
         !g->ssd_streaming && !g->ssd_streaming_cold &&
         ds4_layer_compress_ratio(il) == 128u &&
-        layer->attn_q_a->type == DS4_TENSOR_Q8_0 &&
-        layer->attn_kv->type == DS4_TENSOR_Q8_0 &&
+        qkv_proj_q8 &&
         g->cuda_qkv_pair && !metal_graph_use_reference_qkv_pair_proj() &&
         !metal_graph_use_reference_compressor_pair_proj() &&
         layer->attn_compressor_kv && layer->attn_compressor_gate &&
@@ -23253,9 +24210,8 @@ static bool metal_graph_encode_decode_layer_phase(
             qkv_pair_quad_fused = true;
         }
     }
-    if (!resume_after_qa_kv_raw && ok && !qkv_pair_quad_fused && qkv_rms_fused &&
-        layer->attn_q_a->type == DS4_TENSOR_Q8_0 &&
-        layer->attn_kv->type == DS4_TENSOR_Q8_0 &&
+    if (!resume_after_qa_kv_raw && ok && !qkv_pair_quad_fused &&
+        qkv_rms_fused && qkv_proj_q8 &&
         g->cuda_qkv_pair && !metal_graph_use_reference_qkv_pair_proj()) {
         qkv_pair_projected = ds4_gpu_matmul_q8_0_pair_tensor(
                 metal_graph_qr(g),
@@ -23270,6 +24226,27 @@ static bool metal_graph_encode_decode_layer_phase(
                 metal_graph_attn_norm(g),
                 1) != 0;
     }
+    if (!resume_after_qa_kv_raw && ok && qkv_rms_fused && qkv_proj_q4 &&
+        g->cuda_qkv_pair &&
+        !qkv_pair_projected && !metal_graph_use_reference_qkv_pair_proj()) {
+        const int pair_rc = ds4_gpu_matmul_q4_K_pair_tensor(
+                metal_graph_qr(g),
+                metal_graph_kv_raw(g),
+                model->map,
+                model->size,
+                layer->attn_q_a->abs_offset,
+                layer->attn_kv->abs_offset,
+                DS4_N_EMBD,
+                q_rank,
+                DS4_N_HEAD_DIM,
+                metal_graph_attn_norm(g),
+                1);
+        if (pair_rc < 0) {
+            ok = false;
+        } else {
+            qkv_pair_projected = pair_rc > 0;
+        }
+    }
     if (!resume_after_qa_kv_raw && ok && !qkv_pair_projected) {
         ok = metal_graph_matmul_dense_quant_tensor(metal_graph_qr(g),
                                                    model,
@@ -23558,25 +24535,39 @@ static bool metal_graph_encode_decode_layer_phase(
             ok = false;
         }
         bool comp_state_already_stored = qkv_pair_quad_fused;
-        /* Quad projection: the attention and indexer compressor pairs share
-         * the normalized input and the F16 matvec shape, so a single dispatch
-         * covers all four matrices with unchanged per-row reduction trees.
-         * Removes one dispatch per decode layer.  Either preceding fused
-         * QKV path may already have performed the same work. */
+        /* Ratio-4 attention and indexer compressor pairs consume the same
+         * normalized row.  On the normal FULL path and the DSpark exact-union
+         * prefix, one Metal dispatch can project all four F16 matrices and
+         * append both recurrent states without changing either reduction
+         * tree.  The disable is authoritative; the enable also permits
+         * focused experiments in other decode phases. */
         int quad_store = comp_state_already_stored ? 1 : 0;
-        if (ok && quad_store == 0 && ratio == 4u &&
+#if defined(__APPLE__) && !defined(DS4_NO_GPU)
+        const bool quad_store_forced =
+            getenv("DS4_METAL_ENABLE_COMPRESSOR_QUAD_STORE") != NULL;
+        const bool quad_store_scope =
+            phase == METAL_DECODE_LAYER_FULL ||
+            (phase == METAL_DECODE_LAYER_TO_ROUTER &&
+             g->spec_exactn_union_collect_routes) ||
+            quad_store_forced;
+        const bool quad_store_device =
+            quad_store_forced ||
+            ds4_gpu_f16_quad_compressor_store_auto_available() != 0;
+        if (ok && quad_store == 0 && ratio == 4u && quad_store_scope &&
+            quad_store_device &&
             !metal_graph_use_reference_compressor_pair_proj() &&
+            getenv("DS4_METAL_DISABLE_COMPRESSOR_QUAD_STORE") == NULL &&
             getenv("DS4_METAL_DISABLE_PRE_M5_COMPRESSOR_QUAD_STORE") == NULL &&
-            (ds4_gpu_device_is_pre_m5_apple_silicon() ||
-             ds4_gpu_device_is_m5_apple_silicon()) &&
             layer->indexer_compressor_kv && layer->indexer_compressor_gate &&
             layer->indexer_compressor_ape &&
             layer->indexer_compressor_kv->type == DS4_TENSOR_F16 &&
             layer->indexer_compressor_gate->type == DS4_TENSOR_F16 &&
             layer->indexer_compressor_kv->dim[0] == DS4_N_EMBD &&
             layer->indexer_compressor_gate->dim[0] == DS4_N_EMBD &&
-            layer->indexer_compressor_kv->dim[1] == 2u * DS4_N_INDEXER_HEAD_DIM &&
-            layer->indexer_compressor_gate->dim[1] == 2u * DS4_N_INDEXER_HEAD_DIM) {
+            layer->indexer_compressor_kv->dim[1] ==
+                2u * DS4_N_INDEXER_HEAD_DIM &&
+            layer->indexer_compressor_gate->dim[1] ==
+                2u * DS4_N_INDEXER_HEAD_DIM) {
             quad_store =
                 ds4_gpu_matmul_f16_quad_compressor_store_tensor(
                         metal_graph_comp_kv_cur(g),
@@ -23604,12 +24595,14 @@ static bool metal_graph_encode_decode_layer_phase(
                         ratio,
                         pos);
         }
+#endif
         if (quad_store < 0) {
             ok = false;
         } else if (quad_store > 0) {
             comp_state_already_stored = true;
         }
-        if (ok && quad_store == 0 && !metal_graph_use_reference_compressor_pair_proj()) {
+        if (ok && !comp_state_already_stored &&
+            !metal_graph_use_reference_compressor_pair_proj()) {
             const int fused_store =
                 ds4_gpu_matmul_f16_pair_compressor_store_tensor(
                         metal_graph_comp_kv_cur(g),
@@ -23643,7 +24636,7 @@ static bool metal_graph_encode_decode_layer_phase(
                                                       metal_graph_attn_norm(g),
                                                       1) != 0;
             }
-        } else if (quad_store == 0) {
+        } else if (!comp_state_already_stored) {
             if (ok) ok = ds4_gpu_matmul_f16_tensor(metal_graph_comp_kv_cur(g), model->map, model->size,
                                                      layer->attn_compressor_kv->abs_offset,
                                                      DS4_N_EMBD, comp_width,
@@ -23738,11 +24731,18 @@ static bool metal_graph_encode_decode_layer_phase(
                 ok = false;
             }
             bool index_state_already_stored = quad_store > 0;
-            if (ok && quad_store == 0 && !metal_graph_use_reference_compressor_pair_proj()) {
+            ds4_gpu_tensor *index_comp_kv = quad_store > 0
+                ? metal_graph_index_comp_kv_cur(g)
+                : metal_graph_comp_kv_cur(g);
+            ds4_gpu_tensor *index_comp_sc = quad_store > 0
+                ? metal_graph_index_comp_sc_cur(g)
+                : metal_graph_comp_sc_cur(g);
+            if (ok && !index_state_already_stored &&
+                !metal_graph_use_reference_compressor_pair_proj()) {
                 const int fused_store =
                     ds4_gpu_matmul_f16_pair_compressor_store_tensor(
-                            metal_graph_comp_kv_cur(g),
-                            metal_graph_comp_sc_cur(g),
+                            index_comp_kv,
+                            index_comp_sc,
                             g->layer_index_state_kv[il],
                             g->layer_index_state_score[il],
                             model->map,
@@ -23761,8 +24761,8 @@ static bool metal_graph_encode_decode_layer_phase(
                 } else if (fused_store > 0) {
                     index_state_already_stored = true;
                 } else {
-                    ok = ds4_gpu_matmul_f16_pair_tensor(metal_graph_comp_kv_cur(g),
-                                                          metal_graph_comp_sc_cur(g),
+                    ok = ds4_gpu_matmul_f16_pair_tensor(index_comp_kv,
+                                                          index_comp_sc,
                                                           model->map,
                                                           model->size,
                                                           layer->indexer_compressor_kv->abs_offset,
@@ -23772,20 +24772,20 @@ static bool metal_graph_encode_decode_layer_phase(
                                                           metal_graph_attn_norm(g),
                                                           1) != 0;
                 }
-            } else if (quad_store == 0) {
-                if (ok) ok = ds4_gpu_matmul_f16_tensor(metal_graph_comp_kv_cur(g), model->map, model->size,
+            } else if (!index_state_already_stored) {
+                if (ok) ok = ds4_gpu_matmul_f16_tensor(index_comp_kv, model->map, model->size,
                                                          layer->indexer_compressor_kv->abs_offset,
                                                          DS4_N_EMBD, index_width,
                                                          metal_graph_attn_norm(g), 1) != 0;
-                if (ok) ok = ds4_gpu_matmul_f16_tensor(metal_graph_comp_sc_cur(g), model->map, model->size,
+                if (ok) ok = ds4_gpu_matmul_f16_tensor(index_comp_sc, model->map, model->size,
                                                          layer->indexer_compressor_gate->abs_offset,
                                                          DS4_N_EMBD, index_width,
                                                          metal_graph_attn_norm(g), 1) != 0;
             }
             DS4_METAL_PROFILE_DECODE_STAGE("indexer_compressor_proj");
             const uint32_t index_row = g->layer_n_index_comp[il];
-            if (ok) ok = ds4_gpu_compressor_update_tensor(metal_graph_comp_kv_cur(g),
-                                                            metal_graph_comp_sc_cur(g),
+            if (ok) ok = ds4_gpu_compressor_update_tensor(index_comp_kv,
+                                                            index_comp_sc,
                                                             g->layer_index_state_kv[il],
                                                             g->layer_index_state_score[il],
                                                             g->layer_index_comp_cache[il],
@@ -23881,10 +24881,10 @@ static bool metal_graph_encode_decode_layer_phase(
                 g->layer_n_index_comp[il] > DS4_N_INDEXER_TOP_K) {
                 const uint64_t indexer_q_dim = (uint64_t)DS4_N_INDEXER_HEAD * DS4_N_INDEXER_HEAD_DIM;
                 if (!layer->indexer_attn_q_b ||
-                    !tensor_type_is_f16_or_q8_0(layer->indexer_attn_q_b->type) ||
+                    !tensor_type_is_indexer_q(layer->indexer_attn_q_b->type) ||
                     layer->indexer_attn_q_b->dim[0] != q_rank ||
                     layer->indexer_attn_q_b->dim[1] != indexer_q_dim) {
-                    fprintf(stderr, "ds4: Metal graph indexer q projection expects F16 or Q8_0 weights\n");
+                    fprintf(stderr, "ds4: Metal graph indexer q projection expects F16, Q8_0, or Q4_K weights\n");
                     ok = false;
                 }
                 if (ok && (!layer->indexer_proj ||
@@ -24267,16 +25267,32 @@ static bool metal_graph_encode_decode_layer_phase(
         cuda_tp_attn_requested &&
         !metal_graph_directional_steering_attn_enabled(g) &&
         cuda_tp_partner_tier >= 0 &&
-        (n_groups % 2u) == 0u;
+        (n_groups % 2u) == 0u &&
+        layer->attn_output_a->type == DS4_TENSOR_Q8_0 &&
+        layer->attn_output_b->type == DS4_TENSOR_Q8_0;
     ds4_gpu_tensor *tp_attn_a = NULL;   /* rank partials consumed directly */
     ds4_gpu_tensor *tp_attn_b = NULL;   /* by the HC expand */
-    const bool fuse_attn_out_hc =
+    const bool fuse_attn_out_hc_q8 =
         !cuda_tp_attn &&
         g->tp_world < 2 &&
         layer->attn_output_a->type == DS4_TENSOR_Q8_0 &&
         layer->attn_output_b->type == DS4_TENSOR_Q8_0 &&
         !metal_graph_directional_steering_attn_enabled(g) &&
         !metal_graph_use_reference_attn_out_hc();
+#if !defined(DS4_NO_GPU) && !defined(DS4_ROCM_BUILD)
+    const bool fuse_attn_out_hc_q4 =
+        !cuda_tp_attn &&
+        g->tp_world < 2 &&
+        layer->attn_output_a->type == DS4_TENSOR_Q4_K &&
+        layer->attn_output_b->type == DS4_TENSOR_Q4_K &&
+        !metal_graph_directional_steering_attn_enabled(g) &&
+        !metal_graph_use_reference_attn_out_hc() &&
+        ds4_gpu_matmul_q4_K_hc_expand_available() != 0;
+#else
+    const bool fuse_attn_out_hc_q4 = false;
+#endif
+    const bool fuse_attn_out_hc =
+        fuse_attn_out_hc_q8 || fuse_attn_out_hc_q4;
     const bool fuse_tp_attn_out_hc =
         cuda_tp_attn &&
         !metal_graph_use_reference_attn_out_hc() &&
@@ -24301,10 +25317,14 @@ static bool metal_graph_encode_decode_layer_phase(
         memset(&islb_key, 0, sizeof(islb_key));
         islb_key.il = il;
         islb_key.island = 1u;
-        islb_key.cur_hc = (void *)metal_graph_cur_hc(g);
-        islb_key.after_attn_hc = (void *)metal_graph_after_attn_hc(g);
-        islb_key.after_ffn_hc = (void *)metal_graph_after_ffn_hc(g);
-        islb_key.attn_norm = (void *)metal_graph_attn_norm(g);
+        islb_key.variant = g->decode_graph_variant;
+        islb_key.cur_hc = DS4_DECODE_GRAPH_TENSOR_KEY(metal_graph_cur_hc(g));
+        islb_key.after_attn_hc =
+            DS4_DECODE_GRAPH_TENSOR_KEY(metal_graph_after_attn_hc(g));
+        islb_key.after_ffn_hc =
+            DS4_DECODE_GRAPH_TENSOR_KEY(metal_graph_after_ffn_hc(g));
+        islb_key.attn_norm =
+            DS4_DECODE_GRAPH_TENSOR_KEY(metal_graph_attn_norm(g));
         for (;;) {
             const int islb_state = ds4_gpu_decode_graph_begin(&islb_key);
             if (islb_state == 1) return ok;
@@ -24330,6 +25350,7 @@ static bool metal_graph_encode_decode_layer_phase(
                 cuda_tp_home_tier, cuda_tp_partner_tier, n_groups);
         ok = false;
     }
+#undef DS4_DECODE_GRAPH_TENSOR_KEY
     if (ok && cuda_tp_attn) {
         const uint32_t tp_groups = n_groups / 2u;
         const uint64_t tp_heads_bytes = (uint64_t)tp_groups * group_dim * sizeof(float);
@@ -24447,7 +25468,7 @@ static bool metal_graph_encode_decode_layer_phase(
                     DS4_N_HC) != 0;
             if (ok) cuda_tp_attn_hc_fused = true;
         }
-    } else if (ok && fuse_attn_out_hc) {
+    } else if (ok && fuse_attn_out_hc_q8) {
         ok = ds4_gpu_attention_output_low_q8_tensor(metal_graph_attn_low(g),
                                                       model->map,
                                                       model->size,
@@ -24470,6 +25491,29 @@ static bool metal_graph_encode_decode_layer_phase(
                                                         DS4_N_EMBD,
                                                         DS4_N_HC) != 0;
         }
+    } else if (ok && fuse_attn_out_hc_q4) {
+        ok = metal_graph_attention_output_dense_quant_low(
+                metal_graph_attn_low(g), g, model,
+                layer->attn_output_a,
+                group_dim, rank, 0, n_groups,
+                metal_graph_heads(g), true);
+#if !defined(DS4_NO_GPU) && !defined(DS4_ROCM_BUILD)
+        if (ok) {
+            ok = ds4_gpu_matmul_q4_K_hc_expand_tensor(
+                    metal_graph_after_attn_hc(g),
+                    metal_graph_attn_out(g),
+                    model->map, model->size,
+                    layer->attn_output_b->abs_offset,
+                    (uint64_t)n_groups * rank,
+                    DS4_N_EMBD,
+                    metal_graph_attn_low(g),
+                    metal_graph_cur_hc(g),
+                    metal_graph_hc_split(g),
+                    DS4_N_EMBD, DS4_N_HC) != 0;
+        }
+#else
+        ok = false;
+#endif
     } else if (ok && g->tp_world == 2) {
         /* Group-sliced attention output: this rank computes its half of the
          * output groups and the matching k-window of the expand projection,
@@ -24503,7 +25547,8 @@ static bool metal_graph_encode_decode_layer_phase(
                                                           rank,
                                                           0,
                                                           n_groups,
-                                                          metal_graph_heads(g));
+                                                          metal_graph_heads(g),
+                                                          true);
         if (ok) ok = metal_graph_matmul_dense_quant_tensor(attn_out_dst,
                                                            model,
                                                            layer->attn_output_b,
@@ -24588,13 +25633,12 @@ static bool metal_graph_encode_decode_layer_phase(
     bool ffn_hc_producer_pre_norm_fused = false;
     if (ok && !tp_ablate_hcpre) {
         const bool fuse_norm_mix =
-            hc_dim == 16384u && mix_hc == 24u &&
-            layer->hc_ffn_fn->type == DS4_TENSOR_F16 &&
-            !metal_graph_use_reference_hc_decode() &&
+            metal_graph_use_hc_norm_mix_f16(layer->hc_ffn_fn,
+                                             hc_dim,
+                                             mix_hc) &&
             getenv("DS4_METAL_DISABLE_PRE_M5_HC_NORM_MIX_FUSE") == NULL &&
             (ds4_gpu_device_is_pre_m5_apple_silicon() ||
-             ds4_gpu_device_is_m5_apple_silicon()) &&
-            ds4_gpu_hc_rms_norm_mix_f16_available() != 0;
+             ds4_gpu_device_is_m5_apple_silicon());
 #if defined(__APPLE__)
         const bool fuse_producer_pre_norm =
             fuse_norm_mix && fuse_hc_norm &&
@@ -25349,6 +26393,7 @@ static bool metal_graph_encode_decode_layer_phase(
         metal_graph_decode_cuda_selected_slots_expected(g, layer);
     const bool overlap_selected_shared =
         ok &&
+        !external_routed &&
         g->tp_world < 2 &&
         !decode_stage_profile &&
         !metal_graph_decode_cpu_router_applicable(g, layer) &&
@@ -25365,8 +26410,30 @@ static bool metal_graph_encode_decode_layer_phase(
          (mxfp4_selected_shared_overlap &&
           metal_graph_use_iq2_selected_async_load(g)) ||
          cuda_selected_shared_overlap);
+    bool cuda_selected_event_pipeline = false;
+    bool cuda_selected_event_required = false;
+#if !defined(__APPLE__) && !defined(DS4_ROCM_BUILD) && !defined(DS4_NO_GPU)
+    cuda_selected_event_required =
+        ds4_gpu_cuda_stream_selected_event_pipeline_required() != 0;
+    cuda_selected_event_pipeline =
+        ds4_gpu_cuda_stream_selected_event_pipeline_enabled() != 0 &&
+        cuda_selected_shared_overlap &&
+        async_selected_load;
+    if (cuda_selected_event_required &&
+        !cuda_selected_event_pipeline) {
+        ds4_gpu_cuda_stream_selected_event_note_candidate();
+        ds4_gpu_cuda_stream_selected_event_note_failure(1);
+        fprintf(stderr,
+                "ds4: required CUDA selected-expert event pipeline is not "
+                "eligible at layer %u\n",
+                il);
+        (void)ds4_gpu_cuda_stream_selected_event_abort();
+        return false;
+    }
+#endif
     const bool selected_readahead_shared_delay =
         ok &&
+        !external_routed &&
         g->tp_world < 2 &&
         !overlap_selected_shared &&
         !decode_stage_profile &&
@@ -25377,6 +26444,7 @@ static bool metal_graph_encode_decode_layer_phase(
         getenv("DS4_MOE_REPLAY_SELECTED_IDS") == NULL;
     const bool cuda_stream_selected_load =
         ok &&
+        !external_routed &&
         !overlap_selected_shared &&
         !selected_readahead_shared_delay &&
         g->ssd_streaming &&
@@ -25527,24 +26595,72 @@ static bool metal_graph_encode_decode_layer_phase(
     }
     if (overlap_selected_shared) {
         uint64_t selected_event = 0;
-        if (ok) ok = ds4_gpu_signal_selected_readback_ready(&selected_event) != 0;
+#if !defined(__APPLE__) && !defined(DS4_ROCM_BUILD) && !defined(DS4_NO_GPU)
+        if (ok && cuda_selected_event_pipeline) {
+            ds4_gpu_cuda_stream_selected_event_note_candidate();
+            if (ds4_gpu_signal_selected_readback_ready_async(
+                        &selected_event) == 0) {
+                ds4_gpu_cuda_stream_selected_event_note_failure(
+                    cuda_selected_event_required ? 1 : 0);
+                if (cuda_selected_event_required) {
+                    fprintf(stderr,
+                            "ds4: required CUDA selected compute-ready "
+                            "event failed at layer %u\n",
+                            il);
+                    (void)ds4_gpu_cuda_stream_selected_event_abort();
+                    return false;
+                }
+                ds4_gpu_cuda_stream_selected_event_note_fallback();
+                cuda_selected_event_pipeline = false;
+            }
+        }
+#endif
+        if (ok && !cuda_selected_event_pipeline) {
+            ok = ds4_gpu_signal_selected_readback_ready(&selected_event) != 0;
+        }
         metal_graph_selected_async_load async_load = {0};
         bool async_load_started = false;
+        const bool iq2_metal_merged_boundary =
+            iq2_selected_shared_overlap &&
+            !cuda_selected_shared_overlap;
         const bool async_early_commit =
             async_selected_load &&
-            metal_graph_use_iq2_selected_async_early_commit(g);
+            !cuda_selected_event_pipeline &&
+            (iq2_metal_merged_boundary ?
+                 metal_graph_use_iq2_selected_async_early_commit(g) :
+                 metal_graph_use_selected_async_early_commit_legacy(g));
         if (ok && async_selected_load) {
-            ok = metal_graph_selected_async_load_start(&async_load,
+            /* Failure to acquire the optional worker retains the established
+             * synchronous selected-id/load fallback below. */
+            async_load_started =
+                metal_graph_selected_async_load_start(&async_load,
                                                        g,
                                                        model,
                                                        layer,
                                                        il,
                                                        selected_event,
+                                                       cuda_selected_event_pipeline,
+                                                       cuda_selected_event_required,
                                                        gate_expert_bytes,
                                                        down_expert_bytes);
-            async_load_started = ok;
+#if !defined(__APPLE__) && !defined(DS4_ROCM_BUILD) && !defined(DS4_NO_GPU)
+            if (!async_load_started && cuda_selected_event_pipeline) {
+                ds4_gpu_cuda_stream_selected_event_note_failure(
+                    cuda_selected_event_required ? 1 : 0);
+                if (cuda_selected_event_required) {
+                    fprintf(stderr,
+                            "ds4: required CUDA selected-expert worker "
+                            "could not start at layer %u\n",
+                            il);
+                    (void)ds4_gpu_cuda_stream_selected_event_abort();
+                    return false;
+                }
+                ds4_gpu_cuda_stream_selected_event_note_fallback();
+                cuda_selected_event_pipeline = false;
+            }
+#endif
         }
-        if (ok && async_early_commit) {
+        if (ok && async_early_commit && async_load_started) {
             ok = ds4_gpu_flush_commands() != 0;
         }
         if (ok && fuse_shared_gate_up) {
@@ -25589,13 +26705,22 @@ static bool metal_graph_encode_decode_layer_phase(
         }
         DS4_METAL_PROFILE_DECODE_STAGE("shared_down");
         if (async_load_started) {
-            const bool flush_ok = ds4_gpu_flush_commands() != 0;
+            const bool flush_ok = cuda_selected_event_pipeline ||
+                ds4_gpu_flush_commands() != 0;
             bool finish_ok =
                 metal_graph_selected_async_load_finish(&async_load);
-            if (!finish_ok && async_load.ids_ok) {
+            if (!finish_ok && async_load.ids_ok &&
+                !async_load.fail_closed &&
+                !cuda_selected_event_required &&
+                async_load.upload_event_value == 0) {
                 /* The worker read valid ids but could not stage the load
                  * (it is not allowed to wait on in-flight cache entries).
                  * This thread is, so retry the same load synchronously. */
+#if !defined(__APPLE__) && !defined(DS4_ROCM_BUILD) && !defined(DS4_NO_GPU)
+                if (cuda_selected_event_pipeline) {
+                    ds4_gpu_cuda_stream_selected_event_note_fallback();
+                }
+#endif
                 const ds4_gpu_stream_expert_table retry_table =
                     graph_stream_expert_table_make(model,
                                                    layer,
@@ -25611,6 +26736,29 @@ static bool metal_graph_encode_decode_layer_phase(
                             async_load.selected_ids,
                             DS4_N_EXPERT_USED) != 0;
             }
+#if !defined(__APPLE__) && !defined(DS4_ROCM_BUILD) && !defined(DS4_NO_GPU)
+            if (cuda_selected_event_pipeline && !finish_ok) {
+                ds4_gpu_cuda_stream_selected_event_note_failure(
+                    cuda_selected_event_required ? 1 : 0);
+            }
+            if (cuda_selected_event_pipeline && finish_ok &&
+                async_load.upload_event_value != 0 &&
+                ds4_gpu_stream_expert_cache_wait_selected_upload(
+                    async_load.upload_event_value,
+                    "selected expert upload") == 0) {
+                ds4_gpu_cuda_stream_selected_event_note_failure(
+                    cuda_selected_event_required ? 1 : 0);
+                if (cuda_selected_event_required) {
+                    finish_ok = false;
+                } else {
+                    ds4_gpu_cuda_stream_selected_event_note_fallback();
+                    finish_ok = ds4_gpu_synchronize() != 0;
+                }
+            }
+            if (cuda_selected_event_pipeline && !finish_ok) {
+                (void)ds4_gpu_cuda_stream_selected_event_abort();
+            }
+#endif
             ok = ok && flush_ok && finish_ok;
         } else if (ok) {
             ok = ds4_gpu_commit_and_wait_selected_readback(selected_event,
@@ -25637,6 +26785,11 @@ static bool metal_graph_encode_decode_layer_phase(
                             DS4_N_EXPERT_USED) != 0;
             }
         }
+#if !defined(__APPLE__) && !defined(DS4_ROCM_BUILD) && !defined(DS4_NO_GPU)
+        if (!ok && cuda_selected_event_pipeline) {
+            (void)ds4_gpu_cuda_stream_selected_event_abort();
+        }
+#endif
         if (ok) ok = ds4_gpu_routed_moe_one_tensor(metal_graph_routed_out(g),
                                                      metal_graph_routed_gate(g),
                                                      metal_graph_routed_up(g),
@@ -26974,7 +28127,8 @@ static bool metal_graph_attention_output_dense_quant_low(
         uint64_t                rank,
         uint32_t                group0,
         uint32_t                group_cnt,
-        const ds4_gpu_tensor *heads) {
+        const ds4_gpu_tensor *heads,
+        bool                   resident_decode) {
     (void)g;
     if (!low || !model || !out_a || !heads ||
         group_dim == 0 || rank == 0 || group_cnt == 0) {
@@ -26990,16 +28144,23 @@ static bool metal_graph_attention_output_dense_quant_low(
                                                       group_cnt,
                                                       heads) != 0;
     }
+    /* Specialized Q4_K low projection where the backend has it.  Zero falls
+     * through to the generic per-group loop; a negative REQUIRE/error result
+     * fails closed after a backend may have submitted work. */
     if (out_a->type == DS4_TENSOR_Q4_K) {
-        return ds4_gpu_attention_output_low_q4_K_slice_tensor(low,
-                                                             model->map,
-                                                             model->size,
-                                                             out_a->abs_offset,
-                                                             group_dim,
-                                                             rank,
-                                                             group0,
-                                                             group_cnt,
-                                                             heads) != 0;
+        const int q4_slice_rc =
+            ds4_gpu_attention_output_low_q4_K_slice_tensor(low,
+                                                           model->map,
+                                                           model->size,
+                                                           out_a->abs_offset,
+                                                           group_dim,
+                                                           rank,
+                                                           group0,
+                                                           group_cnt,
+                                                           heads,
+                                                           resident_decode ? 1 : 0);
+        if (q4_slice_rc > 0) return true;
+        if (q4_slice_rc < 0) return false;
     }
     uint64_t row_bytes = 0;
     if (!metal_graph_dense_quant_row_bytes(out_a, group_dim, &row_bytes)) return false;
@@ -27071,7 +28232,8 @@ static bool metal_graph_attention_output_dense_quant_tp(
                                                      rank,
                                                      group0,
                                                      group_cnt,
-                                                     heads)) {
+                                                     heads,
+                                                     true)) {
         return false;
     }
     return metal_graph_matmul_dense_quant_kslice(out,
@@ -27118,24 +28280,25 @@ static bool metal_graph_attention_output_dense_quant_batch(
                                                         heads,
                                                         n_tokens) != 0;
     }
-    if (out_a->type == DS4_TENSOR_Q4_K && n_tokens >= 32u) {
-        if (ds4_gpu_attention_output_q4_K_batch_tensor(out,
-                                                       low,
-                                                       metal_graph_batch_group_tmp(g),
-                                                       metal_graph_batch_low_tmp(g),
-                                                       model->map,
-                                                       model->size,
-                                                       out_a->abs_offset,
-                                                       out_b->abs_offset,
-                                                       out_b->type,
-                                                       group_dim,
-                                                       rank,
-                                                       n_groups,
-                                                       out_dim,
-                                                       heads,
-                                                       n_tokens) != 0) {
-            return true;
-        }
+    if (out_a->type == DS4_TENSOR_Q4_K && n_tokens >= 2u) {
+        const int tiny_rc = ds4_gpu_attention_output_q4_K_batch_tensor(
+                out,
+                low,
+                metal_graph_batch_group_tmp(g),
+                metal_graph_batch_low_tmp(g),
+                model->map,
+                model->size,
+                out_a->abs_offset,
+                out_b->abs_offset,
+                out_b->type,
+                group_dim,
+                rank,
+                n_groups,
+                out_dim,
+                heads,
+                n_tokens);
+        if (tiny_rc > 0) return true;
+        if (tiny_rc < 0) return false;
     }
 
     const uint64_t heads_row_elems = (uint64_t)n_groups * group_dim;
@@ -27163,7 +28326,8 @@ static bool metal_graph_attention_output_dense_quant_batch(
                                                           rank,
                                                           0,
                                                           n_groups,
-                                                          heads_row);
+                                                          heads_row,
+                                                          false);
         if (ok) ok = metal_graph_matmul_dense_quant_tensor(out_row,
                                                            model,
                                                            out_b,
@@ -27937,15 +29101,21 @@ static DS4_MAYBE_UNUSED bool metal_graph_pre_m5_q2_decode_schedule_eligible(
 }
 
 static uint32_t metal_graph_token_split_after_layers(void) {
+#if defined(__APPLE__)
     uint32_t split_after_layers = 4;
-#ifndef DS4_ROCM_BUILD
+#else
+    /* Metal flushes submit the encoded prefix without waiting, allowing the GPU
+     * to start it while the CPU encodes the suffix.  CUDA and ROCm implement
+     * this API as a device-wide synchronization, so splitting there only drains
+     * the launch pipeline in the middle of every token. */
+    uint32_t split_after_layers = 0;
+#endif
     const char *split_env = getenv("DS4_METAL_GRAPH_TOKEN_SPLIT_LAYERS");
     if (split_env && split_env[0]) {
         char *end = NULL;
         unsigned long v = strtoul(split_env, &end, 10);
         if (end != split_env && v <= DS4_N_LAYER) split_after_layers = (uint32_t)v;
     }
-#endif
     return split_after_layers;
 }
 
@@ -28367,6 +29537,51 @@ static bool metal_graph_dspark_capture_decode_layer(
     return metal_graph_dspark_capture_hc(g, metal_graph_cur_hc(g), (uint32_t)slot);
 }
 
+/* A DSpark prefill capture needs the complete row matrix for stage-0 and the
+ * final row for the next decode proposal.  On Metal, mirror the last value
+ * from the weighted-sum kernel itself: the fallback below would otherwise
+ * force a compute-encoder close plus a separate blit encoder for 16 KiB.
+ * Keep the historical sequence selectable as a correctness/perf kill switch. */
+static bool metal_graph_dspark_capture_rows_and_last(
+        ds4_gpu_tensor       *batch_dst,
+        ds4_gpu_tensor       *last_dst,
+        const ds4_gpu_tensor *hc,
+        const ds4_gpu_tensor *weights,
+        uint32_t              n_tokens) {
+    if (!batch_dst || !last_dst || !hc || !weights || n_tokens == 0) {
+        return false;
+    }
+#ifdef __APPLE__
+    const bool fused_last_enabled =
+        metal_graph_tp_env_flag(
+                "DS4_METAL_ENABLE_DSPARK_CAPTURE_FUSED_LAST", false) &&
+        !metal_graph_tp_env_flag(
+                "DS4_METAL_DISABLE_DSPARK_CAPTURE_FUSED_LAST", false);
+    if (fused_last_enabled) {
+        return ds4_gpu_hc_weighted_sum_capture_last_tensor(batch_dst,
+                                                            last_dst,
+                                                            hc,
+                                                            weights,
+                                                            DS4_N_EMBD,
+                                                            DS4_N_HC) != 0;
+    }
+#endif
+    const uint64_t embd_bytes = (uint64_t)DS4_N_EMBD * sizeof(float);
+    ds4_gpu_tensor *last_src =
+        ds4_gpu_tensor_view(batch_dst,
+                            (uint64_t)(n_tokens - 1u) * embd_bytes,
+                            embd_bytes);
+    const bool ok = last_src &&
+        ds4_gpu_hc_weighted_sum_tensor(batch_dst,
+                                       hc,
+                                       weights,
+                                       DS4_N_EMBD,
+                                       DS4_N_HC) != 0 &&
+        ds4_gpu_tensor_copy(last_dst, 0, last_src, 0, embd_bytes) != 0;
+    ds4_gpu_tensor_free(last_src);
+    return ok;
+}
+
 static bool metal_graph_dspark_capture_prefill_layer(
         ds4_gpu_graph *g,
         uint32_t       il,
@@ -28386,28 +29601,17 @@ static bool metal_graph_dspark_capture_prefill_layer(
                                 ((uint64_t)slot * g->prefill_cap *
                                  DS4_N_EMBD) * sizeof(float),
                                 (uint64_t)n_tokens * embd_bytes);
-        ds4_gpu_tensor *last_src =
-            batch_dst ?
-                ds4_gpu_tensor_view(batch_dst,
-                                    (uint64_t)(n_tokens - 1u) * embd_bytes,
-                                    embd_bytes) : NULL;
         ds4_gpu_tensor *last_dst =
             ds4_gpu_tensor_view(g->dspark_target_hidden,
                                 (uint64_t)slot * embd_bytes,
                                 embd_bytes);
-        bool ok = batch_dst && last_src && last_dst &&
-                  ds4_gpu_hc_weighted_sum_tensor(batch_dst,
-                                                 metal_graph_batch_cur_hc(g),
-                                                 g->dspark_hc_mean_rows,
-                                                 DS4_N_EMBD,
-                                                 DS4_N_HC) != 0 &&
-                  ds4_gpu_tensor_copy(last_dst,
-                                      0,
-                                      last_src,
-                                      0,
-                                      embd_bytes) != 0;
+        bool ok = metal_graph_dspark_capture_rows_and_last(
+                batch_dst,
+                last_dst,
+                metal_graph_batch_cur_hc(g),
+                g->dspark_hc_mean_rows,
+                n_tokens);
         ds4_gpu_tensor_free(last_dst);
-        ds4_gpu_tensor_free(last_src);
         ds4_gpu_tensor_free(batch_dst);
         if (ok) {
             metal_graph_dspark_capture_note_slot(g, (uint32_t)slot);
@@ -28456,35 +29660,34 @@ static bool metal_graph_dspark_capture_prefill_rows(
                             (((uint64_t)(uint32_t)slot * g->prefill_cap +
                               row0) * DS4_N_EMBD) * sizeof(float),
                             (uint64_t)n_tokens * embd_bytes);
-    bool ok = batch_dst &&
-              ds4_gpu_hc_weighted_sum_tensor(batch_dst,
-                                             metal_graph_batch_cur_hc(g),
-                                             g->dspark_hc_mean_rows,
-                                             DS4_N_EMBD,
-                                             DS4_N_HC) != 0;
+    const bool reaches_last = row0 + n_tokens == chunk_len;
+    ds4_gpu_tensor *last_dst = reaches_last ?
+        ds4_gpu_tensor_view(g->dspark_target_hidden,
+                            (uint64_t)(uint32_t)slot * embd_bytes,
+                            embd_bytes) : NULL;
+    bool ok = reaches_last ?
+        metal_graph_dspark_capture_rows_and_last(
+                batch_dst,
+                last_dst,
+                metal_graph_batch_cur_hc(g),
+                g->dspark_hc_mean_rows,
+                n_tokens) :
+        batch_dst &&
+        ds4_gpu_hc_weighted_sum_tensor(batch_dst,
+                                       metal_graph_batch_cur_hc(g),
+                                       g->dspark_hc_mean_rows,
+                                       DS4_N_EMBD,
+                                       DS4_N_HC) != 0;
     if (!ok) fprintf(stderr, "ds4: pipeline capture rows FAIL il=%u row0=%u n=%u dst=%d\n",
                      il, row0, n_tokens, batch_dst != NULL);
-    if (ok && row0 + n_tokens == chunk_len) {
-        ds4_gpu_tensor *last_src =
-            ds4_gpu_tensor_view(batch_dst,
-                                (uint64_t)(n_tokens - 1u) * embd_bytes,
-                                embd_bytes);
-        ds4_gpu_tensor *last_dst =
-            ds4_gpu_tensor_view(g->dspark_target_hidden,
-                                (uint64_t)(uint32_t)slot * embd_bytes,
-                                embd_bytes);
-        ok = last_src && last_dst &&
-             ds4_gpu_tensor_copy(last_dst, 0, last_src, 0, embd_bytes) != 0;
-        ds4_gpu_tensor_free(last_dst);
-        ds4_gpu_tensor_free(last_src);
-        if (ok) {
-            metal_graph_dspark_capture_note_slot(g, (uint32_t)slot);
-            ok = metal_graph_dspark_capture_batch_note_slot(g,
-                                                            (uint32_t)slot,
-                                                            chunk_start,
-                                                            chunk_len);
-        }
+    if (ok && reaches_last) {
+        metal_graph_dspark_capture_note_slot(g, (uint32_t)slot);
+        ok = metal_graph_dspark_capture_batch_note_slot(g,
+                                                        (uint32_t)slot,
+                                                        chunk_start,
+                                                        chunk_len);
     }
+    ds4_gpu_tensor_free(last_dst);
     ds4_gpu_tensor_free(batch_dst);
     return ok;
 }
@@ -28559,28 +29762,17 @@ static bool metal_graph_dspark_capture_verified_suffix_layer(
                             (((uint64_t)(uint32_t)slot * g->prefill_cap +
                               1u) * DS4_N_EMBD) * sizeof(float),
                             (uint64_t)n_tokens * embd_bytes);
-    ds4_gpu_tensor *last_src =
-        batch_dst ?
-            ds4_gpu_tensor_view(batch_dst,
-                                (uint64_t)(n_tokens - 1u) * embd_bytes,
-                                embd_bytes) : NULL;
     ds4_gpu_tensor *last_dst =
         ds4_gpu_tensor_view(g->dspark_target_hidden,
                             (uint64_t)(uint32_t)slot * embd_bytes,
                             embd_bytes);
-    bool ok = batch_dst && last_src && last_dst &&
-              ds4_gpu_hc_weighted_sum_tensor(batch_dst,
-                                             metal_graph_batch_cur_hc(g),
-                                             g->dspark_hc_mean_rows,
-                                             DS4_N_EMBD,
-                                             DS4_N_HC) != 0 &&
-              ds4_gpu_tensor_copy(last_dst,
-                                  0,
-                                  last_src,
-                                  0,
-                                  embd_bytes) != 0;
+    bool ok = metal_graph_dspark_capture_rows_and_last(
+            batch_dst,
+            last_dst,
+            metal_graph_batch_cur_hc(g),
+            g->dspark_hc_mean_rows,
+            n_tokens);
     ds4_gpu_tensor_free(last_dst);
-    ds4_gpu_tensor_free(last_src);
     ds4_gpu_tensor_free(batch_dst);
     if (ok) {
         metal_graph_dspark_capture_note_slot(g, (uint32_t)slot);
@@ -29603,32 +30795,53 @@ static bool metal_graph_encode_layer_attention_batch(
     }
     DS4_METAL_PROFILE_ATTN_STAGE("norm");
     DS4_METAL_PROFILE_Q_STAGE("pre_q");
-    if (ok) ok = metal_graph_matmul_q8_0_named_tensor("attn_q_a",
-                                                      il,
-                                                      pos0,
-                                                      metal_graph_batch_qr(g),
-                                                      model,
-                                                      layer->attn_q_a,
-                                                      DS4_N_EMBD,
-                                                      q_rank,
-                                                      metal_graph_batch_attn_norm(g),
-                                                      n_tokens);
+    bool qkv_q4_pair_projected = false;
+    if (ok && qkv_rms_fused && g->cuda_qkv_pair &&
+        !metal_graph_use_reference_qkv_pair_proj() && n_tokens >= 2u &&
+        layer->attn_q_a->type == DS4_TENSOR_Q4_K &&
+        layer->attn_kv->type == DS4_TENSOR_Q4_K) {
+        const int pair_rc = ds4_gpu_matmul_q4_K_pair_tensor(
+            metal_graph_batch_qr(g), metal_graph_batch_kv_raw(g),
+            model->map, model->size,
+            layer->attn_q_a->abs_offset, layer->attn_kv->abs_offset,
+            DS4_N_EMBD, q_rank, DS4_N_HEAD_DIM,
+            metal_graph_batch_attn_norm(g), n_tokens);
+        if (pair_rc < 0) {
+            ok = false;
+        } else {
+            qkv_q4_pair_projected = pair_rc > 0;
+        }
+    }
+    if (ok && !qkv_q4_pair_projected) {
+        ok = metal_graph_matmul_q8_0_named_tensor("attn_q_a",
+                                                 il,
+                                                 pos0,
+                                                 metal_graph_batch_qr(g),
+                                                 model,
+                                                 layer->attn_q_a,
+                                                 DS4_N_EMBD,
+                                                 q_rank,
+                                                 metal_graph_batch_attn_norm(g),
+                                                 n_tokens);
+    }
     if (ok) {
         metal_graph_debug_dump_tensor("q_lora", metal_graph_batch_qr(g),
                                       (uint64_t)n_tokens * q_rank, il, pos0);
     }
     DS4_METAL_PROFILE_Q_STAGE("q_a");
     if (qkv_rms_fused) {
-        if (ok) ok = metal_graph_matmul_q8_0_named_tensor("attn_kv",
-                                                          il,
-                                                          pos0,
-                                                          metal_graph_batch_kv_raw(g),
-                                                          model,
-                                                          layer->attn_kv,
-                                                          DS4_N_EMBD,
-                                                          DS4_N_HEAD_DIM,
-                                                          metal_graph_batch_attn_norm(g),
-                                                          n_tokens);
+        if (ok && !qkv_q4_pair_projected) {
+            ok = metal_graph_matmul_q8_0_named_tensor("attn_kv",
+                                                     il,
+                                                     pos0,
+                                                     metal_graph_batch_kv_raw(g),
+                                                     model,
+                                                     layer->attn_kv,
+                                                     DS4_N_EMBD,
+                                                     DS4_N_HEAD_DIM,
+                                                     metal_graph_batch_attn_norm(g),
+                                                     n_tokens);
+        }
         if (ok) {
             metal_graph_debug_dump_tensor("KVraw", metal_graph_batch_kv_raw(g),
                                           (uint64_t)n_tokens * DS4_N_HEAD_DIM, il, pos0);
@@ -29688,29 +30901,38 @@ static bool metal_graph_encode_layer_attention_batch(
         ok = false;
     }
     bool q_b_f16_out = false;
-    if (ok && !q_path_debug && layer->attn_q_b->type == DS4_TENSOR_Q8_0) {
-        q_b_f16_out = ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor(tp_q ? tp_q : metal_graph_batch_q(g),
-                                                                     tp_q_half ? tp_q_half : g->batch_q_half,
-                                                                     model->map,
-                                                                     model->size,
-                                                                     layer->attn_q_b->abs_offset,
-                                                                     q_rank,
-                                                                     q_dim,
-                                                                     tp_qr_norm ? tp_qr_norm : metal_graph_batch_qr_norm(g),
-                                                                     tp_rows,
-                                                                     DS4_N_HEAD,
-                                                                     DS4_N_HEAD_DIM,
-                                                                     DS4_N_ROT,
-                                                                     pos0 + tp_row0,
-                                                                     compressed ? (uint32_t)DS4_ROPE_ORIG_CTX : 0,
-                                                                     false,
-                                                                     freq_base,
-                                                                     freq_scale,
-                                                                     ext_factor,
-                                                                     attn_factor,
-                                                                     DS4_ROPE_YARN_BETA_FAST,
-                                                                     DS4_ROPE_YARN_BETA_SLOW,
-                                                                     DS4_RMS_EPS) != 0;
+    const bool q_b_f16_weight =
+        layer->attn_q_b->type == DS4_TENSOR_Q8_0 ||
+        (layer->attn_q_b->type == DS4_TENSOR_Q4_K &&
+         tp_rows >= 32u);
+    if (ok && !q_path_debug && q_b_f16_weight) {
+        const int q_b_f16_rc =
+            ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor(
+                tp_q ? tp_q : metal_graph_batch_q(g),
+                tp_q_half ? tp_q_half : g->batch_q_half,
+                model->map,
+                model->size,
+                layer->attn_q_b->abs_offset,
+                layer->attn_q_b->type,
+                q_rank,
+                q_dim,
+                tp_qr_norm ? tp_qr_norm : metal_graph_batch_qr_norm(g),
+                tp_rows,
+                DS4_N_HEAD,
+                DS4_N_HEAD_DIM,
+                DS4_N_ROT,
+                pos0 + tp_row0,
+                compressed ? (uint32_t)DS4_ROPE_ORIG_CTX : 0,
+                false,
+                freq_base,
+                freq_scale,
+                ext_factor,
+                attn_factor,
+                DS4_ROPE_YARN_BETA_FAST,
+                DS4_ROPE_YARN_BETA_SLOW,
+                DS4_RMS_EPS);
+        if (q_b_f16_rc < 0) ok = false;
+        q_b_f16_out = q_b_f16_rc > 0;
     }
     if (q_b_f16_out) {
         DS4_METAL_PROFILE_Q_STAGE("q_b");
@@ -30099,6 +31321,7 @@ static bool metal_graph_encode_layer_attention_batch(
             const bool aligned_chunk =
                 getenv("DS4_CUDA_NO_COMPRESSOR_PREFILL_BATCH") == NULL &&
                 !g->spec_capture_prefixes &&
+                !g->spec_force_sequential_compressor &&
                 (pos0 % ratio) == 0u && (n_tokens % ratio) == 0u;
             if (aligned_chunk) {
                 const uint32_t comp_before = g->layer_n_comp[il];
@@ -30279,6 +31502,43 @@ static bool metal_graph_encode_layer_attention_batch(
         }
         DS4_METAL_PROFILE_ATTN_STAGE("compressor");
 
+        const bool topk_prefill_needed =
+            ratio == 4 && n_comp > DS4_N_INDEXER_TOP_K;
+#if !defined(DS4_NO_GPU)
+        const bool indexer_query_prune_common =
+            ratio == 4 && zero_prefix && n_tokens >= 32u &&
+            !topk_prefill_needed && !g->quality &&
+            g->placement == NULL && g->tp_world < 2u;
+#endif
+#if defined(__APPLE__) && !defined(DS4_NO_GPU)
+        /* Before the compressed cache grows past top-k, zero-prefix prefill
+         * consumes every compressed row and never reads the transient indexer
+         * query or its per-head weights.  This is also true for the Metal SSD
+         * layer-major path: the current layer is mapped while these dispatches
+         * would run, and skipping them changes no persistent cache state. */
+        const bool prune_unused_indexer_query =
+            indexer_query_prune_common &&
+            ds4_gpu_device_is_pre_m5_apple_silicon() &&
+            getenv("DS4_METAL_DISABLE_PRE_M5_BATCH_INDEXER_QUERY_PRUNE") == NULL;
+#elif defined(DS4_ROCM_BUILD)
+        /* The query projection is equally dead on resident ROCm: before the
+         * compressed cache exceeds top-k, attention consumes every compressed
+         * row and no later stage observes these transient query/weight tensors.
+         * Keep streaming out of scope because its layer-lifetime and overlap
+         * policy are intentionally independent from the resident fast path. */
+        const bool prune_unused_indexer_query =
+            indexer_query_prune_common && !g->ssd_streaming &&
+            getenv("DS4_ROCM_DISABLE_BATCH_INDEXER_QUERY_PRUNE") == NULL;
+#elif !defined(DS4_NO_GPU)
+        /* CUDA has the same resident layer lifetime as ROCm here.  Do not
+         * couple this graph-level pruning to any SSD streaming policy. */
+        const bool prune_unused_indexer_query =
+            indexer_query_prune_common && !g->ssd_streaming &&
+            getenv("DS4_CUDA_DISABLE_BATCH_INDEXER_QUERY_PRUNE") == NULL;
+#else
+        const bool prune_unused_indexer_query = false;
+#endif
+
         if (ok && ratio == 4) {
             const uint32_t index_width = coff * DS4_N_INDEXER_HEAD_DIM;
             if (!layer->indexer_compressor_kv || !layer->indexer_compressor_gate ||
@@ -30315,14 +31575,14 @@ static bool metal_graph_encode_layer_attention_batch(
                                                   (uint64_t)index_width * n_tokens,
                                                   il,
                                                   pos0);
-            if (ok) ok = metal_graph_matmul_plain_tensor(metal_graph_batch_indexer_q(g),
+            if (ok && !prune_unused_indexer_query) ok = metal_graph_matmul_plain_tensor(metal_graph_batch_indexer_q(g),
                                                           model,
                                                           layer->indexer_attn_q_b,
                                                           q_rank,
                                                           (uint64_t)DS4_N_INDEXER_HEAD * DS4_N_INDEXER_HEAD_DIM,
                                                           metal_graph_batch_qr_norm(g),
                                                           n_tokens);
-            if (ok) ok = ds4_gpu_rope_tail_tensor(metal_graph_batch_indexer_q(g),
+            if (ok && !prune_unused_indexer_query) ok = ds4_gpu_rope_tail_tensor(metal_graph_batch_indexer_q(g),
                                                     n_tokens,
                                                     DS4_N_INDEXER_HEAD,
                                                     DS4_N_INDEXER_HEAD_DIM,
@@ -30336,10 +31596,10 @@ static bool metal_graph_encode_layer_attention_batch(
                                                     attn_factor,
                                                     DS4_ROPE_YARN_BETA_FAST,
                                                     DS4_ROPE_YARN_BETA_SLOW) != 0;
-            if (ok) ok = ds4_gpu_dsv4_indexer_qat_tensor(metal_graph_batch_indexer_q(g),
+            if (ok && !prune_unused_indexer_query) ok = ds4_gpu_dsv4_indexer_qat_tensor(metal_graph_batch_indexer_q(g),
                                                           n_tokens * DS4_N_INDEXER_HEAD,
                                                           DS4_N_INDEXER_HEAD_DIM) != 0;
-            if (ok) ok = ds4_gpu_matmul_f16_tensor(metal_graph_batch_indexer_weights(g),
+            if (ok && !prune_unused_indexer_query) ok = ds4_gpu_matmul_f16_tensor(metal_graph_batch_indexer_weights(g),
                                                      model->map,
                                                      model->size,
                                                      layer->indexer_proj->abs_offset,
@@ -30424,6 +31684,7 @@ static bool metal_graph_encode_layer_attention_batch(
                 const bool aligned_chunk =
                     getenv("DS4_CUDA_NO_COMPRESSOR_PREFILL_BATCH") == NULL &&
                     !g->spec_capture_prefixes &&
+                    !g->spec_force_sequential_compressor &&
                     (pos0 % ratio) == 0u && (n_tokens % ratio) == 0u;
                 if (aligned_chunk) {
                     const uint32_t index_before = g->layer_n_index_comp[il];
@@ -30779,7 +32040,6 @@ static bool metal_graph_encode_layer_attention_batch(
             if (ok) batch_attention_done = true;
         }
 
-        const bool topk_prefill_needed = ratio == 4 && n_comp > DS4_N_INDEXER_TOP_K;
         if (ok && !batch_attention_done && zero_prefix &&
             topk_prefill_needed && n_comp != 0) {
             const float index_scale = 1.0f / sqrtf((float)(DS4_N_INDEXER_HEAD_DIM * DS4_N_INDEXER_HEAD));
@@ -31114,8 +32374,61 @@ static bool metal_graph_encode_layer_attention_batch(
     const bool attn_out_debug =
         metal_graph_debug_wants("attn_low", il, pos0) ||
         metal_graph_debug_wants("attn_out", il, pos0);
+    bool attn_out_hc_fused = false;
+#ifdef __APPLE__
+    if (ok && !attn_out_debug && !tp_row_split_attn &&
+        !metal_graph_directional_steering_attn_enabled(g)) {
+        int fused_rc = 0;
+        if (layer->attn_output_a->type == DS4_TENSOR_Q8_0 &&
+            layer->attn_output_b->type == DS4_TENSOR_Q8_0) {
+            fused_rc = ds4_gpu_attention_output_q8_batch_hc_tensor(
+                metal_graph_batch_attn_out(g),
+                after_attn_hc_view,
+                metal_graph_batch_cur_hc(g),
+                hc_split_view,
+                metal_graph_batch_attn_low(g),
+                metal_graph_batch_group_tmp(g),
+                metal_graph_batch_low_tmp(g),
+                model->map,
+                model->size,
+                layer->attn_output_a->abs_offset,
+                layer->attn_output_b->abs_offset,
+                group_dim,
+                rank,
+                n_groups,
+                DS4_N_EMBD,
+                metal_graph_batch_heads(g),
+                n_tokens,
+                DS4_N_HC);
+        } else if (layer->attn_output_a->type == DS4_TENSOR_Q4_K &&
+                   layer->attn_output_b->type == DS4_TENSOR_Q4_K) {
+            fused_rc = ds4_gpu_attention_output_q4_K_batch_hc_tensor(
+                metal_graph_batch_attn_out(g),
+                after_attn_hc_view,
+                metal_graph_batch_cur_hc(g),
+                hc_split_view,
+                metal_graph_batch_attn_low(g),
+                metal_graph_batch_group_tmp(g),
+                metal_graph_batch_low_tmp(g),
+                model->map,
+                model->size,
+                layer->attn_output_a->abs_offset,
+                layer->attn_output_b->abs_offset,
+                layer->attn_output_b->type,
+                group_dim,
+                rank,
+                n_groups,
+                DS4_N_EMBD,
+                metal_graph_batch_heads(g),
+                n_tokens,
+                DS4_N_HC);
+        }
+        if (fused_rc < 0) ok = false;
+        attn_out_hc_fused = fused_rc > 0;
+    }
+#endif
     bool attn_out_f16 = false;
-    if (ok &&
+    if (ok && !attn_out_hc_fused &&
         !attn_out_debug &&
         !tp_row_split_attn &&
         layer->attn_output_a->type == DS4_TENSOR_Q8_0 &&
@@ -31134,7 +32447,7 @@ static bool metal_graph_encode_layer_attention_batch(
                                                                     metal_graph_batch_heads(g),
                                                                     n_tokens) != 0;
     }
-    if (!attn_out_f16) {
+    if (!attn_out_f16 && !attn_out_hc_fused) {
         if (ok) {
             ok = metal_graph_attention_output_dense_quant_batch(tp_attn_out ? tp_attn_out : metal_graph_batch_attn_out(g),
                                                                 metal_graph_batch_attn_low(g),
@@ -31178,10 +32491,13 @@ static bool metal_graph_encode_layer_attention_batch(
         ds4_gpu_tensor_free(recv_half);
         if (!ok) fprintf(stderr, "ds4: TP prefill attention row gate failed (layer %u)\n", il);
     }
-    if (ok && !attn_out_f16 && metal_graph_directional_steering_attn_enabled(g)) {
+    if (ok && !attn_out_f16 && !attn_out_hc_fused &&
+        metal_graph_directional_steering_attn_enabled(g)) {
         ok = metal_graph_apply_directional_steering_attn(g, metal_graph_batch_attn_out(g), il, n_tokens);
     }
-    if (ok && attn_out_f16) {
+    if (ok && attn_out_hc_fused) {
+        /* The fused output-B epilogue already wrote after_attn_hc_view. */
+    } else if (ok && attn_out_f16) {
         ok = ds4_gpu_hc_expand_split_half_tensor(after_attn_hc_view,
                                                  g->batch_q_half,
                                                  metal_graph_batch_cur_hc(g),
@@ -31703,7 +33019,7 @@ static bool metal_graph_encode_layer_ffn_batch(
                                                x_row,
                                                NULL,
                                                il,
-                                               false) != 0;
+                                               !g->ssd_streaming) != 0;
             ds4_gpu_tensor_free(w_row);
             ds4_gpu_tensor_free(sel_row);
             ds4_gpu_tensor_free(x_row);
@@ -31745,7 +33061,7 @@ static bool metal_graph_encode_layer_ffn_batch(
                                                il,
                                                n_tokens,
                                                &g->batch_routed_mid_is_f16,
-                                               false) != 0;
+                                               !g->ssd_streaming) != 0;
     }
     if (ok) {
         metal_graph_debug_dump_tensor("ffn_moe_gate_clamped", metal_graph_batch_routed_gate(g),
@@ -31927,7 +33243,8 @@ static bool metal_graph_eval_token_raw_swa_streaming(
         const ds4_weights     *weights,
         int                    token,
         uint32_t               pos,
-        float                 *logits) {
+        float                 *logits,
+        int                   *top_id) {
     if (g->raw_cap == 0) {
         fprintf(stderr, "ds4: Metal graph raw KV cache is not allocated\n");
         return false;
@@ -31937,6 +33254,13 @@ static bool metal_graph_eval_token_raw_swa_streaming(
         glm_graph_env_present("DS4_ROCM_GRAPH_TOKEN_PROFILE",
                               "DS4_METAL_GRAPH_TOKEN_PROFILE");
     const bool throttle = graph_power_throttle_enabled(g);
+    const bool need_output = logits != NULL || top_id != NULL;
+    if (need_output && !weights_have_output_head(weights)) {
+        fprintf(stderr,
+                "ds4: SSD streaming decode requested logits/top-1 without "
+                "a complete output head\n");
+        return false;
+    }
     const double t0 = (profile || throttle) ? now_sec() : 0.0;
     const uint32_t raw_row = pos % g->raw_cap;
     const uint32_t n_raw = metal_graph_raw_span_for_batch(g, pos, 1);
@@ -31992,25 +33316,38 @@ static bool metal_graph_eval_token_raw_swa_streaming(
                 ok = metal_graph_dspark_capture_decode_layer(g, il);
             }
         }
-        if (ok && logits) {
+        if (ok && need_output) {
             ok = metal_graph_encode_output_head(g, model, weights, weights->output->dim[1]);
         }
+        if (ok && top_id) {
+            ok = ds4_gpu_argmax_tensor(metal_graph_comp_selected(g),
+                                       metal_graph_logits(g),
+                                       DS4_N_VOCAB) != 0;
+        }
         const double t_encoded = (profile || throttle) ? now_sec() : 0.0;
         if (ok) ok = ds4_gpu_end_commands() != 0;
         const double t_done = (profile || throttle) ? now_sec() : 0.0;
+        if (ok && top_id) {
+            int32_t device_top = -1;
+            ok = ds4_gpu_tensor_read(metal_graph_comp_selected(g), 0,
+                                     &device_top, sizeof(device_top)) != 0 &&
+                 device_top >= 0 && (uint32_t)device_top < DS4_N_VOCAB;
+            if (ok) *top_id = (int)device_top;
+        }
         if (ok && logits) {
             ok = ds4_gpu_tensor_read(metal_graph_logits(g), 0, logits, (uint64_t)DS4_N_VOCAB * sizeof(float)) != 0;
         }
         const double t_read = (profile || throttle) ? now_sec() : 0.0;
         if (profile) {
             fprintf(stderr,
-                    "ds4: metal SSD streaming batched token pos=%u encode=%.3f ms execute=%.3f ms read=%.3f ms total=%.3f ms logits=%d\n",
+                    "ds4: metal SSD streaming batched token pos=%u encode=%.3f ms execute=%.3f ms read=%.3f ms total=%.3f ms logits=%d top=%d\n",
                     pos,
                     (t_encoded - t0) * 1000.0,
                     (t_done - t_encoded) * 1000.0,
                     (t_read - t_done) * 1000.0,
                     (t_read - t0) * 1000.0,
-                    logits != NULL);
+                    logits != NULL,
+                    top_id != NULL);
         }
         if (ok && throttle) {
             graph_power_note_decode_token(g, t_read - t0);
@@ -32034,7 +33371,7 @@ static bool metal_graph_eval_token_raw_swa_streaming(
         }
         if (!static_decode_map && il + 1 < DS4_N_LAYER) {
             metal_graph_stream_readahead_layer_decode(model, weights, il + 1);
-        } else if (!static_decode_map && logits) {
+        } else if (!static_decode_map && need_output) {
             metal_graph_stream_readahead_output(model, weights);
         }
         if (ok) ok = ds4_gpu_begin_commands() != 0;
@@ -32067,31 +33404,44 @@ static bool metal_graph_eval_token_raw_swa_streaming(
         }
     }
 
-    if (ok && logits && !static_decode_map) ok = metal_graph_stream_map_output(model, weights);
+    if (ok && need_output && !static_decode_map) ok = metal_graph_stream_map_output(model, weights);
     const double t_head0 = profile ? now_sec() : 0.0;
-    if (ok && logits) ok = ds4_gpu_begin_commands() != 0;
-    if (ok && logits) ok = metal_graph_encode_output_head(g, model, weights, weights->output->dim[1]);
+    if (ok && need_output) ok = ds4_gpu_begin_commands() != 0;
+    if (ok && need_output) ok = metal_graph_encode_output_head(g, model, weights, weights->output->dim[1]);
+    if (ok && top_id) {
+        ok = ds4_gpu_argmax_tensor(metal_graph_comp_selected(g),
+                                   metal_graph_logits(g),
+                                   DS4_N_VOCAB) != 0;
+    }
     const double t_head_encoded = profile ? now_sec() : 0.0;
-    if (ok && logits) ok = ds4_gpu_end_commands() != 0;
+    if (ok && need_output) ok = ds4_gpu_end_commands() != 0;
     const double t_done = (profile || throttle) ? now_sec() : 0.0;
+    if (ok && top_id) {
+        int32_t device_top = -1;
+        ok = ds4_gpu_tensor_read(metal_graph_comp_selected(g), 0,
+                                 &device_top, sizeof(device_top)) != 0 &&
+             device_top >= 0 && (uint32_t)device_top < DS4_N_VOCAB;
+        if (ok) *top_id = (int)device_top;
+    }
     if (ok && logits) {
         ok = ds4_gpu_tensor_read(metal_graph_logits(g), 0, logits, (uint64_t)DS4_N_VOCAB * sizeof(float)) != 0;
     }
     const double t_read = (profile || throttle) ? now_sec() : 0.0;
 
     if (profile) {
-        if (logits) {
+        if (need_output) {
             encode_s += t_head_encoded - t_head0;
             execute_s += t_done - t_head_encoded;
         }
         fprintf(stderr,
-                "ds4: metal SSD streaming token pos=%u encode=%.3f ms execute=%.3f ms read=%.3f ms total=%.3f ms logits=%d\n",
+                "ds4: metal SSD streaming token pos=%u encode=%.3f ms execute=%.3f ms read=%.3f ms total=%.3f ms logits=%d top=%d\n",
                 pos,
                 encode_s * 1000.0,
                 execute_s * 1000.0,
                 (t_read - t_done) * 1000.0,
                 (t_read - t0) * 1000.0,
-                logits != NULL);
+                logits != NULL,
+                top_id != NULL);
     }
     if (ok) graph_power_note_decode_token(g, t_read - t0);
     if (!ok) {
@@ -32111,7 +33461,8 @@ static bool metal_graph_eval_token_raw_swa(
         uint32_t               pos,
         float                 *logits) {
     if (g && g->ssd_streaming) {
-        return metal_graph_eval_token_raw_swa_streaming(g, model, weights, token, pos, logits);
+        return metal_graph_eval_token_raw_swa_streaming(
+                g, model, weights, token, pos, logits, NULL);
     }
 
     const bool profile =
@@ -32576,34 +33927,50 @@ typedef struct {
  * Keeping intermediate rows device-resident avoids turning verification into a
  * sequence of large CPU readbacks. */
 static bool metal_graph_eval_token_raw_swa_top(
-        ds4_gpu_graph *g,
-        const ds4_model       *model,
-        const ds4_weights     *weights,
-        int                    token,
-        uint32_t               pos,
-        int                   *top_id,
-        float                 *logits,
-        bool                   allow_split_top1,
+        ds4_gpu_graph           *g,
+        const ds4_model         *model,
+        const ds4_weights       *weights,
+        int                      token,
+        uint32_t                 pos,
+        int                     *top_id,
+        float                   *logits,
+        bool                     allow_split_top1,
         metal_graph_top2_result *top2,
-        bool                   force_fast_attention) {
+        bool                     force_fast_attention) {
     if (!top_id) return false;
     if (top2) memset(top2, 0, sizeof(*top2));
+    /* SSD-backed decode must use the mapper on every backend.  CUDA's
+     * optional approximate/split helpers are resident-only; run the exact
+     * streaming path instead.  TP output owns only a vocabulary slice, so a
+     * local full-row argmax would be stale and is rejected until it has an
+     * explicit cross-rank merge. */
+    if (g && g->ssd_streaming) {
+        if (g->tp_world >= 2u) {
+            fprintf(stderr,
+                    "ds4: SSD streaming top-1 is unsupported with tensor "
+                    "parallel output\n");
+            return false;
+        }
+        return metal_graph_eval_token_raw_swa_streaming(
+                g, model, weights, token, pos, logits, top_id);
+    }
 
     const bool fast_attention =
         allow_split_top1 &&
         logits == NULL &&
         (force_fast_attention || metal_graph_cuda_greedy_splitkv_requested());
-    if (top2) top2->fast_attention = fast_attention;
-    const int old_fast_attention =
-        ds4_gpu_set_decode_fast_attention(fast_attention ? 1 : 0);
-    const bool profile = getenv("DS4_METAL_GRAPH_TOKEN_PROFILE") != NULL;
-    const double t0 = profile ? now_sec() : 0.0;
     const bool split_top1 =
         allow_split_top1 &&
         logits == NULL &&
         top2 == NULL &&
+        g &&
         g->cuda_tp_output &&
         metal_graph_cuda_greedy_split_top1_requested();
+    if (top2) top2->fast_attention = fast_attention;
+    const int old_fast_attention =
+        ds4_gpu_set_decode_fast_attention(fast_attention ? 1 : 0);
+    const bool profile = getenv("DS4_METAL_GRAPH_TOKEN_PROFILE") != NULL;
+    const double t0 = profile ? now_sec() : 0.0;
     if (split_top1) {
         int output_tiers[DS4_MAX_GPUS] = {0};
         uint32_t output_ways = 0;
@@ -32690,7 +34057,7 @@ static bool metal_graph_eval_token_raw_swa_top(
                                  0,
                                  values,
                                  sizeof(values)) != 0;
-        if (ok && ids[0] <= (uint32_t)INT32_MAX && ids[1] <= (uint32_t)INT32_MAX) {
+        if (ok && ids[0] < DS4_N_VOCAB && ids[1] < DS4_N_VOCAB) {
             top2->id0 = (int)ids[0];
             top2->id1 = (int)ids[1];
             top2->value0 = values[0];
@@ -32701,7 +34068,11 @@ static bool metal_graph_eval_token_raw_swa_top(
             ok = false;
         }
     } else if (ok) {
-        ok = ds4_gpu_tensor_read(metal_graph_comp_selected(g), 0, top_id, sizeof(*top_id)) != 0;
+        int32_t device_top = -1;
+        ok = ds4_gpu_tensor_read(metal_graph_comp_selected(g), 0,
+                                 &device_top, sizeof(device_top)) != 0 &&
+             device_top >= 0 && (uint32_t)device_top < DS4_N_VOCAB;
+        if (ok) *top_id = (int)device_top;
     }
     if (ok && logits) {
         ok = ds4_gpu_tensor_read(metal_graph_logits(g), 0, logits, (uint64_t)DS4_N_VOCAB * sizeof(float)) != 0;
@@ -32894,7 +34265,7 @@ static bool dspark_draft_block_ready(
         !g->dspark_draft_tokens || !g->dspark_draft_hc ||
         dw->block_size == 0 ||
         dw->block_size > DS4_DSPARK_MAX_BLOCK_SIZE ||
-        g->dspark_block_size != dw->block_size ||
+        g->dspark_block_size < dw->block_size ||
         !dw->has_noise_token_id) {
         return false;
     }
@@ -32910,10 +34281,9 @@ static bool dspark_stage_input_ready(
     if (!g || !dw ||
         dw->block_size == 0 ||
         dw->block_size > DS4_DSPARK_MAX_BLOCK_SIZE ||
-        g->dspark_block_size != dw->block_size ||
+        g->dspark_block_size < dw->block_size ||
         !g->dspark_main_x || !g->dspark_draft_hc ||
-        !g->dspark_target_hc || !g->dspark_stage_input_hc ||
-        !g->dspark_position_ids) {
+        !g->dspark_target_hc || !g->dspark_stage_input_hc) {
         return false;
     }
     if (dw->block_size == UINT32_MAX) return false;
@@ -32922,9 +34292,7 @@ static bool dspark_stage_input_ready(
     return ds4_gpu_tensor_bytes(g->dspark_target_hc) >=
                hc_dim * sizeof(float) &&
            ds4_gpu_tensor_bytes(g->dspark_stage_input_hc) >=
-               rows * hc_dim * sizeof(float) &&
-           ds4_gpu_tensor_bytes(g->dspark_position_ids) >=
-               rows * sizeof(int32_t);
+               rows * hc_dim * sizeof(float);
 }
 
 static bool dspark_stage_cache_ready(
@@ -33011,6 +34379,80 @@ static bool metal_graph_probe_dspark_noncausal_attention(
     return ok;
 }
 
+/* DSpark setup embeds one real token followed by repeated noise tokens.  The
+ * generic batched embedding binds the complete vocabulary table even though
+ * setup reads only two rows.  On Metal, encode the two exact rows once and
+ * duplicate the noise HC row on-device; non-Apple backends retain their
+ * established batched embedding path. */
+static bool metal_graph_embed_dspark_draft_rows(
+        ds4_gpu_graph      *g,
+        const ds4_model    *base_model,
+        const ds4_weights  *base_weights,
+        const ds4_dspark_weights *dw,
+        int                 token) {
+    if (!base_model ||
+        !dspark_draft_block_ready(g, base_weights, dw, token)) {
+        return false;
+    }
+
+#if !defined(__APPLE__)
+    return ds4_gpu_embed_tokens_hc_tensor(
+            g->dspark_draft_hc,
+            g->dspark_draft_tokens,
+            base_model->map,
+            base_model->size,
+            base_weights->token_embd->abs_offset,
+            (uint32_t)base_weights->token_embd->dim[1],
+            dw->block_size,
+            DS4_N_EMBD,
+            DS4_N_HC) != 0;
+#else
+    const uint64_t hc_bytes =
+        (uint64_t)DS4_N_HC * DS4_N_EMBD * sizeof(float);
+    ds4_gpu_tensor *token_row =
+        ds4_gpu_tensor_view(g->dspark_draft_hc, 0, hc_bytes);
+    ds4_gpu_tensor *noise_row = dw->block_size > 1u
+        ? ds4_gpu_tensor_view(g->dspark_draft_hc, hc_bytes, hc_bytes)
+        : NULL;
+    bool ok = token_row != NULL &&
+              (dw->block_size == 1u || noise_row != NULL);
+    const uint32_t n_vocab =
+        (uint32_t)base_weights->token_embd->dim[1];
+    if (ok) {
+        ok = ds4_gpu_embed_token_hc_tensor(
+                token_row,
+                base_model->map,
+                base_model->size,
+                base_weights->token_embd->abs_offset,
+                n_vocab,
+                (uint32_t)token,
+                DS4_N_EMBD,
+                DS4_N_HC) != 0;
+    }
+    if (ok && noise_row) {
+        ok = ds4_gpu_embed_token_hc_tensor(
+                noise_row,
+                base_model->map,
+                base_model->size,
+                base_weights->token_embd->abs_offset,
+                n_vocab,
+                dw->noise_token_id,
+                DS4_N_EMBD,
+                DS4_N_HC) != 0;
+    }
+    for (uint32_t i = 2; ok && i < dw->block_size; i++) {
+        ok = ds4_gpu_tensor_copy(g->dspark_draft_hc,
+                                 (uint64_t)i * hc_bytes,
+                                 noise_row,
+                                 0,
+                                 hc_bytes) != 0;
+    }
+    ds4_gpu_tensor_free(noise_row);
+    ds4_gpu_tensor_free(token_row);
+    return ok;
+#endif
+}
+
 static bool metal_graph_prepare_dspark_setup_block(
         ds4_gpu_graph            *g,
         const ds4_model          *base_model,
@@ -33031,11 +34473,6 @@ static bool metal_graph_prepare_dspark_setup_block(
 
     const uint64_t hc_dim = (uint64_t)DS4_N_HC * DS4_N_EMBD;
     const uint64_t hc_bytes = hc_dim * sizeof(float);
-    int32_t positions[DS4_DSPARK_MAX_BLOCK_SIZE + 1u];
-    positions[0] = (int32_t)pos;
-    for (uint32_t i = 0; i < dw->block_size; i++) {
-        positions[i + 1u] = (int32_t)(pos + i);
-    }
     int32_t ids[DS4_DSPARK_MAX_BLOCK_SIZE];
     ids[0] = (int32_t)token;
     for (uint32_t i = 1; i < dw->block_size; i++) {
@@ -33045,129 +34482,102 @@ static bool metal_graph_prepare_dspark_setup_block(
     bool ok = ds4_gpu_tensor_write(g->dspark_draft_tokens,
                                    0,
                                    ids,
-                                   (uint64_t)dw->block_size * sizeof(ids[0])) != 0 &&
-              ds4_gpu_tensor_write(g->dspark_position_ids,
+                                   (uint64_t)dw->block_size * sizeof(ids[0])) != 0;
+    if (ok) ok = ds4_gpu_begin_commands() != 0;
+    if (ok) ok = metal_graph_embed_dspark_draft_rows(g,
+                                                      base_model,
+                                                      base_weights,
+                                                      dw,
+                                                      token);
+    if (ok) {
+        ok = ds4_gpu_repeat_hc_tensor(g->dspark_target_hc,
+                                      g->dspark_main_x,
+                                      DS4_N_EMBD,
+                                      DS4_N_HC) != 0;
+    }
+    if (ok) {
+        ok = ds4_gpu_tensor_copy(g->dspark_stage_input_hc,
+                                 0,
+                                 g->dspark_target_hc,
+                                 0,
+                                 hc_bytes) != 0;
+    }
+    if (ok) {
+        ok = ds4_gpu_tensor_copy(g->dspark_stage_input_hc,
+                                 hc_bytes,
+                                 g->dspark_draft_hc,
+                                 0,
+                                 (uint64_t)dw->block_size * hc_bytes) != 0;
+    }
+    if (ok) ok = ds4_gpu_end_commands() != 0;
+    if (!ok) (void)ds4_gpu_synchronize();
+    return ok;
+}
+
+static bool metal_graph_prepare_dspark_stage0_setup_block(
+        ds4_gpu_graph            *g,
+        const ds4_model          *base_model,
+        const ds4_weights        *base_weights,
+        const ds4_model          *dspark_model,
+        const ds4_dspark_weights *dw,
+        int                       token,
+        uint32_t                  pos) {
+    if (!g || !base_model || !dspark_model ||
+        !dspark_stage0_weights_ready(g, dw) ||
+        !dspark_draft_block_ready(g, base_weights, dw, token) ||
+        !dspark_stage_input_ready(g, dw)) {
+        return false;
+    }
+    if (pos > (uint32_t)INT32_MAX ||
+        dw->block_size > (uint32_t)INT32_MAX ||
+        pos > (uint32_t)INT32_MAX - dw->block_size) {
+        return false;
+    }
+
+    const ds4_dspark_stage_weights *stage0 = &dw->stage[0];
+    const uint64_t in_dim = (uint64_t)dw->target_layer_count * DS4_N_EMBD;
+    const uint64_t hc_dim = (uint64_t)DS4_N_HC * DS4_N_EMBD;
+    const uint64_t hc_bytes = hc_dim * sizeof(float);
+    int32_t ids[DS4_DSPARK_MAX_BLOCK_SIZE];
+    ids[0] = (int32_t)token;
+    for (uint32_t i = 1; i < dw->block_size; i++) {
+        ids[i] = (int32_t)dw->noise_token_id;
+    }
+
+    /* DS4_DSPARK_PROP_PROFILE=1: break the setup block into phases to
+     * localize the TP-only prop_setup inflation (26ms vs 1.3ms single). */
+    const bool prop_profile = getenv("DS4_DSPARK_PROP_PROFILE") != NULL;
+    const double pp_t0 = prop_profile ? now_sec() : 0.0;
+    bool ok = ds4_gpu_tensor_write(g->dspark_draft_tokens,
                                    0,
-                                   positions,
-                                   ((uint64_t)dw->block_size + 1u) *
-                                       sizeof(positions[0])) != 0;
+                                   ids,
+                                   (uint64_t)dw->block_size * sizeof(ids[0])) != 0;
+    const double pp_t1 = prop_profile ? now_sec() : 0.0;
     if (ok) ok = ds4_gpu_begin_commands() != 0;
+    const double pp_t2 = prop_profile ? now_sec() : 0.0;
+    if (ok) {
+        ok = metal_graph_matmul_plain_tensor(g->dspark_stage0_proj,
+                                             dspark_model,
+                                             stage0->main_proj,
+                                             in_dim,
+                                             DS4_N_EMBD,
+                                             g->dspark_target_hidden,
+                                             1);
+    }
     if (ok) {
-        ok = ds4_gpu_embed_tokens_hc_tensor(g->dspark_draft_hc,
-                                            g->dspark_draft_tokens,
-                                            base_model->map,
-                                            base_model->size,
-                                            base_weights->token_embd->abs_offset,
-                                            (uint32_t)base_weights->token_embd->dim[1],
-                                            dw->block_size,
+        ok = ds4_gpu_rms_norm_weight_tensor(g->dspark_main_x,
+                                            g->dspark_stage0_proj,
+                                            dspark_model->map,
+                                            dspark_model->size,
+                                            stage0->main_norm->abs_offset,
                                             DS4_N_EMBD,
-                                            DS4_N_HC) != 0;
-    }
-    if (ok) {
-        ok = ds4_gpu_repeat_hc_tensor(g->dspark_target_hc,
-                                      g->dspark_main_x,
-                                      DS4_N_EMBD,
-                                      DS4_N_HC) != 0;
-    }
-    if (ok) {
-        ok = ds4_gpu_tensor_copy(g->dspark_stage_input_hc,
-                                 0,
-                                 g->dspark_target_hc,
-                                 0,
-                                 hc_bytes) != 0;
-    }
-    if (ok) {
-        ok = ds4_gpu_tensor_copy(g->dspark_stage_input_hc,
-                                 hc_bytes,
-                                 g->dspark_draft_hc,
-                                 0,
-                                 (uint64_t)dw->block_size * hc_bytes) != 0;
-    }
-    if (ok) ok = ds4_gpu_end_commands() != 0;
-    if (!ok) (void)ds4_gpu_synchronize();
-    return ok;
-}
-
-static bool metal_graph_prepare_dspark_stage0_setup_block(
-        ds4_gpu_graph            *g,
-        const ds4_model          *base_model,
-        const ds4_weights        *base_weights,
-        const ds4_model          *dspark_model,
-        const ds4_dspark_weights *dw,
-        int                       token,
-        uint32_t                  pos) {
-    if (!g || !base_model || !dspark_model ||
-        !dspark_stage0_weights_ready(g, dw) ||
-        !dspark_draft_block_ready(g, base_weights, dw, token) ||
-        !dspark_stage_input_ready(g, dw)) {
-        return false;
-    }
-    if (pos > (uint32_t)INT32_MAX ||
-        dw->block_size > (uint32_t)INT32_MAX ||
-        pos > (uint32_t)INT32_MAX - dw->block_size) {
-        return false;
-    }
-
-    const ds4_dspark_stage_weights *stage0 = &dw->stage[0];
-    const uint64_t in_dim = (uint64_t)dw->target_layer_count * DS4_N_EMBD;
-    const uint64_t hc_dim = (uint64_t)DS4_N_HC * DS4_N_EMBD;
-    const uint64_t hc_bytes = hc_dim * sizeof(float);
-    int32_t positions[DS4_DSPARK_MAX_BLOCK_SIZE + 1u];
-    positions[0] = (int32_t)pos;
-    for (uint32_t i = 0; i < dw->block_size; i++) {
-        positions[i + 1u] = (int32_t)(pos + i);
-    }
-    int32_t ids[DS4_DSPARK_MAX_BLOCK_SIZE];
-    ids[0] = (int32_t)token;
-    for (uint32_t i = 1; i < dw->block_size; i++) {
-        ids[i] = (int32_t)dw->noise_token_id;
-    }
-
-    /* DS4_DSPARK_PROP_PROFILE=1: break the setup block into phases to
-     * localize the TP-only prop_setup inflation (26ms vs 1.3ms single). */
-    const bool prop_profile = getenv("DS4_DSPARK_PROP_PROFILE") != NULL;
-    const double pp_t0 = prop_profile ? now_sec() : 0.0;
-    bool ok = ds4_gpu_tensor_write(g->dspark_draft_tokens,
-                                   0,
-                                   ids,
-                                   (uint64_t)dw->block_size * sizeof(ids[0])) != 0 &&
-              ds4_gpu_tensor_write(g->dspark_position_ids,
-                                   0,
-                                   positions,
-                                   ((uint64_t)dw->block_size + 1u) *
-                                       sizeof(positions[0])) != 0;
-    const double pp_t1 = prop_profile ? now_sec() : 0.0;
-    if (ok) ok = ds4_gpu_begin_commands() != 0;
-    const double pp_t2 = prop_profile ? now_sec() : 0.0;
-    if (ok) {
-        ok = metal_graph_matmul_plain_tensor(g->dspark_stage0_proj,
-                                             dspark_model,
-                                             stage0->main_proj,
-                                             in_dim,
-                                             DS4_N_EMBD,
-                                             g->dspark_target_hidden,
-                                             1);
-    }
-    if (ok) {
-        ok = ds4_gpu_rms_norm_weight_tensor(g->dspark_main_x,
-                                            g->dspark_stage0_proj,
-                                            dspark_model->map,
-                                            dspark_model->size,
-                                            stage0->main_norm->abs_offset,
-                                            DS4_N_EMBD,
-                                            DS4_RMS_EPS) != 0;
-    }
-    if (ok) {
-        ok = ds4_gpu_embed_tokens_hc_tensor(g->dspark_draft_hc,
-                                            g->dspark_draft_tokens,
-                                            base_model->map,
-                                            base_model->size,
-                                            base_weights->token_embd->abs_offset,
-                                            (uint32_t)base_weights->token_embd->dim[1],
-                                            dw->block_size,
-                                            DS4_N_EMBD,
-                                            DS4_N_HC) != 0;
+                                            DS4_RMS_EPS) != 0;
     }
+    if (ok) ok = metal_graph_embed_dspark_draft_rows(g,
+                                                      base_model,
+                                                      base_weights,
+                                                      dw,
+                                                      token);
     if (ok) {
         ok = ds4_gpu_repeat_hc_tensor(g->dspark_target_hc,
                                       g->dspark_main_x,
@@ -33791,25 +35201,55 @@ static bool metal_graph_eval_dspark_stage_block(
                                                  q_dim,
                                                  metal_graph_batch_qr_norm(g),
                                                  draft);
-    if (ok) ok = ds4_gpu_head_rms_norm_tensor(metal_graph_batch_q(g),
-                                               draft,
-                                               DS4_N_HEAD,
-                                               DS4_N_HEAD_DIM,
-                                               DS4_RMS_EPS) != 0;
-    if (ok) ok = ds4_gpu_rope_tail_tensor(metal_graph_batch_q(g),
+    bool q_norm_rope_fused = false;
+#if !defined(DS4_NO_GPU) && !defined(__APPLE__) && !defined(DS4_ROCM_BUILD)
+    /* DSpark's tiny CUDA batches otherwise launch separate head-normalization
+     * and RoPE kernels at every support stage.  The fused kernel is already
+     * used by the target graph and preserves the same operation order within
+     * each head.  Keep a kill switch and the established two-kernel fallback
+     * so unsupported shapes never make the proposer unavailable. */
+    if (ok && getenv("DS4_CUDA_DSPARK_NO_Q_NORM_ROPE_FUSION") == NULL) {
+        q_norm_rope_fused = ds4_gpu_head_rms_norm_rope_tail_tensor(
+                metal_graph_batch_q(g),
+                draft,
+                DS4_N_HEAD,
+                DS4_N_HEAD_DIM,
+                DS4_N_ROT,
+                pos,
+                0,
+                false,
+                freq_base,
+                freq_scale,
+                ext_factor,
+                attn_factor,
+                DS4_ROPE_YARN_BETA_FAST,
+                DS4_ROPE_YARN_BETA_SLOW,
+                DS4_RMS_EPS) != 0;
+    }
+#endif
+    if (ok && !q_norm_rope_fused) {
+        ok = ds4_gpu_head_rms_norm_tensor(metal_graph_batch_q(g),
                                            draft,
                                            DS4_N_HEAD,
                                            DS4_N_HEAD_DIM,
-                                           DS4_N_ROT,
-                                           pos,
-                                           0,
-                                           false,
-                                           freq_base,
-                                           freq_scale,
-                                           ext_factor,
-                                           attn_factor,
-                                           DS4_ROPE_YARN_BETA_FAST,
-                                           DS4_ROPE_YARN_BETA_SLOW) != 0;
+                                           DS4_RMS_EPS) != 0;
+    }
+    if (ok && !q_norm_rope_fused) {
+        ok = ds4_gpu_rope_tail_tensor(metal_graph_batch_q(g),
+                                       draft,
+                                       DS4_N_HEAD,
+                                       DS4_N_HEAD_DIM,
+                                       DS4_N_ROT,
+                                       pos,
+                                       0,
+                                       false,
+                                       freq_base,
+                                       freq_scale,
+                                       ext_factor,
+                                       attn_factor,
+                                       DS4_ROPE_YARN_BETA_FAST,
+                                       DS4_ROPE_YARN_BETA_SLOW) != 0;
+    }
     DS4_DSPARK_PROFILE_STAGE("q_path");
 
     if (ok) ok = metal_graph_matmul_plain_tensor(metal_graph_batch_kv_raw(g),
@@ -34062,106 +35502,6 @@ static bool metal_graph_eval_dspark_stage_chain(
     return true;
 }
 
-/* Keep the support KV ring aligned while the scheduler skips proposals. */
-static bool metal_graph_dspark_ring_maintain(
-        ds4_gpu_graph            *g,
-        const ds4_model          *dspark_model,
-        const ds4_dspark_weights *dw,
-        uint32_t                  pos) {
-    if (!g || !dspark_model || !dw ||
-        !g->dspark_capture_valid ||
-        g->dspark_cache_len == 0 ||
-        !metal_graph_dspark_cache_ends_at(g, pos) ||
-        !dspark_stage0_weights_ready(g, dw) ||
-        !dspark_stage_cache_ready(g, dw) ||
-        !metal_graph_batch_kv_raw(g) || !metal_graph_batch_kv(g)) {
-        return false;
-    }
-    for (uint32_t stage = 0; stage < dw->n_stages; stage++) {
-        if (!dspark_stage_block_ready(g, dw, stage)) return false;
-    }
-
-    const ds4_dspark_stage_weights *stage0 = &dw->stage[0];
-    const uint64_t in_dim = (uint64_t)dw->target_layer_count * DS4_N_EMBD;
-    ds4_gpu_tensor *kv_raw_view =
-        ds4_gpu_tensor_view(metal_graph_batch_kv_raw(g),
-                            0,
-                            (uint64_t)DS4_N_HEAD_DIM * sizeof(float));
-    ds4_gpu_tensor *kv_view =
-        ds4_gpu_tensor_view(metal_graph_batch_kv(g),
-                            0,
-                            (uint64_t)DS4_N_HEAD_DIM * sizeof(float));
-    bool ok = kv_raw_view && kv_view && ds4_gpu_begin_commands() != 0;
-    if (ok) {
-        ok = metal_graph_matmul_plain_tensor(g->dspark_stage0_proj,
-                                             dspark_model,
-                                             stage0->main_proj,
-                                             in_dim,
-                                             DS4_N_EMBD,
-                                             g->dspark_target_hidden,
-                                             1);
-    }
-    if (ok) {
-        ok = ds4_gpu_rms_norm_weight_tensor(g->dspark_main_x,
-                                            g->dspark_stage0_proj,
-                                            dspark_model->map,
-                                            dspark_model->size,
-                                            stage0->main_norm->abs_offset,
-                                            DS4_N_EMBD,
-                                            DS4_RMS_EPS) != 0;
-    }
-    for (uint32_t stage = 0; ok && stage < dw->n_stages; stage++) {
-        const ds4_layer_weights *block = &dw->stage[stage].block;
-        ok = metal_graph_matmul_plain_tensor(kv_raw_view,
-                                             dspark_model,
-                                             block->attn_kv,
-                                             DS4_N_EMBD,
-                                             DS4_N_HEAD_DIM,
-                                             g->dspark_main_x,
-                                             1);
-        if (ok) ok = ds4_gpu_rms_norm_weight_rows_tensor(
-                         kv_view,
-                         kv_raw_view,
-                         dspark_model->map,
-                         dspark_model->size,
-                         block->attn_kv_a_norm->abs_offset,
-                         DS4_N_HEAD_DIM,
-                         1,
-                         DS4_RMS_EPS) != 0;
-        if (ok) ok = ds4_gpu_rope_tail_tensor(kv_view,
-                                               1,
-                                               1,
-                                               DS4_N_HEAD_DIM,
-                                               DS4_N_ROT,
-                                               pos,
-                                               0,
-                                               false,
-                                               DS4_ROPE_FREQ_BASE,
-                                               1.0f,
-                                               0.0f,
-                                               1.0f,
-                                               DS4_ROPE_YARN_BETA_FAST,
-                                               DS4_ROPE_YARN_BETA_SLOW) != 0;
-        if (ok) ok = ds4_gpu_dsv4_fp8_kv_quantize_tensor(kv_view,
-                                                          1,
-                                                          DS4_N_HEAD_DIM,
-                                                          DS4_N_ROT) != 0;
-        if (ok) ok = ds4_gpu_store_raw_kv_batch_tensor(
-                         g->dspark_raw_cache[stage],
-                         kv_view,
-                         g->dspark_cache_cap,
-                         pos,
-                         1,
-                         DS4_N_HEAD_DIM) != 0;
-    }
-    if (ok) ok = ds4_gpu_end_commands() != 0;
-    else (void)ds4_gpu_synchronize();
-    ds4_gpu_tensor_free(kv_view);
-    ds4_gpu_tensor_free(kv_raw_view);
-    if (ok) (void)metal_graph_dspark_cache_claim_appended_row(g, pos);
-    return ok;
-}
-
 static ds4_gpu_tensor *metal_graph_dspark_final_output_hc(const ds4_gpu_graph *g) {
     if (!g) return NULL;
     if (getenv("DS4_DSPARK_DISABLE_FINAL_OUTPUT_ALIAS") == NULL &&
@@ -34487,13 +35827,36 @@ static bool metal_graph_eval_dspark_base_logits_from_hidden(
                                 DS4_N_VOCAB * sizeof(float));
     bool ok = output_norm && logits;
     if (ok) ok = ds4_gpu_begin_commands() != 0;
-    if (ok) ok = metal_graph_matmul_plain_tensor(logits,
-                                                  base_model,
-                                                  base_weights->output,
-                                                  DS4_N_EMBD,
-                                                  DS4_N_VOCAB,
-                                                  output_norm,
-                                                  dw->block_size);
+#if !defined(DS4_NO_GPU) && !defined(__APPLE__) && !defined(DS4_ROCM_BUILD)
+    /* The exact CUDA Q8 tensor-core vocab kernel starts at eight rows.  DSpark
+     * produces five, while both backing workspaces have spare rows.  Reuse the
+     * verifier's zero-padding helper so the five meaningful rows take the MMA
+     * path; the padded rows are never exposed to the proposer. */
+    const bool use_cuda_padded_head =
+        base_weights->output->type == DS4_TENSOR_Q8_0 &&
+        dw->block_size > 1u && dw->block_size < 8u &&
+        getenv("DS4_CUDA_DSPARK_NO_PADDED_HEAD") == NULL;
+#else
+    const bool use_cuda_padded_head = false;
+#endif
+    if (ok && use_cuda_padded_head) {
+        ok = metal_graph_output_logits_head_matmul(
+                g,
+                base_model,
+                base_weights,
+                metal_graph_batch_ffn_norm(g),
+                g->spec_logits,
+                dw->block_size,
+                DS4_N_VOCAB);
+    } else if (ok) {
+        ok = metal_graph_matmul_plain_tensor(logits,
+                                             base_model,
+                                             base_weights->output,
+                                             DS4_N_EMBD,
+                                             DS4_N_VOCAB,
+                                             output_norm,
+                                             dw->block_size);
+    }
     if (ok) ok = ds4_gpu_end_commands() != 0;
     if (!ok) (void)ds4_gpu_synchronize();
 
@@ -35151,6 +36514,166 @@ static uint32_t dspark_confident_prefix_len(
     return confidence_len;
 }
 
+/* Keep the Q8 confidence/Markov chain on the GPU and read one compact result at
+ * the end.  Unsupported layouts and every validation discrepancy are
+ * fail-closed: the caller immediately runs the established per-row path.
+ *
+ * This is deliberately opt-in.  The GPU reproduces the Q8 activation
+ * quantization and Markov tie-break, but the confidence accumulation runs on
+ * a different processor and therefore still needs the backend parity oracle
+ * before this can become a default. */
+static bool dspark_apply_markov_confidence_device_runtime(
+        ds4_gpu_graph            *g,
+        const ds4_model          *dspark_model,
+        const ds4_dspark_weights *dw,
+        int                       first_prev_token,
+        float                     confidence_threshold,
+        int32_t                   proposal[DS4_DSPARK_MAX_BLOCK_SIZE],
+        uint32_t                 *proposal_len,
+        uint32_t                 *confidence_len,
+        uint32_t                 *confidence_prefix_len,
+        bool                      reuse_first_confidence,
+        float                    *confidence0,
+        bool                     *attempted,
+        bool                     *policy_mismatch) {
+    if (attempted) *attempted = false;
+    if (policy_mismatch) *policy_mismatch = false;
+#if !defined(DS4_ROCM_BUILD)
+#if defined(__APPLE__)
+    const char *device_enable =
+        getenv("DS4_METAL_DSPARK_DEVICE_PROPOSER");
+    const char *device_disable =
+        getenv("DS4_METAL_DSPARK_NO_DEVICE_PROPOSER");
+#else
+    const char *device_enable =
+        getenv("DS4_CUDA_DSPARK_DEVICE_PROPOSER");
+#endif
+    const bool device_requested =
+        device_enable && device_enable[0] &&
+        strcmp(device_enable, "0") != 0 &&
+        strcmp(device_enable, "off") != 0 &&
+        strcmp(device_enable, "false") != 0 &&
+#if defined(__APPLE__)
+        device_disable == NULL;
+#else
+        getenv("DS4_CUDA_DSPARK_NO_DEVICE_PROPOSER") == NULL;
+#endif
+    if (!g || !g->dspark_device_proposal || !g->spec_logits ||
+        !metal_graph_batch_ffn_norm(g) || !dspark_model || !dw ||
+        !proposal || !proposal_len || !confidence_len ||
+        !confidence_prefix_len || !confidence0 ||
+        first_prev_token < 0 ||
+        (uint32_t)first_prev_token >= DS4_N_VOCAB ||
+        confidence_threshold <= 0.0f || confidence_threshold > 1.0f ||
+        dw->block_size == 0 ||
+        dw->block_size > DS4_GPU_DSPARK_MAX_DRAFTS ||
+        g->placement != NULL ||
+#if !defined(__APPLE__)
+        g->ssd_streaming ||
+#endif
+        g->quality ||
+        g->active_tier != 0 || g->dspark_exec_tier != 0 ||
+        g->tp_world != 0 || !reuse_first_confidence ||
+        !device_requested ||
+        getenv("DS4_DSPARK_NO_GPU_MARKOV") != NULL ||
+        dspark_markov_bias_disabled() ||
+        !dspark_markov_probe_ready(dw) ||
+        !dspark_confidence_probe_ready(dw)) {
+        return false;
+    }
+
+    const ds4_dspark_stage_weights *final =
+        &dw->stage[dw->n_stages - 1u];
+    if (final->markov_w1->type != DS4_TENSOR_Q8_0 ||
+        final->markov_w2->type != DS4_TENSOR_Q8_0 ||
+        final->confidence_proj->type != DS4_TENSOR_Q8_0) {
+        return false;
+    }
+
+    if (attempted) *attempted = true;
+    ds4_gpu_dspark_device_proposal result;
+    memset(&result, 0, sizeof(result));
+    if (!ds4_gpu_dspark_markov_confidence_q8_tensor(
+                g->dspark_device_proposal,
+                g->spec_logits,
+                metal_graph_batch_ffn_norm(g),
+                dspark_model->map,
+                dspark_model->size,
+                final->markov_w1->abs_offset,
+                final->markov_w2->abs_offset,
+                final->confidence_proj->abs_offset,
+                (uint32_t)first_prev_token,
+                DS4_N_VOCAB,
+                dw->markov_rank,
+                DS4_N_EMBD,
+                dw->block_size,
+                confidence_threshold,
+                reuse_first_confidence ? 1 : 0,
+                *confidence0) ||
+        !ds4_gpu_tensor_read(g->dspark_device_proposal,
+                             0,
+                             &result,
+                             sizeof(result))) {
+        return false;
+    }
+
+    if (result.status != 1u || result.reserved != 0u ||
+        result.confidence_len == 0u ||
+        result.confidence_len > dw->block_size ||
+        result.proposal_len > result.confidence_len ||
+        result.proposal_len > dw->block_size ||
+        !((result.proposal_len == dw->block_size &&
+           result.confidence_len == dw->block_size) ||
+          (result.proposal_len < dw->block_size &&
+           result.confidence_len == result.proposal_len + 1u))) {
+        return false;
+    }
+    for (uint32_t i = 0; i < result.confidence_len; i++) {
+        if (!isfinite(result.confidence_logits[i])) return false;
+    }
+    for (uint32_t i = 0; i < result.proposal_len; i++) {
+        if (result.tokens[i] < 0 ||
+            (uint32_t)result.tokens[i] >= DS4_N_VOCAB) {
+            return false;
+        }
+    }
+
+    const uint32_t cpu_prefix =
+        dspark_confident_prefix_len(result.confidence_logits,
+                                    result.confidence_len,
+                                    confidence_threshold);
+    /* The device may stop on a libdevice sigmoid rounding difference.  If
+     * the established CPU policy accepts that row, the device did not
+     * compute its Markov token and the result is incomplete. */
+    if (cpu_prefix > result.proposal_len) {
+        if (policy_mismatch) *policy_mismatch = true;
+        return false;
+    }
+
+    for (uint32_t i = 0; i < result.proposal_len; i++) {
+        proposal[i] = result.tokens[i];
+    }
+    *proposal_len = result.proposal_len;
+    *confidence_len = result.confidence_len;
+    *confidence_prefix_len = cpu_prefix;
+    *confidence0 = result.confidence_logits[0];
+    return true;
+#else
+    (void)g;
+    (void)dspark_model;
+    (void)dw;
+    (void)first_prev_token;
+    (void)confidence_threshold;
+    (void)proposal;
+    (void)proposal_len;
+    (void)confidence_len;
+    (void)confidence_prefix_len;
+    (void)reuse_first_confidence;
+    (void)confidence0;
+    return false;
+#endif
+}
+
 static bool metal_graph_eval_mtp_draft_from_hc(
         ds4_gpu_graph       *g,
         const ds4_model       *base_model,
@@ -35684,6 +37207,38 @@ static void gpu_graph_report_prefill_display_progress(
                      (int)(start + (uint32_t)done), total);
 }
 
+#ifdef __APPLE__
+typedef struct {
+    ds4_session_progress_fn display_progress;
+    void                   *ud;
+    int                     current;
+    int                     total;
+    volatile uint32_t       completed;
+} metal_graph_flush_progress_ctx;
+
+/* Metal completion threads only publish readiness.  User callbacks can own
+ * sockets, terminal state, or non-atomic session fields, so they stay on the
+ * graph caller thread and are emitted in monotonically increasing order. */
+static void metal_graph_flush_progress_mark(void *vctx) {
+    metal_graph_flush_progress_ctx *ctx = vctx;
+    __atomic_store_n(&ctx->completed, 1u, __ATOMIC_RELEASE);
+}
+
+static void metal_graph_flush_progress_report_ready(
+        metal_graph_flush_progress_ctx *ctx,
+        uint32_t                        submitted,
+        uint32_t                       *reported) {
+    if (!ctx || !reported) return;
+    while (*reported < submitted &&
+           __atomic_load_n(&ctx[*reported].completed, __ATOMIC_ACQUIRE)) {
+        metal_graph_flush_progress_ctx *ready = &ctx[*reported];
+        ready->display_progress(
+            ready->ud, "prefill_display", ready->current, ready->total);
+        (*reported)++;
+    }
+}
+#endif
+
 typedef struct {
     int tier;
     uint32_t first_layer;
@@ -36057,6 +37612,16 @@ static bool metal_graph_prefill_layer_major(
      */
     const bool throttle = graph_power_throttle_enabled(g);
     const bool callback_split = display_progress != NULL && n_tokens >= 32;
+#ifdef __APPLE__
+    /* A display-only split must preserve layer boundaries for truthful UI
+     * progress, but it does not need a host-side drain after every layer. */
+    const bool progress_flush =
+        callback_split && n_tokens <= 2048u && !g->ssd_streaming &&
+        !split_profile && !throttle && imatrix == NULL &&
+        getenv("DS4_METAL_DISABLE_PREFILL_FLUSH_PROGRESS") == NULL;
+#else
+    const bool progress_flush = false;
+#endif
     const bool split_commands = g->ssd_streaming ||
                                 split_profile || throttle || callback_split ||
                                 n_tokens > 2048 || imatrix != NULL;
@@ -36283,7 +37848,23 @@ static bool metal_graph_prefill_layer_major(
         return false;
     }
 
+#ifdef __APPLE__
+    metal_graph_flush_progress_ctx flush_ctx[DS4_N_LAYER];
+    memset(flush_ctx, 0, sizeof(flush_ctx));
+    uint32_t flush_submitted = 0u;
+    uint32_t flush_reported = 0u;
+    if (progress_flush && ok) {
+        ok = ds4_gpu_begin_commands() != 0;
+    }
+#endif
+
     for (uint32_t il = 0; ok && il < DS4_N_LAYER; il++) {
+#ifdef __APPLE__
+        if (progress_flush) {
+            metal_graph_flush_progress_report_ready(
+                flush_ctx, flush_submitted, &flush_reported);
+        }
+#endif
         double layer_elapsed = 0.0;
         const bool layer_selected_addr =
             batch_selected_addr &&
@@ -36485,7 +38066,7 @@ static bool metal_graph_prefill_layer_major(
                     (t_ffn_done - t_ffn_encoded) * 1000.0);
         } else {
             const double t_chunk0 = (profile || throttle) ? now_sec() : 0.0;
-            ok = ds4_gpu_begin_commands() != 0;
+            if (!progress_flush) ok = ds4_gpu_begin_commands() != 0;
             if (ok) ok = metal_graph_encode_layer_batch(g,
                                                         model,
                                                         &weights->layer[il],
@@ -36512,7 +38093,31 @@ static bool metal_graph_prefill_layer_major(
             }
 #endif
             const double t_encoded = (profile || throttle) ? now_sec() : 0.0;
-            if (ok) ok = ds4_gpu_end_commands() != 0;
+#ifdef __APPLE__
+            if (ok && progress_flush) {
+                uint64_t pdone =
+                    (uint64_t)n_tokens * (il + 1u) / (uint32_t)DS4_N_LAYER;
+                if (il + 1u == (uint32_t)DS4_N_LAYER) pdone = n_tokens;
+                flush_ctx[il] = (metal_graph_flush_progress_ctx){
+                    display_progress,
+                    display_progress_ud,
+                    (int)(start + (uint32_t)pdone),
+                    prompt->len,
+                    0u,
+                };
+                ok = ds4_gpu_flush_commands_progress(
+                         metal_graph_flush_progress_mark,
+                         &flush_ctx[il]) != 0;
+                if (ok) {
+                    flush_submitted = il + 1u;
+                    metal_graph_flush_progress_report_ready(
+                        flush_ctx, flush_submitted, &flush_reported);
+                }
+            } else
+#endif
+            if (ok) {
+                ok = ds4_gpu_end_commands() != 0;
+            }
             const double t_done = (profile || throttle) ? now_sec() : 0.0;
 #ifdef DS4_ROCM_BUILD
             if (ok) {
@@ -36584,12 +38189,14 @@ static bool metal_graph_prefill_layer_major(
             return false;
         }
         graph_power_note_prefill_layer(g, il, layer_elapsed);
-        gpu_graph_report_prefill_display_progress(display_progress,
-                                                  display_progress_ud,
-                                                  start,
-                                                  n_tokens,
-                                                  il + 1,
-                                                  prompt->len);
+        if (!progress_flush) {
+            gpu_graph_report_prefill_display_progress(display_progress,
+                                                      display_progress_ud,
+                                                      start,
+                                                      n_tokens,
+                                                      il + 1,
+                                                      prompt->len);
+        }
         if (show_progress) {
             fprintf(stderr, "ds4: gpu prefill layer %u/%u\r", il + 1, (uint32_t)DS4_N_LAYER);
             fflush(stderr);
@@ -36609,6 +38216,18 @@ static bool metal_graph_prefill_layer_major(
         }
         return false;
     }
+#ifdef __APPLE__
+    /* Every flush opens the next batch.  If no output head follows, close the
+     * final empty batch and wait here so completion contexts stay alive. */
+    if (progress_flush && !logits) {
+        ok = ds4_gpu_end_commands() != 0;
+        if (ok) {
+            metal_graph_flush_progress_report_ready(
+                flush_ctx, flush_submitted, &flush_reported);
+            ok = flush_reported == flush_submitted;
+        }
+    }
+#endif
 #ifdef __APPLE__
     /* Zero-prefix masks are shared across the 43 per-layer command batches,
      * then become dead weight. Release them before the output head and later
@@ -36633,9 +38252,15 @@ static bool metal_graph_prefill_layer_major(
     if (g->ssd_streaming) ds4_gpu_release_q8_f16_cache();
 #endif
     if (!metal_graph_seed_streaming_expert_cache_from_hotlist(g, model, weights)) {
+#ifdef __APPLE__
+        if (progress_flush) (void)ds4_gpu_synchronize();
+#endif
         return false;
     }
     if (!metal_graph_seed_streaming_expert_cache_from_prefill(g, model, weights)) {
+#ifdef __APPLE__
+        if (progress_flush) (void)ds4_gpu_synchronize();
+#endif
         return false;
     }
 
@@ -36678,15 +38303,34 @@ static bool metal_graph_prefill_layer_major(
     }
     if (ok && logits) {
         g->cur_hc_by_tier[g->active_tier] = last_hc;
-        ok = ds4_gpu_begin_commands() != 0;
+        if (!progress_flush || !ds4_gpu_commands_active()) {
+            ok = ds4_gpu_begin_commands() != 0;
+        }
     }
     if (ok && logits) ok = metal_graph_encode_output_head(g, model, weights, weights->output->dim[1]);
     const double t_head_encoded = profile ? now_sec() : 0.0;
     if (ok && logits) ok = ds4_gpu_end_commands() != 0;
+#ifdef __APPLE__
+    if (progress_flush && !ds4_gpu_commands_active()) {
+        metal_graph_flush_progress_report_ready(
+            flush_ctx, flush_submitted, &flush_reported);
+        if (ok && flush_reported != flush_submitted) ok = false;
+    }
+#endif
     const double t_head_done = profile ? now_sec() : 0.0;
     g->cur_hc_by_tier[g->active_tier] = saved_cur;
     if (last_hc) ds4_gpu_tensor_free(last_hc);
-    if (!ok) return false;
+    if (!ok) {
+#ifdef __APPLE__
+        /* A failed head setup/encode can leave the empty post-flush batch
+         * open.  Close it and join completion hooks before stack contexts go
+         * out of scope. */
+        if (progress_flush && ds4_gpu_commands_active()) {
+            (void)ds4_gpu_end_commands();
+        }
+#endif
+        return false;
+    }
 
     const double t_before_read = profile ? now_sec() : 0.0;
     if (logits) {
@@ -36767,6 +38411,44 @@ static bool metal_graph_prefill_raw_swa(
                                            display_progress_ud);
 }
 
+static uint32_t metal_graph_prefill_chunk_rows_at(
+        const ds4_gpu_graph *g,
+        uint32_t range_start,
+        uint32_t pos0,
+        uint32_t remaining) {
+    if (!g || remaining == 0u || g->prefill_cap == 0u) return 0u;
+    uint32_t cap = g->prefill_cap;
+    if (range_start != 0u && cap > g->raw_cap) cap = g->raw_cap;
+    if (cap == 0u) return 0u;
+
+    if (range_start != 0u) {
+        const uint32_t mod = pos0 % g->prefill_cap;
+        if (mod != 0u) {
+            const uint32_t to_boundary = g->prefill_cap - mod;
+            if (to_boundary < cap) cap = to_boundary;
+        }
+    }
+    return remaining < cap ? remaining : cap;
+}
+
+/* Maximum batch width the chunk executor below can emit.  If the first chunk
+ * is smaller than cap it reaches an absolute prefill boundary, so the next
+ * chunk can use the full cap; if it is cap-sized, it is already the maximum. */
+static uint32_t metal_graph_prefill_max_chunk_rows(
+        const ds4_gpu_graph *g,
+        uint32_t start,
+        uint32_t n_tokens) {
+    const uint32_t first = metal_graph_prefill_chunk_rows_at(
+        g, start, start, n_tokens);
+    if (first == 0u || first >= n_tokens) return first;
+
+    uint32_t cap = g->prefill_cap;
+    if (start != 0u && cap > g->raw_cap) cap = g->raw_cap;
+    const uint32_t remaining = n_tokens - first;
+    const uint32_t later = remaining < cap ? remaining : cap;
+    return first > later ? first : later;
+}
+
 static uint32_t metal_graph_visual_prefill_chunk(
         const ds4_gpu_graph *g,
         const token_vec     *prompt,
@@ -36860,21 +38542,16 @@ static bool metal_graph_prefill_chunked_range(
             return true;
         }
         const uint32_t remaining = end - pos0;
-        uint32_t local_cap = chunk_cap;
-        if (start != 0 && g->prefill_cap != 0) {
-            const uint32_t mod = pos0 % g->prefill_cap;
-            if (mod != 0) {
-                const uint32_t to_boundary = g->prefill_cap - mod;
-                if (to_boundary < local_cap) local_cap = to_boundary;
-            }
-        }
+        uint32_t local_cap = metal_graph_prefill_chunk_rows_at(
+            g, start, pos0, remaining);
+        if (local_cap == 0u) return false;
         if (g->deepseek4_vision_weights &&
             prompt->v[pos0] >= (int)DS4_N_VOCAB) {
             local_cap = chunk_cap;
         }
-        uint32_t chunk = remaining < local_cap ? remaining : local_cap;
-        chunk = metal_graph_visual_prefill_chunk(g, prompt, pos0, end, local_cap);
-        if (chunk == 0) return false;
+        const uint32_t chunk = metal_graph_visual_prefill_chunk(
+            g, prompt, pos0, end, local_cap);
+        if (chunk == 0u) return false;
         const uint32_t chunk_end = pos0 + chunk;
         float *chunk_logits = (progress || chunk_end == end) ? logits : NULL;
         bool ok = metal_graph_prefill_layer_major(g,
@@ -36984,33 +38661,52 @@ static bool metal_graph_verify_suffix_tops_impl(
         const ds4_weights     *weights,
         const token_vec       *prompt,
         uint32_t               start,
-        uint32_t               n_tokens,
+        uint32_t               eval_rows,
+        uint32_t               top_rows,
         bool                   capture_prefix1,
         bool                   capture_dspark_hidden,
         int                   *row_tops,
         float                 *row_logits,
         ds4_verify_suffix_timing *timing) {
     if (timing) memset(timing, 0, sizeof(*timing));
-    if (n_tokens == 0 || n_tokens > g->prefill_cap || !g->spec_logits) return false;
-    if (start > (uint32_t)prompt->len || n_tokens > (uint32_t)prompt->len - start) return false;
-    const uint32_t top_rows = n_tokens > 1 ? n_tokens - 1 : 0;
+    if (eval_rows == 0 || eval_rows > g->prefill_cap ||
+        top_rows > eval_rows || !g->spec_logits) return false;
+    if (start > (uint32_t)prompt->len ||
+        eval_rows > (uint32_t)prompt->len - start) return false;
     if (top_rows && !row_tops) return false;
 
+    /* A dynamic SSD decode may leave only the output-head view installed.
+     * Verification immediately needs token, every non-routed layer weight,
+     * and the output head in one layer-major command stream. Install that
+     * complete static set explicitly; Metal keeps the separate DSpark support
+     * views while replacing only this target model's views. */
+    if (g->ssd_streaming) {
+        if (!metal_graph_stream_map_decode_static_all(model, weights)) {
+            return false;
+        }
+        g->streaming_static_decode_map_current =
+            metal_graph_stream_decode_static_map_enabled() &&
+            metal_graph_stream_decode_static_map_state_cache_enabled();
+    }
+
     const double upload_t0 = timing ? now_sec() : 0.0;
-    bool ok = metal_graph_upload_prompt_tokens(metal_graph_prefill_tokens(g), prompt, start, n_tokens);
+    bool ok = metal_graph_upload_prompt_tokens(metal_graph_prefill_tokens(g),
+                                                prompt,
+                                                start,
+                                                eval_rows);
     if (ok) ok = metal_graph_upload_prompt_embeddings_hc(metal_graph_batch_cur_hc(g),
                                                          metal_graph_prefill_tokens(g),
                                                          model,
                                                          weights,
                                                          prompt,
                                                          start,
-                                                         n_tokens);
+                                                         eval_rows);
     if (!ok) return false;
 
     const bool saved_capture = g->spec_capture_prefixes;
     g->spec_capture_prefixes =
-        capture_prefix1 && n_tokens > 1u &&
-        n_tokens <= DS4_SPEC_PREFIX_SLOTS + 1u;
+        capture_prefix1 && eval_rows > 1u &&
+        eval_rows <= DS4_SPEC_PREFIX_SLOTS + 1u;
     const char *split_head_env = getenv("DS4_DSPARK_VERIFY_SPLIT_HEAD");
     const bool fuse_head =
         !split_head_env || !split_head_env[0] ||
@@ -37029,8 +38725,8 @@ static bool metal_graph_verify_suffix_tops_impl(
      * layer reconstructs the routed result while preserving their KV state. */
     g->tp_batch_rows = (g->tp_world == 2 &&
                         g->tp_batch_out != NULL && g->tp_batch_in != NULL &&
-                        n_tokens <= (uint32_t)DS4_TP_BATCH_MAX_ROWS)
-                       ? n_tokens : 0;
+                        eval_rows <= (uint32_t)DS4_TP_BATCH_MAX_ROWS)
+                       ? eval_rows : 0;
 #ifdef DS4_ROCM_BUILD
     bool rocm_dspark_fast = false;
     const char *verify_fast_env =
@@ -37040,7 +38736,7 @@ static bool metal_graph_verify_suffix_tops_impl(
             verify_fast_env[0] != '0' :
             ds4_dspark_rocm_gfx1151_fast_path();
     rocm_dspark_fast =
-        n_tokens >= 2u && n_tokens <= 6u && verify_fast_enabled;
+        eval_rows >= 2u && eval_rows <= 6u && verify_fast_enabled;
     if (rocm_dspark_fast) ds4_gpu_set_dspark_verify_mode(true);
 #endif
     const double layer_t0 = timing ? now_sec() : 0.0;
@@ -37058,7 +38754,7 @@ static bool metal_graph_verify_suffix_tops_impl(
         capture_dspark_hidden &&
         metal_graph_dspark_capture_verified_suffix_begin(g,
                                                          start,
-                                                         n_tokens,
+                                                         eval_rows,
                                                          true);
     static int verify_profile_left = -1;
     if (verify_profile_left < 0) {
@@ -37079,12 +38775,12 @@ static bool metal_graph_verify_suffix_tops_impl(
                                             &weights->layer[il],
                                             il,
                                             start,
-                                            n_tokens);
+                                            eval_rows);
         if (ok && dspark_capture_active) {
             ok = metal_graph_dspark_capture_verified_suffix_layer(g,
                                                                   il,
                                                                   start,
-                                                                  n_tokens);
+                                                                  eval_rows);
         }
         if (ok && selected_profile) {
             ok = ds4_gpu_end_commands() != 0 &&
@@ -37092,7 +38788,7 @@ static bool metal_graph_verify_suffix_tops_impl(
                          g,
                          &weights->layer[il],
                          il,
-                         n_tokens,
+                         eval_rows,
                          "DSpark verifier selected profile") &&
                  ds4_gpu_begin_commands() != 0;
         }
@@ -37112,7 +38808,7 @@ static bool metal_graph_verify_suffix_tops_impl(
         ok = metal_graph_encode_output_head_batch(g,
                                                   model,
                                                   weights,
-                                                  n_tokens,
+                                                  eval_rows,
                                                   weights->output->dim[1]);
     }
     if (ok && fuse_head) {
@@ -37141,13 +38837,13 @@ static bool metal_graph_verify_suffix_tops_impl(
         if (sel) {
             uint8_t seen[1024] = {0};
             uint32_t distinct = 0;
-            for (uint32_t r = 0; r < n_tokens; r++)
+            for (uint32_t r = 0; r < eval_rows; r++)
                 for (uint32_t k = 0; k < DS4_N_EXPERT_USED; k++) {
                     const int32_t id = sel[r * DS4_N_EXPERT_USED + k];
                     if (id >= 0 && id < 1024 && !seen[id]) { seen[id] = 1; distinct++; }
                 }
             fprintf(stderr, "ds4: DSpark verify block: %u rows, %u distinct experts of %u slots (last layer)\n",
-                    n_tokens, distinct, n_tokens * DS4_N_EXPERT_USED);
+                    eval_rows, distinct, eval_rows * DS4_N_EXPERT_USED);
         }
     }
 #ifdef DS4_ROCM_BUILD
@@ -37171,7 +38867,7 @@ static bool metal_graph_verify_suffix_tops_impl(
         if (ok) ok = metal_graph_encode_output_head_batch(g,
                                                           model,
                                                           weights,
-                                                          n_tokens,
+                                                          eval_rows,
                                                           weights->output->dim[1]);
         if (ok) {
             if (top_rows == 1) {
@@ -37227,7 +38923,8 @@ static bool metal_graph_verify_suffix_tops_impl(
         ok = ds4_gpu_tensor_read(g->spec_logits,
                                    0,
                                    row_logits,
-                                   (uint64_t)n_tokens * DS4_N_VOCAB * sizeof(row_logits[0])) != 0;
+                                   (uint64_t)eval_rows * DS4_N_VOCAB *
+                                       sizeof(row_logits[0])) != 0;
     }
     if (timing) timing->read_ms += (now_sec() - read_t0) * 1000.0;
     return ok;
@@ -37251,6 +38948,8 @@ static bool metal_graph_verify_suffix_tops(
     const bool ok = metal_graph_verify_suffix_tops_impl(g, model, weights,
                                                         prompt, start,
                                                         n_tokens,
+                                                        n_tokens > 1u ?
+                                                            n_tokens - 1u : 0u,
                                                         capture_prefix1,
                                                         capture_dspark_hidden,
                                                         row_tops, row_logits,
@@ -37259,6 +38958,36 @@ static bool metal_graph_verify_suffix_tops(
     return ok;
 }
 
+/* DSpark's current target logits already verify draft[0].  A rollback/replay
+ * cycle therefore needs target rows only for draft[0..N-2], whose N-1 tops
+ * verify draft[1..N-1].  This dedicated entry point deliberately disables
+ * transient prefix/hidden captures because every verifier side effect is
+ * rolled back before the exact replay. */
+static bool metal_graph_verify_suffix_acceptance_tops(
+        ds4_gpu_graph *g,
+        const ds4_model       *model,
+        const ds4_weights     *weights,
+        const token_vec       *prompt,
+        uint32_t               start,
+        uint32_t               draft_n,
+        int                   *row_tops,
+        ds4_verify_suffix_timing *timing) {
+    if (draft_n <= 1u) return false;
+    const uint32_t eval_rows = draft_n - 1u;
+    ds4_gpu_tp_keepalive_pause(1);
+    const bool ok = metal_graph_verify_suffix_tops_impl(g, model, weights,
+                                                        prompt, start,
+                                                        eval_rows,
+                                                        eval_rows,
+                                                        false,
+                                                        false,
+                                                        row_tops,
+                                                        NULL,
+                                                        timing);
+    ds4_gpu_tp_keepalive_pause(0);
+    return ok;
+}
+
 static bool metal_graph_read_spec_logits_row(ds4_gpu_graph *g, uint32_t row, float *logits) {
     if (!g || !g->spec_logits || !logits || row >= g->prefill_cap) return false;
     const uint64_t row_bytes = (uint64_t)DS4_N_VOCAB * sizeof(float);
@@ -37276,13 +39005,15 @@ static bool metal_graph_read_spec_logits_row(ds4_gpu_graph *g, uint32_t row, flo
  * decode kernels and cache update order, but encodes the two proposed tokens
  * layer-by-layer in one command stream.  It returns the exact target top after
  * token0, and exact logits after token1. */
-static bool metal_graph_verify_decode2_exact(
+static bool metal_graph_verify_decode2_exact_impl(
         ds4_gpu_graph *g,
         const ds4_model       *model,
         const ds4_weights     *weights,
         int                    token0,
         int                    token1,
         uint32_t               start,
+        bool                   capture_prefix1,
+        int                    expected_token1,
         int                   *top0,
         int                   *top1,
         float                 *logits0,
@@ -37299,6 +39030,8 @@ static bool metal_graph_verify_decode2_exact(
     ds4_gpu_tensor *saved_after_by_tier[DS4_MAX_GPUS] = {0};
     const int saved_active_tier = g->active_tier;
     const bool saved_capture = g->spec_capture_prefixes;
+    const bool saved_disable_decode_graphs =
+        g->spec_disable_decode_graphs;
 
     bool ok = true;
     for (int t = 0; t < DS4_MAX_GPUS; t++) {
@@ -37343,7 +39076,8 @@ static bool metal_graph_verify_decode2_exact(
                                                DS4_N_EMBD,
                                                DS4_N_HC) != 0;
 
-    g->spec_capture_prefixes = true;
+    g->spec_capture_prefixes = capture_prefix1;
+    g->spec_disable_decode_graphs = true;
     if (ok) ok = ds4_gpu_begin_commands() != 0;
     for (uint32_t il = 0; ok && il < DS4_N_LAYER; il++) {
         const uint32_t pos0 = start;
@@ -37408,6 +39142,7 @@ static bool metal_graph_verify_decode2_exact(
     if (ok) ok = ds4_gpu_end_commands() != 0;
     else (void)ds4_gpu_synchronize();
     g->spec_capture_prefixes = saved_capture;
+    g->spec_disable_decode_graphs = saved_disable_decode_graphs;
 
     if (ok) {
         ok = metal_graph_set_active_tier_no_copy(g, cur_tier);
@@ -37439,10 +39174,12 @@ static bool metal_graph_verify_decode2_exact(
         }
     }
 
-    if (ok) {
+    const bool need_token1_head =
+        !ok || expected_token1 < 0 || *top0 == expected_token1;
+    if (ok && need_token1_head) {
         ok = metal_graph_set_active_tier_no_copy(g, cur_tier);
     }
-    if (ok) {
+    if (ok && need_token1_head) {
         const bool split_top1 =
             logits1 == NULL &&
             top1 != NULL &&
@@ -37506,6 +39243,1156 @@ static bool metal_graph_verify_decode2_exact(
     return ok;
 }
 
+enum { DS4_CUDA_EXACTN_MAX_ROWS = 5 };
+
+typedef struct {
+    double setup_ms;
+    double layer_ms;
+    double head_ms;
+    double read_ms;
+    uint64_t graph_captures;
+    uint64_t graph_replays;
+    uint64_t graph_warms;
+    uint64_t graph_no_slots;
+    uint64_t graph_failures;
+    bool graphs_attempted;
+    bool graphs_used;
+    bool batch_head_attempted;
+    bool batch_head_used;
+    bool batch_head_fallback;
+} ds4_cuda_exactn_timing;
+
+/* CUDA Graphs are an independent exact-N experiment.  Stable storage-address
+ * keys make recreated row views safe, while the global disable remains the
+ * emergency rollback for graph capture itself. */
+static bool metal_graph_cuda_exactn_graphs_requested(
+        const ds4_gpu_graph *g,
+        uint32_t             n_tokens) {
+#if !defined(DS4_NO_GPU) && !defined(__APPLE__) && !defined(DS4_ROCM_BUILD)
+    return g && n_tokens >= 2u &&
+           n_tokens <= DS4_CUDA_EXACTN_MAX_ROWS &&
+           g->placement == NULL && g->active_tier == 0 &&
+           g->tp_world <= 1u && !g->ssd_streaming &&
+           metal_graph_tp_env_flag(
+                   "DS4_CUDA_DSPARK_EXACTN_GRAPHS", false) &&
+           !metal_graph_tp_env_flag(
+                   "DS4_CUDA_DISABLE_DSPARK_EXACTN_GRAPHS", false) &&
+           ds4_gpu_decode_graphs_supported() != 0;
+#else
+    (void)g;
+    (void)n_tokens;
+    return false;
+#endif
+}
+
+/* Keep the exact-N head experiment narrower than the resident verifier gate:
+ * only Q8 output weights have a multi-row CUDA entry point that explicitly
+ * preserves the ordinary one-row reduction order.  The disable variable wins
+ * so a deployed command line can retain the enable flag during an A/B. */
+static bool metal_graph_cuda_exactn_batch_head_requested(
+        const ds4_gpu_graph *g,
+        const ds4_weights   *weights,
+        uint32_t             n_tokens) {
+#if !defined(__APPLE__) && !defined(DS4_ROCM_BUILD)
+    return g && weights && weights->output && n_tokens >= 2u &&
+           n_tokens <= DS4_CUDA_EXACTN_MAX_ROWS &&
+           g->placement == NULL && g->active_tier == 0 &&
+           g->head_tier == 0 &&
+           weights->output->type == DS4_TENSOR_Q8_0 &&
+           weights->output->ndim == 2 &&
+           weights->output->dim[0] == DS4_N_EMBD &&
+           weights->output->dim[1] == DS4_N_VOCAB &&
+           metal_graph_batch_ffn_norm(g) && g->spec_logits &&
+           ds4_gpu_tensor_bytes(metal_graph_batch_ffn_norm(g)) >=
+               (uint64_t)n_tokens * DS4_N_EMBD * sizeof(float) &&
+           ds4_gpu_tensor_bytes(g->spec_logits) >=
+               (uint64_t)n_tokens * DS4_N_VOCAB * sizeof(float) &&
+           metal_graph_tp_env_flag(
+                   "DS4_CUDA_DSPARK_EXACTN_BATCH_HEAD", false) &&
+           !metal_graph_tp_env_flag(
+                   "DS4_CUDA_DISABLE_DSPARK_EXACTN_BATCH_HEAD", false);
+#else
+    (void)g;
+    (void)weights;
+    (void)n_tokens;
+    return false;
+#endif
+}
+
+/* Canonical one-row output-head prefix, stopping immediately before the vocab
+ * projection.  exact-N invokes it once per row, but binds output_norm to a
+ * distinct contiguous row.  Thus HC collapse and normalization retain the
+ * exact one-row kernels while the expensive Q8 vocabulary projection can be
+ * issued once for all rows. */
+static bool metal_graph_encode_output_head_norm_exact_row(
+        ds4_gpu_graph    *g,
+        const ds4_model  *model,
+        const ds4_weights *weights,
+        ds4_gpu_tensor   *norm_dst) {
+    if (!g || !model || !weights || !norm_dst || g->placement != NULL ||
+        g->active_tier != 0 || g->head_tier != 0 ||
+        ds4_gpu_tensor_bytes(norm_dst) <
+            (uint64_t)DS4_N_EMBD * sizeof(float)) {
+        return false;
+    }
+
+    const uint64_t hc_dim = (uint64_t)DS4_N_HC * DS4_N_EMBD;
+    ds4_gpu_tensor *saved_norm = g->output_norm_by_tier[0];
+    g->output_norm_by_tier[0] = norm_dst;
+    bool ok = ds4_gpu_rms_norm_plain_tensor(
+                      metal_graph_flat_hc(g),
+                      metal_graph_cur_hc(g),
+                      (uint32_t)hc_dim,
+                      DS4_RMS_EPS) != 0;
+    if (ok) {
+        ok = ds4_gpu_matmul_f16_tensor(
+                     metal_graph_output_pre(g),
+                     model->map,
+                     model->size,
+                     weights->output_hc_fn->abs_offset,
+                     hc_dim,
+                     DS4_N_HC,
+                     metal_graph_flat_hc(g),
+                     1) != 0;
+    }
+    if (ok) {
+        ok = ds4_gpu_output_hc_weights_tensor(
+                     metal_graph_output_weights(g),
+                     metal_graph_output_pre(g),
+                     model->map,
+                     model->size,
+                     weights->output_hc_scale->abs_offset,
+                     weights->output_hc_base->abs_offset,
+                     DS4_N_HC,
+                     DS4_HC_EPS) != 0;
+    }
+    if (ok) {
+        ok = ds4_gpu_hc_weighted_sum_tensor(
+                     metal_graph_output_embd(g),
+                     metal_graph_cur_hc(g),
+                     metal_graph_output_weights(g),
+                     DS4_N_EMBD,
+                     DS4_N_HC) != 0;
+    }
+    if (ok) {
+        ok = ds4_gpu_rms_norm_weight_tensor(
+                     norm_dst,
+                     metal_graph_output_embd(g),
+                     model->map,
+                     model->size,
+                     weights->output_norm->abs_offset,
+                     DS4_N_EMBD,
+                     DS4_RMS_EPS) != 0;
+    }
+    g->output_norm_by_tier[0] = saved_norm;
+    return ok;
+}
+
+/* Resident exact-N verifier for single-device CUDA.
+ *
+ * This is the N-row form of the canonical exact-2 tape above: every row still
+ * executes the ordinary one-token layers in the same autoregressive order.
+ * By default its output head is one-row too; the Q8-only opt-in preserves the
+ * one-row HC/norm prefix and batches just the exact-row vocab projection. The
+ * remaining batching is lifetime/dispatch batching:
+ * row-local hidden states stay in the existing prefill workspace, all layer
+ * launches share one command stream, and the CPU reads N-1 top ids plus the
+ * final logits after the whole block.  Persistent KV/compressor state is
+ * therefore directly committable only when every draft matches.  The caller
+ * owns a pre-cycle frontier and must restore it on a partial match or error. */
+static bool metal_graph_verify_decode_exactn_cuda_resident_impl(
+        ds4_gpu_graph *g,
+        const ds4_model       *model,
+        const ds4_weights     *weights,
+        const int             *tokens,
+        uint32_t               n_tokens,
+        uint32_t               start,
+        int                   *row_tops,
+        float                 *last_logits,
+        ds4_cuda_exactn_timing *timing) {
+    if (timing) memset(timing, 0, sizeof(*timing));
+    if (!g || !model || !weights || !tokens || !row_tops || !last_logits ||
+        n_tokens < 2u || n_tokens > DS4_CUDA_EXACTN_MAX_ROWS ||
+        n_tokens > g->prefill_cap || g->raw_cap == 0 ||
+        g->ssd_streaming || g->placement != NULL || g->tp_world > 1u ||
+        g->active_tier != 0 || g->emb_tier != 0 || g->head_tier != 0 ||
+        !g->batch_cur_hc_by_tier[0] || !g->batch_next_hc_by_tier[0] ||
+        !g->spec_logits ||
+        ds4_gpu_tensor_bytes(g->spec_logits) <
+            (uint64_t)n_tokens * DS4_N_VOCAB * sizeof(float) ||
+        !g->batch_router_selected_by_tier[0] ||
+        ds4_gpu_tensor_bytes(g->batch_router_selected_by_tier[0]) <
+            (uint64_t)(n_tokens - 1u) * sizeof(int32_t)) {
+        return false;
+    }
+
+    const double setup_t0 = timing ? now_sec() : 0.0;
+    const uint64_t hc_dim = (uint64_t)DS4_N_HC * DS4_N_EMBD;
+    ds4_gpu_tensor *cur_rows[DS4_CUDA_EXACTN_MAX_ROWS] = {0};
+    ds4_gpu_tensor *next_rows[DS4_CUDA_EXACTN_MAX_ROWS] = {0};
+    ds4_gpu_tensor *logits_rows[DS4_CUDA_EXACTN_MAX_ROWS] = {0};
+    ds4_gpu_tensor *top_rows[DS4_CUDA_EXACTN_MAX_ROWS - 1u] = {0};
+    ds4_gpu_tensor *top_span = NULL;
+    ds4_gpu_tensor *saved_cur = g->cur_hc_by_tier[0];
+    ds4_gpu_tensor *saved_after = g->after_ffn_hc_by_tier[0];
+    ds4_gpu_tensor *saved_logits = g->logits_by_tier[0];
+    const bool saved_capture = g->spec_capture_prefixes;
+    const bool saved_disable_decode_graphs =
+        g->spec_disable_decode_graphs;
+    const uint32_t saved_decode_graph_variant =
+        g->decode_graph_variant;
+    const bool try_exactn_graphs =
+        metal_graph_cuda_exactn_graphs_requested(g, n_tokens);
+#if !defined(DS4_NO_GPU) && !defined(__APPLE__) && !defined(DS4_ROCM_BUILD)
+    uint64_t graph_captures_before = 0;
+    uint64_t graph_replays_before = 0;
+    uint64_t graph_warms_before = 0;
+    uint64_t graph_no_slots_before = 0;
+    uint64_t graph_failures_before = 0;
+    if (try_exactn_graphs) {
+        ds4_gpu_decode_graph_counters(&graph_captures_before,
+                                      &graph_replays_before,
+                                      &graph_warms_before,
+                                      &graph_no_slots_before,
+                                      &graph_failures_before);
+    }
+#endif
+    if (timing) timing->graphs_attempted = try_exactn_graphs;
+    bool commands_open = false;
+    bool ok = metal_graph_set_active_tier_no_copy(g, 0);
+
+    /* The generic batch path may swap these two scratch owners.  Exact-N
+     * graph keys have five row slots, so canonicalize their backing order
+     * before creating views; otherwise an intervening odd-layer batch could
+     * invert every address and permanently exhaust the disjoint cache with a
+     * second set of equivalent keys.  Both buffers are scratch and every cur
+     * row is overwritten by the embedding below. */
+    ds4_gpu_tensor *exactn_cur_base = g->batch_cur_hc_by_tier[0];
+    ds4_gpu_tensor *exactn_next_base = g->batch_next_hc_by_tier[0];
+#if !defined(DS4_NO_GPU) && !defined(__APPLE__) && !defined(DS4_ROCM_BUILD)
+    if (try_exactn_graphs &&
+        ds4_gpu_tensor_storage_key(exactn_next_base) <
+            ds4_gpu_tensor_storage_key(exactn_cur_base)) {
+        ds4_gpu_tensor *tmp = exactn_cur_base;
+        exactn_cur_base = exactn_next_base;
+        exactn_next_base = tmp;
+    }
+#endif
+
+    for (uint32_t row = 0; ok && row < n_tokens; row++) {
+        cur_rows[row] = ds4_gpu_tensor_view(
+                exactn_cur_base,
+                (uint64_t)row * hc_dim * sizeof(float),
+                hc_dim * sizeof(float));
+        next_rows[row] = ds4_gpu_tensor_view(
+                exactn_next_base,
+                (uint64_t)row * hc_dim * sizeof(float),
+                hc_dim * sizeof(float));
+        ok = cur_rows[row] && next_rows[row];
+        if (ok) {
+            ok = ds4_gpu_embed_token_hc_tensor(
+                         cur_rows[row],
+                         model->map,
+                         model->size,
+                         weights->token_embd->abs_offset,
+                         (uint32_t)weights->token_embd->dim[1],
+                         (uint32_t)tokens[row],
+                         DS4_N_EMBD,
+                         DS4_N_HC) != 0;
+        }
+    }
+
+    g->spec_capture_prefixes = false;
+    g->spec_disable_decode_graphs =
+        saved_disable_decode_graphs || !try_exactn_graphs;
+    if (timing) timing->setup_ms += (now_sec() - setup_t0) * 1000.0;
+    const double layer_t0 = timing ? now_sec() : 0.0;
+    if (ok) {
+        ok = ds4_gpu_begin_commands() != 0;
+        commands_open = ok;
+    }
+    for (uint32_t il = 0; ok && il < DS4_N_LAYER; il++) {
+        for (uint32_t row = 0; ok && row < n_tokens; row++) {
+            const uint32_t pos = start + row;
+            g->decode_graph_variant =
+                DS4_DECODE_GRAPH_VARIANT_EXACTN | (row + 1u);
+            g->cur_hc_by_tier[0] = cur_rows[row];
+            g->after_ffn_hc_by_tier[0] = next_rows[row];
+            ok = metal_graph_encode_decode_layer(
+                    g,
+                    model,
+                    &weights->layer[il],
+                    il,
+                    pos,
+                    g->layer_raw_cache[il],
+                    g->raw_cap,
+                    pos % g->raw_cap,
+                    metal_graph_raw_span_for_batch(g, pos, 1),
+                    tokens[row]);
+        }
+        if (ok) {
+            for (uint32_t row = 0; row < n_tokens; row++) {
+                ds4_gpu_tensor *tmp = cur_rows[row];
+                cur_rows[row] = next_rows[row];
+                next_rows[row] = tmp;
+            }
+        }
+    }
+    g->decode_graph_variant = saved_decode_graph_variant;
+    if (commands_open) {
+        if (ok) {
+            ok = ds4_gpu_end_commands() != 0;
+        } else {
+            (void)ds4_gpu_synchronize();
+        }
+        commands_open = false;
+    }
+#if !defined(DS4_NO_GPU) && !defined(__APPLE__) && !defined(DS4_ROCM_BUILD)
+    if (try_exactn_graphs && timing) {
+        uint64_t graph_captures_after = 0;
+        uint64_t graph_replays_after = 0;
+        uint64_t graph_warms_after = 0;
+        uint64_t graph_no_slots_after = 0;
+        uint64_t graph_failures_after = 0;
+        ds4_gpu_decode_graph_counters(&graph_captures_after,
+                                      &graph_replays_after,
+                                      &graph_warms_after,
+                                      &graph_no_slots_after,
+                                      &graph_failures_after);
+        timing->graph_captures =
+            graph_captures_after >= graph_captures_before ?
+                graph_captures_after - graph_captures_before : 0;
+        timing->graph_replays =
+            graph_replays_after >= graph_replays_before ?
+                graph_replays_after - graph_replays_before : 0;
+        timing->graph_warms =
+            graph_warms_after >= graph_warms_before ?
+                graph_warms_after - graph_warms_before : 0;
+        timing->graph_no_slots =
+            graph_no_slots_after >= graph_no_slots_before ?
+                graph_no_slots_after - graph_no_slots_before : 0;
+        timing->graph_failures =
+            graph_failures_after >= graph_failures_before ?
+                graph_failures_after - graph_failures_before : 0;
+        timing->graphs_used =
+            timing->graph_captures != 0 || timing->graph_replays != 0;
+    }
+#endif
+    if (timing) timing->layer_ms += (now_sec() - layer_t0) * 1000.0;
+
+    const double head_t0 = timing ? now_sec() : 0.0;
+    if (ok) {
+        top_span = ds4_gpu_tensor_view(
+                g->batch_router_selected_by_tier[0],
+                0,
+                (uint64_t)(n_tokens - 1u) * sizeof(int32_t));
+        ok = top_span != NULL;
+    }
+    for (uint32_t row = 0; ok && row < n_tokens; row++) {
+        logits_rows[row] = ds4_gpu_tensor_view(
+                g->spec_logits,
+                (uint64_t)row * DS4_N_VOCAB * sizeof(float),
+                (uint64_t)DS4_N_VOCAB * sizeof(float));
+        ok = logits_rows[row] != NULL;
+        if (ok && row + 1u < n_tokens) {
+            top_rows[row] = ds4_gpu_tensor_view(
+                    top_span,
+                    (uint64_t)row * sizeof(int32_t),
+                    sizeof(int32_t));
+            ok = top_rows[row] != NULL;
+        }
+    }
+
+    const bool try_batch_head = ok &&
+        metal_graph_cuda_exactn_batch_head_requested(g, weights, n_tokens);
+    bool batch_head_ok = false;
+    if (try_batch_head) {
+        if (timing) timing->batch_head_attempted = true;
+        batch_head_ok = ds4_gpu_begin_commands() != 0;
+        commands_open = batch_head_ok;
+        for (uint32_t row = 0; batch_head_ok && row < n_tokens; row++) {
+            ds4_gpu_tensor *norm_row = ds4_gpu_tensor_view(
+                    metal_graph_batch_ffn_norm(g),
+                    (uint64_t)row * DS4_N_EMBD * sizeof(float),
+                    (uint64_t)DS4_N_EMBD * sizeof(float));
+            g->cur_hc_by_tier[0] = cur_rows[row];
+            batch_head_ok = norm_row &&
+                metal_graph_encode_output_head_norm_exact_row(
+                        g, model, weights, norm_row);
+            ds4_gpu_tensor_free(norm_row);
+        }
+        if (batch_head_ok) {
+            batch_head_ok =
+                ds4_gpu_matmul_q8_0_decode_rows_exact_tensor(
+                        g->spec_logits,
+                        model->map,
+                        model->size,
+                        weights->output->abs_offset,
+                        DS4_N_EMBD,
+                        DS4_N_VOCAB,
+                        metal_graph_batch_ffn_norm(g),
+                        n_tokens) != 0;
+        }
+        for (uint32_t row = 0;
+             batch_head_ok && row + 1u < n_tokens;
+             row++) {
+            batch_head_ok = ds4_gpu_argmax_tensor(
+                    top_rows[row], logits_rows[row], DS4_N_VOCAB) != 0;
+        }
+        if (commands_open) {
+            if (batch_head_ok) {
+                batch_head_ok = ds4_gpu_end_commands() != 0;
+                if (!batch_head_ok) (void)ds4_gpu_synchronize();
+            } else {
+                (void)ds4_gpu_synchronize();
+            }
+            commands_open = false;
+        }
+        if (timing) {
+            timing->batch_head_used = batch_head_ok;
+            timing->batch_head_fallback = !batch_head_ok;
+        }
+    }
+
+    /* An ineligible or failed experimental head always returns to the proven
+     * one-row sequence.  Head scratch is non-persistent, so retrying does not
+     * alter the verifier frontier. */
+    if (ok && !batch_head_ok) {
+        ok = ds4_gpu_begin_commands() != 0;
+        commands_open = ok;
+        for (uint32_t row = 0; ok && row < n_tokens; row++) {
+            g->cur_hc_by_tier[0] = cur_rows[row];
+            g->logits_by_tier[0] = logits_rows[row];
+            ok = metal_graph_encode_output_head(
+                    g, model, weights, weights->output->dim[1]);
+            if (ok && row + 1u < n_tokens) {
+                ok = ds4_gpu_argmax_tensor(
+                             top_rows[row], logits_rows[row],
+                             DS4_N_VOCAB) != 0;
+            }
+        }
+        if (commands_open) {
+            if (ok) {
+                ok = ds4_gpu_end_commands() != 0;
+            } else {
+                (void)ds4_gpu_synchronize();
+            }
+            commands_open = false;
+        }
+    }
+    if (timing) timing->head_ms += (now_sec() - head_t0) * 1000.0;
+
+    int32_t tops_i32[DS4_CUDA_EXACTN_MAX_ROWS - 1u] = {0};
+    const double read_t0 = timing ? now_sec() : 0.0;
+    if (ok) {
+        ok = ds4_gpu_tensor_read(
+                     top_span,
+                     0,
+                     tops_i32,
+                     (uint64_t)(n_tokens - 1u) * sizeof(tops_i32[0])) != 0 &&
+             ds4_gpu_tensor_read(
+                     logits_rows[n_tokens - 1u],
+                     0,
+                     last_logits,
+                     (uint64_t)DS4_N_VOCAB * sizeof(last_logits[0])) != 0;
+    }
+    if (ok) {
+        for (uint32_t row = 0; row + 1u < n_tokens; row++) {
+            row_tops[row] = tops_i32[row];
+        }
+    }
+    if (timing) timing->read_ms += (now_sec() - read_t0) * 1000.0;
+
+    g->spec_capture_prefixes = saved_capture;
+    g->spec_disable_decode_graphs = saved_disable_decode_graphs;
+    g->decode_graph_variant = saved_decode_graph_variant;
+    g->cur_hc_by_tier[0] = saved_cur;
+    g->after_ffn_hc_by_tier[0] = saved_after;
+    g->logits_by_tier[0] = saved_logits;
+    for (uint32_t row = 0; row + 1u < n_tokens; row++) {
+        ds4_gpu_tensor_free(top_rows[row]);
+    }
+    ds4_gpu_tensor_free(top_span);
+    for (uint32_t row = 0; row < n_tokens; row++) {
+        ds4_gpu_tensor_free(logits_rows[row]);
+        ds4_gpu_tensor_free(next_rows[row]);
+        ds4_gpu_tensor_free(cur_rows[row]);
+    }
+    return ok;
+}
+
+enum { DS4_METAL_EXACTN_UNION_MAX_ROWS = 5 };
+
+typedef struct {
+    bool batch_head_attempted;
+    bool batch_head_used;
+    bool batch_head_fallback;
+} ds4_metal_exactn_union_timing;
+
+static DS4_MAYBE_UNUSED bool metal_graph_metal_exactn_batch_head_requested(
+        const ds4_gpu_graph *g,
+        const ds4_weights   *weights,
+        uint32_t             n_tokens) {
+#if defined(__APPLE__)
+    return g && weights && weights->output &&
+           n_tokens >= 2u &&
+           n_tokens <= DS4_METAL_EXACTN_UNION_MAX_ROWS &&
+           n_tokens <= g->prefill_cap &&
+           g->ssd_streaming && g->placement == NULL &&
+           g->tp_world <= 1u && g->active_tier == 0 &&
+           g->head_tier == 0 &&
+           weights->output->type == DS4_TENSOR_Q8_0 &&
+           weights->output->ndim == 2 &&
+           weights->output->dim[0] == DS4_N_EMBD &&
+           weights->output->dim[1] == DS4_N_VOCAB &&
+           metal_graph_batch_ffn_norm(g) && g->spec_logits &&
+           ds4_gpu_tensor_bytes(metal_graph_batch_ffn_norm(g)) >=
+               (uint64_t)n_tokens * DS4_N_EMBD * sizeof(float) &&
+           ds4_gpu_tensor_bytes(g->spec_logits) >=
+               (uint64_t)n_tokens * DS4_N_VOCAB * sizeof(float) &&
+           metal_graph_tp_env_flag(
+                   "DS4_METAL_DSPARK_EXACTN_BATCH_HEAD", false) &&
+           !metal_graph_tp_env_flag(
+                   "DS4_METAL_DISABLE_DSPARK_EXACTN_BATCH_HEAD", false);
+#else
+    (void)g;
+    (void)weights;
+    (void)n_tokens;
+    return false;
+#endif
+}
+
+/* Row-local aliases needed across the TO_ROUTER/FROM_ROUTER split.  The
+ * ordinary decode tape intentionally reuses its Class-P tensors at every
+ * dispatch.  That is safe for a complete row, but not when N router prefixes
+ * are encoded before any routed tail consumes them.  These views borrow the
+ * existing prefill workspace, so no new resident allocation is needed. */
+typedef struct {
+    int tier;
+    bool active;
+    ds4_gpu_tensor *after_attn_hc;
+    ds4_gpu_tensor *ffn_cur;
+    ds4_gpu_tensor *ffn_norm;
+    ds4_gpu_tensor *hc_split;
+    ds4_gpu_tensor *hc_pre;
+    ds4_gpu_tensor *hc_post;
+    ds4_gpu_tensor *hc_comb;
+    ds4_gpu_tensor *router_selected;
+    ds4_gpu_tensor *router_weights;
+    ds4_gpu_tensor *saved_cur_hc;
+    ds4_gpu_tensor *saved_after_ffn_hc;
+    ds4_gpu_tensor *saved_after_attn_hc;
+    ds4_gpu_tensor *saved_ffn_cur;
+    ds4_gpu_tensor *saved_ffn_norm;
+    ds4_gpu_tensor *saved_hc_split;
+    ds4_gpu_tensor *saved_hc_pre;
+    ds4_gpu_tensor *saved_hc_post;
+    ds4_gpu_tensor *saved_hc_comb;
+    ds4_gpu_tensor *saved_router_selected;
+    ds4_gpu_tensor *saved_router_weights;
+} metal_graph_exactn_union_row_alias;
+
+static void metal_graph_release_exactn_union_row_alias(
+        ds4_gpu_graph                    *g,
+        metal_graph_exactn_union_row_alias *alias) {
+    if (!alias) return;
+    if (g && alias->active) {
+        const int t = alias->tier;
+        g->cur_hc_by_tier[t] = alias->saved_cur_hc;
+        g->after_ffn_hc_by_tier[t] = alias->saved_after_ffn_hc;
+        g->after_attn_hc_by_tier[t] = alias->saved_after_attn_hc;
+        g->ffn_cur_by_tier[t] = alias->saved_ffn_cur;
+        g->ffn_norm_by_tier[t] = alias->saved_ffn_norm;
+        g->hc_split_by_tier[t] = alias->saved_hc_split;
+        g->hc_pre_by_tier[t] = alias->saved_hc_pre;
+        g->hc_post_by_tier[t] = alias->saved_hc_post;
+        g->hc_comb_by_tier[t] = alias->saved_hc_comb;
+        g->router_selected_by_tier[t] = alias->saved_router_selected;
+        g->router_weights_by_tier[t] = alias->saved_router_weights;
+    }
+    ds4_gpu_tensor_free(alias->router_weights);
+    ds4_gpu_tensor_free(alias->router_selected);
+    ds4_gpu_tensor_free(alias->hc_comb);
+    ds4_gpu_tensor_free(alias->hc_post);
+    ds4_gpu_tensor_free(alias->hc_pre);
+    ds4_gpu_tensor_free(alias->hc_split);
+    ds4_gpu_tensor_free(alias->ffn_norm);
+    ds4_gpu_tensor_free(alias->ffn_cur);
+    ds4_gpu_tensor_free(alias->after_attn_hc);
+    memset(alias, 0, sizeof(*alias));
+}
+
+static DS4_MAYBE_UNUSED bool metal_graph_bind_exactn_union_row(
+        ds4_gpu_graph                    *g,
+        const ds4_gpu_graph              *workspace,
+        uint32_t                          row,
+        ds4_gpu_tensor                   *cur_hc,
+        ds4_gpu_tensor                   *next_hc,
+        metal_graph_exactn_union_row_alias *alias) {
+    if (!alias) return false;
+    if (!g || !workspace || !cur_hc || !next_hc ||
+        g->active_tier < 0 || workspace->active_tier != g->active_tier ||
+        row >= workspace->prefill_cap || alias->active) {
+        return false;
+    }
+    if (!alias->after_attn_hc) {
+        memset(alias, 0, sizeof(*alias));
+        alias->tier = g->active_tier;
+        const uint64_t hc_dim = (uint64_t)DS4_N_HC * DS4_N_EMBD;
+        const uint64_t mix_hc = 2ull * DS4_N_HC +
+                                (uint64_t)DS4_N_HC * DS4_N_HC;
+        alias->after_attn_hc = ds4_gpu_tensor_view(
+                    metal_graph_batch_after_attn_hc(workspace),
+                    (uint64_t)row * hc_dim * sizeof(float),
+                    hc_dim * sizeof(float));
+        alias->ffn_cur = ds4_gpu_tensor_view(
+                    metal_graph_batch_ffn_cur(workspace),
+                    (uint64_t)row * DS4_N_EMBD * sizeof(float),
+                    (uint64_t)DS4_N_EMBD * sizeof(float));
+        alias->ffn_norm = ds4_gpu_tensor_view(
+                    metal_graph_batch_ffn_norm(workspace),
+                    (uint64_t)row * DS4_N_EMBD * sizeof(float),
+                    (uint64_t)DS4_N_EMBD * sizeof(float));
+        alias->hc_split = ds4_gpu_tensor_view(
+                    metal_graph_batch_hc_split(workspace),
+                    (uint64_t)row * mix_hc * sizeof(float),
+                    mix_hc * sizeof(float));
+        alias->hc_pre = ds4_gpu_tensor_view(
+                    alias->hc_split,
+                    0,
+                    (uint64_t)DS4_N_HC * sizeof(float));
+        alias->hc_post = ds4_gpu_tensor_view(
+                    alias->hc_split,
+                    (uint64_t)DS4_N_HC * sizeof(float),
+                    (uint64_t)DS4_N_HC * sizeof(float));
+        alias->hc_comb = ds4_gpu_tensor_view(
+                    alias->hc_split,
+                    2ull * DS4_N_HC * sizeof(float),
+                    (uint64_t)DS4_N_HC * DS4_N_HC * sizeof(float));
+        alias->router_selected = ds4_gpu_tensor_view(
+                    metal_graph_batch_router_selected(workspace),
+                    (uint64_t)row * DS4_N_EXPERT_USED * sizeof(int32_t),
+                    (uint64_t)DS4_N_EXPERT_USED * sizeof(int32_t));
+        alias->router_weights = ds4_gpu_tensor_view(
+                    metal_graph_batch_router_weights(workspace),
+                    (uint64_t)row * DS4_N_EXPERT_USED * sizeof(float),
+                    (uint64_t)DS4_N_EXPERT_USED * sizeof(float));
+    }
+    if (!alias->after_attn_hc || !alias->ffn_cur || !alias->ffn_norm ||
+        !alias->hc_split || !alias->hc_pre || !alias->hc_post ||
+        !alias->hc_comb || !alias->router_selected ||
+        !alias->router_weights || alias->tier != g->active_tier) {
+        metal_graph_release_exactn_union_row_alias(g, alias);
+        return false;
+    }
+
+    const int t = alias->tier;
+    alias->saved_cur_hc = g->cur_hc_by_tier[t];
+    alias->saved_after_ffn_hc = g->after_ffn_hc_by_tier[t];
+    alias->saved_after_attn_hc = g->after_attn_hc_by_tier[t];
+    alias->saved_ffn_cur = g->ffn_cur_by_tier[t];
+    alias->saved_ffn_norm = g->ffn_norm_by_tier[t];
+    alias->saved_hc_split = g->hc_split_by_tier[t];
+    alias->saved_hc_pre = g->hc_pre_by_tier[t];
+    alias->saved_hc_post = g->hc_post_by_tier[t];
+    alias->saved_hc_comb = g->hc_comb_by_tier[t];
+    alias->saved_router_selected = g->router_selected_by_tier[t];
+    alias->saved_router_weights = g->router_weights_by_tier[t];
+    g->cur_hc_by_tier[t] = cur_hc;
+    g->after_ffn_hc_by_tier[t] = next_hc;
+    g->after_attn_hc_by_tier[t] = alias->after_attn_hc;
+    g->ffn_cur_by_tier[t] = alias->ffn_cur;
+    g->ffn_norm_by_tier[t] = alias->ffn_norm;
+    g->hc_split_by_tier[t] = alias->hc_split;
+    g->hc_pre_by_tier[t] = alias->hc_pre;
+    g->hc_post_by_tier[t] = alias->hc_post;
+    g->hc_comb_by_tier[t] = alias->hc_comb;
+    g->router_selected_by_tier[t] = alias->router_selected;
+    g->router_weights_by_tier[t] = alias->router_weights;
+    alias->active = true;
+    return true;
+}
+
+static DS4_MAYBE_UNUSED void metal_graph_unbind_exactn_union_row(
+        ds4_gpu_graph                    *g,
+        metal_graph_exactn_union_row_alias *alias) {
+    if (!g || !alias || !alias->active) return;
+    const int t = alias->tier;
+    g->cur_hc_by_tier[t] = alias->saved_cur_hc;
+    g->after_ffn_hc_by_tier[t] = alias->saved_after_ffn_hc;
+    g->after_attn_hc_by_tier[t] = alias->saved_after_attn_hc;
+    g->ffn_cur_by_tier[t] = alias->saved_ffn_cur;
+    g->ffn_norm_by_tier[t] = alias->saved_ffn_norm;
+    g->hc_split_by_tier[t] = alias->saved_hc_split;
+    g->hc_pre_by_tier[t] = alias->saved_hc_pre;
+    g->hc_post_by_tier[t] = alias->saved_hc_post;
+    g->hc_comb_by_tier[t] = alias->saved_hc_comb;
+    g->router_selected_by_tier[t] = alias->saved_router_selected;
+    g->router_weights_by_tier[t] = alias->saved_router_weights;
+    alias->active = false;
+}
+
+/* Exact speculative Metal verifier with one selected-expert union per layer.
+ *
+ * Attention and router prefixes remain canonical one-token decode dispatches,
+ * in autoregressive row order.  A hard router readback boundary then lets the
+ * backend load the union of N x top-k experts once and expose immutable
+ * per-row address tables to the canonical routed tails.  No command boundary
+ * is permitted between set_row() and routed MoE, or between routed rows.
+ *
+ * The helper only reports acceptance.  Callers own the pre-cycle frontier and
+ * must restore it on a partial match or backend error. */
+static bool metal_graph_verify_decode_exactn_union_impl(
+        ds4_gpu_graph *g,
+        const ds4_model       *model,
+        const ds4_weights     *weights,
+        const int             *tokens,
+        uint32_t               n_tokens,
+        uint32_t               start,
+        int                   *row_tops,
+        float                 *last_logits,
+        ds4_metal_exactn_union_timing *timing) {
+#if defined(__APPLE__)
+    if (timing) memset(timing, 0, sizeof(*timing));
+    const bool exact_rows_profile =
+        getenv("DS4_METAL_DSPARK_EXACT_ROWS_PROFILE") != NULL;
+    const bool async_exact_rows_tails =
+        getenv("DS4_METAL_DSPARK_EXACT_ROWS_ASYNC_TAILS") != NULL;
+    if (!g || !model || !weights || !tokens || !row_tops || !last_logits ||
+        n_tokens < 2u || n_tokens > DS4_METAL_EXACTN_UNION_MAX_ROWS ||
+        n_tokens > g->prefill_cap || g->raw_cap == 0 ||
+        !g->ssd_streaming || g->placement != NULL || g->tp_world > 1u ||
+        g->active_tier != 0 || g->emb_tier != 0 || g->head_tier != 0 ||
+        g->decode_stage_profile || g_expert_profile.active ||
+        metal_graph_debug_get_config()->prefix != NULL ||
+        metal_graph_hc_norm_fusion_check_enabled() ||
+        g->spec_exactn_union_collect_routes ||
+        getenv("DS4_TP_ABLATE") != NULL ||
+        getenv("DS4_METAL_SELECTED_PROFILE") != NULL ||
+        getenv("DS4_METAL_Q4_SELECTED_PROFILE") != NULL ||
+        !g->spec_logits ||
+        ds4_gpu_tensor_bytes(g->spec_logits) <
+            (uint64_t)n_tokens * DS4_N_VOCAB * sizeof(float)) {
+        if (exact_rows_profile) {
+            fprintf(stderr,
+                    "ds4: Metal exact-row union preflight rejected "
+                    "rows=%u prefill=%u raw=%u ssd=%u placement=%u tp=%u "
+                    "active=%d emb=%d head=%d spec_logits=%llu\n",
+                    n_tokens,
+                    g ? g->prefill_cap : 0u,
+                    g ? g->raw_cap : 0u,
+                    g ? (unsigned)g->ssd_streaming : 0u,
+                    g && g->placement ? 1u : 0u,
+                    g ? g->tp_world : 0u,
+                    g ? g->active_tier : -1,
+                    g ? g->emb_tier : -1,
+                    g ? g->head_tier : -1,
+                    (unsigned long long)(g && g->spec_logits
+                        ? ds4_gpu_tensor_bytes(g->spec_logits) : 0u));
+        }
+        return false;
+    }
+    for (uint32_t il = 0; il < DS4_N_LAYER; il++) {
+        const ds4_layer_weights *layer = &weights->layer[il];
+        if (!weights_layer_has_required(layer, il) ||
+            !metal_graph_decode_iq2_selected_slots_expected(g, layer) ||
+            metal_graph_decode_cpu_router_applicable(g, layer)) {
+            if (exact_rows_profile) {
+                fprintf(stderr,
+                        "ds4: Metal exact-row union layout rejected layer=%u "
+                        "required=%u iq2_slots=%u cpu_router=%u hash_router=%u\n",
+                        il,
+                        (unsigned)weights_layer_has_required(layer, il),
+                        (unsigned)metal_graph_decode_iq2_selected_slots_expected(
+                                g, layer),
+                        (unsigned)metal_graph_decode_cpu_router_applicable(
+                                g, layer),
+                        layer->ffn_gate_tid2eid != NULL ? 1u : 0u);
+            }
+            return false;
+        }
+    }
+
+    const uint64_t hc_dim = (uint64_t)DS4_N_HC * DS4_N_EMBD;
+    ds4_gpu_tensor *cur_rows[DS4_METAL_EXACTN_UNION_MAX_ROWS] = {0};
+    ds4_gpu_tensor *next_rows[DS4_METAL_EXACTN_UNION_MAX_ROWS] = {0};
+    metal_graph_exactn_union_row_alias
+        row_aliases[DS4_METAL_EXACTN_UNION_MAX_ROWS] = {0};
+    bool ok = true;
+    for (uint32_t row = 0; ok && row < n_tokens; row++) {
+        cur_rows[row] = ds4_gpu_tensor_view(
+                metal_graph_batch_cur_hc(g),
+                (uint64_t)row * hc_dim * sizeof(float),
+                hc_dim * sizeof(float));
+        next_rows[row] = ds4_gpu_tensor_view(
+                metal_graph_batch_next_hc(g),
+                (uint64_t)row * hc_dim * sizeof(float),
+                hc_dim * sizeof(float));
+        ok = cur_rows[row] && next_rows[row];
+    }
+
+    ds4_gpu_tensor *selected_rows = NULL;
+    if (ok) {
+        selected_rows = ds4_gpu_tensor_view(
+                metal_graph_batch_router_selected(g),
+                0,
+                (uint64_t)n_tokens * DS4_N_EXPERT_USED * sizeof(int32_t));
+        ok = selected_rows != NULL;
+    }
+
+    /* Embeddings are independent rows and can share the same command batch;
+     * the layer tape below never uses the ordinary cur_hc allocation. */
+    if (ok) ok = ds4_gpu_begin_commands() != 0;
+    bool commands_open = ok;
+    for (uint32_t row = 0; ok && row < n_tokens; row++) {
+        ok = ds4_gpu_embed_token_hc_tensor(
+                     cur_rows[row],
+                     model->map,
+                     model->size,
+                     weights->token_embd->abs_offset,
+                     (uint32_t)weights->token_embd->dim[1],
+                     (uint32_t)tokens[row],
+                     DS4_N_EMBD,
+                     DS4_N_HC) != 0;
+    }
+
+    const bool saved_capture = g->spec_capture_prefixes;
+    g->spec_capture_prefixes = false;
+    bool exact_rows_active = false;
+    bool exact_rows_collecting = false;
+    for (uint32_t il = 0; ok && il < DS4_N_LAYER; il++) {
+        const ds4_layer_weights *layer = &weights->layer[il];
+        if (ok) {
+            ok = ds4_gpu_stream_expert_exact_rows_begin_collect() != 0;
+            exact_rows_collecting = ok;
+        }
+        g->spec_exactn_union_collect_routes = ok;
+        for (uint32_t row = 0; ok && row < n_tokens; row++) {
+            const uint32_t pos = start + row;
+            ok = metal_graph_bind_exactn_union_row(
+                    g, g, row, cur_rows[row], next_rows[row],
+                    &row_aliases[row]);
+            if (ok) {
+                ok = metal_graph_encode_decode_layer_phase(
+                        g,
+                        model,
+                        layer,
+                        il,
+                        pos,
+                        g->layer_raw_cache[il],
+                        g->raw_cap,
+                        pos % g->raw_cap,
+                        metal_graph_raw_span_for_batch(g, pos, 1),
+                        tokens[row],
+                        METAL_DECODE_LAYER_TO_ROUTER);
+            }
+            metal_graph_unbind_exactn_union_row(g, &row_aliases[row]);
+        }
+        g->spec_exactn_union_collect_routes = false;
+
+        const uint64_t gate_row_bytes =
+            routed_expert_row_bytes(layer->ffn_gate_exps);
+        const uint64_t down_row_bytes =
+            routed_expert_row_bytes(layer->ffn_down_exps);
+        if (ok &&
+            (gate_row_bytes == 0 || down_row_bytes == 0 ||
+             layer->ffn_gate_exps->dim[1] > UINT64_MAX / gate_row_bytes ||
+             layer->ffn_down_exps->dim[1] > UINT64_MAX / down_row_bytes)) {
+            ok = false;
+        }
+        const uint64_t gate_expert_bytes = ok
+            ? layer->ffn_gate_exps->dim[1] * gate_row_bytes : 0;
+        const uint64_t down_expert_bytes = ok
+            ? layer->ffn_down_exps->dim[1] * down_row_bytes : 0;
+        if (ok) {
+            const ds4_gpu_stream_expert_table table =
+                graph_stream_expert_table_make(model,
+                                               layer,
+                                               il,
+                                               gate_expert_bytes,
+                                               down_expert_bytes);
+            ok = ds4_gpu_stream_expert_exact_rows_prepare(
+                         &table,
+                         selected_rows,
+                         n_tokens,
+                         DS4_N_EXPERT_USED) != 0;
+            exact_rows_active = ok;
+        }
+
+        for (uint32_t row = 0; ok && row < n_tokens; row++) {
+            const uint32_t pos = start + row;
+            ok = metal_graph_bind_exactn_union_row(
+                    g, g, row, cur_rows[row], next_rows[row],
+                    &row_aliases[row]);
+            if (ok) {
+                ok = ds4_gpu_stream_expert_exact_rows_set_row(row) != 0;
+            }
+            if (ok) {
+                ok = metal_graph_encode_decode_layer_phase(
+                        g,
+                        model,
+                        layer,
+                        il,
+                        pos,
+                        g->layer_raw_cache[il],
+                        g->raw_cap,
+                        pos % g->raw_cap,
+                        metal_graph_raw_span_for_batch(g, pos, 1),
+                        tokens[row],
+                        METAL_DECODE_LAYER_FROM_ROUTER);
+            }
+            metal_graph_unbind_exactn_union_row(g, &row_aliases[row]);
+        }
+        /* The exact-row scope owns the private address buffers referenced by
+         * every routed tail.  The default path drains that command buffer
+         * before dropping its strong refs.  The opt-in path commits without a
+         * CPU wait and retains the whole scope through CB completion; queue
+         * order makes the next layer's router-readback event cover these
+         * earlier tails as well. */
+        if (exact_rows_active) {
+            if (commands_open) {
+                if (ok) {
+                    const bool end_ok = async_exact_rows_tails
+                        ? ds4_gpu_stream_expert_exact_rows_end_async() != 0
+                        : ds4_gpu_end_commands() != 0;
+                    if (!end_ok) (void)ds4_gpu_synchronize();
+                    ok = end_ok;
+                } else {
+                    (void)ds4_gpu_end_commands();
+                    (void)ds4_gpu_synchronize();
+                }
+                commands_open = false;
+            }
+        }
+        if (exact_rows_collecting) {
+            ds4_gpu_stream_expert_exact_rows_release();
+            exact_rows_collecting = false;
+            exact_rows_active = false;
+        }
+        if (ok) {
+            for (uint32_t row = 0; row < n_tokens; row++) {
+                ds4_gpu_tensor *tmp = cur_rows[row];
+                cur_rows[row] = next_rows[row];
+                next_rows[row] = tmp;
+            }
+            if (il + 1u < DS4_N_LAYER) {
+                ok = ds4_gpu_begin_commands() != 0;
+                commands_open = ok;
+            }
+        }
+    }
+    g->spec_exactn_union_collect_routes = false;
+    if (exact_rows_active) {
+        if (commands_open) {
+            (void)ds4_gpu_end_commands();
+            (void)ds4_gpu_synchronize();
+            commands_open = false;
+        }
+    }
+    if (exact_rows_collecting) {
+        ds4_gpu_stream_expert_exact_rows_release();
+        exact_rows_collecting = false;
+        exact_rows_active = false;
+    }
+    if (commands_open) {
+        if (ok) {
+            ok = ds4_gpu_end_commands() != 0;
+        } else {
+            (void)ds4_gpu_synchronize();
+        }
+        commands_open = false;
+    }
+    g->spec_capture_prefixes = saved_capture;
+
+    /* Preserve every output row in spec_logits so all canonical output-head
+     * dispatches and argmax reductions can remain in one command batch. */
+    ds4_gpu_tensor *logits_rows[DS4_METAL_EXACTN_UNION_MAX_ROWS] = {0};
+    ds4_gpu_tensor *top_rows[DS4_METAL_EXACTN_UNION_MAX_ROWS - 1u] = {0};
+    ds4_gpu_tensor *top_span = NULL;
+    if (ok) {
+        top_span = ds4_gpu_tensor_view(
+                metal_graph_batch_router_selected(g),
+                0,
+                (uint64_t)(n_tokens - 1u) * sizeof(int32_t));
+        ok = top_span != NULL;
+    }
+    for (uint32_t row = 0; ok && row < n_tokens; row++) {
+        logits_rows[row] = ds4_gpu_tensor_view(
+                g->spec_logits,
+                (uint64_t)row * DS4_N_VOCAB * sizeof(float),
+                (uint64_t)DS4_N_VOCAB * sizeof(float));
+        ok = logits_rows[row] != NULL;
+        if (ok && row + 1u < n_tokens) {
+            top_rows[row] = ds4_gpu_tensor_view(
+                    top_span,
+                    (uint64_t)row * sizeof(int32_t),
+                    sizeof(int32_t));
+            ok = top_rows[row] != NULL;
+        }
+    }
+
+    ds4_gpu_tensor *saved_cur = g->cur_hc_by_tier[0];
+    ds4_gpu_tensor *saved_after = g->after_ffn_hc_by_tier[0];
+    ds4_gpu_tensor *saved_logits = g->logits_by_tier[0];
+    const bool try_batch_head = ok &&
+        metal_graph_metal_exactn_batch_head_requested(
+                g, weights, n_tokens);
+    bool batch_head_ok = false;
+    if (try_batch_head) {
+        if (timing) timing->batch_head_attempted = true;
+        batch_head_ok = ds4_gpu_begin_commands() != 0;
+        commands_open = batch_head_ok;
+        for (uint32_t row = 0;
+             batch_head_ok && row < n_tokens;
+             row++) {
+            ds4_gpu_tensor *norm_row = ds4_gpu_tensor_view(
+                    metal_graph_batch_ffn_norm(g),
+                    (uint64_t)row * DS4_N_EMBD * sizeof(float),
+                    (uint64_t)DS4_N_EMBD * sizeof(float));
+            g->cur_hc_by_tier[0] = cur_rows[row];
+            batch_head_ok = norm_row &&
+                metal_graph_encode_output_head_norm_exact_row(
+                        g, model, weights, norm_row);
+            ds4_gpu_tensor_free(norm_row);
+        }
+        if (batch_head_ok) {
+            batch_head_ok =
+                ds4_gpu_matmul_q8_0_decode_rows_exact_tensor(
+                        g->spec_logits,
+                        model->map,
+                        model->size,
+                        weights->output->abs_offset,
+                        DS4_N_EMBD,
+                        DS4_N_VOCAB,
+                        metal_graph_batch_ffn_norm(g),
+                        n_tokens) != 0;
+        }
+        for (uint32_t row = 0;
+             batch_head_ok && row + 1u < n_tokens;
+             row++) {
+            batch_head_ok = ds4_gpu_argmax_tensor(
+                    top_rows[row], logits_rows[row], DS4_N_VOCAB) != 0;
+        }
+        g->cur_hc_by_tier[0] = saved_cur;
+        g->after_ffn_hc_by_tier[0] = saved_after;
+        g->logits_by_tier[0] = saved_logits;
+        if (commands_open) {
+            if (batch_head_ok) {
+                batch_head_ok = ds4_gpu_end_commands() != 0;
+                if (!batch_head_ok) (void)ds4_gpu_synchronize();
+            } else {
+                (void)ds4_gpu_synchronize();
+            }
+            commands_open = false;
+        }
+        if (timing) {
+            timing->batch_head_used = batch_head_ok;
+            timing->batch_head_fallback = !batch_head_ok;
+        }
+    }
+
+    /* The experimental multi-row projection only touches head scratch.  A
+     * failed or ineligible attempt can therefore retry the established
+     * one-row sequence without restoring the target frontier. */
+    if (ok && !batch_head_ok) {
+        ok = ds4_gpu_begin_commands() != 0;
+        commands_open = ok;
+        for (uint32_t row = 0; ok && row < n_tokens; row++) {
+            g->cur_hc_by_tier[0] = cur_rows[row];
+            g->logits_by_tier[0] = logits_rows[row];
+            ok = metal_graph_encode_output_head(
+                    g, model, weights, weights->output->dim[1]);
+            if (ok && row + 1u < n_tokens) {
+                ok = ds4_gpu_argmax_tensor(top_rows[row],
+                                           logits_rows[row],
+                                           DS4_N_VOCAB) != 0;
+            }
+        }
+        g->cur_hc_by_tier[0] = saved_cur;
+        g->after_ffn_hc_by_tier[0] = saved_after;
+        g->logits_by_tier[0] = saved_logits;
+        if (commands_open) {
+            if (ok) {
+                ok = ds4_gpu_end_commands() != 0;
+            } else {
+                (void)ds4_gpu_synchronize();
+            }
+            commands_open = false;
+        }
+    }
+
+    int32_t tops_i32[DS4_METAL_EXACTN_UNION_MAX_ROWS - 1u] = {0};
+    if (ok) {
+        ok = ds4_gpu_tensor_read(
+                     top_span,
+                     0,
+                     tops_i32,
+                     (uint64_t)(n_tokens - 1u) * sizeof(tops_i32[0])) != 0 &&
+             ds4_gpu_tensor_read(
+                     logits_rows[n_tokens - 1u],
+                     0,
+                     last_logits,
+                     (uint64_t)DS4_N_VOCAB * sizeof(last_logits[0])) != 0;
+    }
+    if (ok) {
+        for (uint32_t row = 0; row + 1u < n_tokens; row++) {
+            row_tops[row] = tops_i32[row];
+        }
+    }
+    for (uint32_t row = 0; row + 1u < n_tokens; row++) {
+        ds4_gpu_tensor_free(top_rows[row]);
+    }
+    ds4_gpu_tensor_free(top_span);
+    for (uint32_t row = 0; row < n_tokens; row++) {
+        ds4_gpu_tensor_free(logits_rows[row]);
+    }
+    ds4_gpu_tensor_free(selected_rows);
+    for (uint32_t row = 0; row < n_tokens; row++) {
+        metal_graph_release_exactn_union_row_alias(
+                g, &row_aliases[row]);
+        ds4_gpu_tensor_free(next_rows[row]);
+        ds4_gpu_tensor_free(cur_rows[row]);
+    }
+    return ok;
+#else
+    (void)g;
+    (void)model;
+    (void)weights;
+    (void)tokens;
+    (void)n_tokens;
+    (void)start;
+    (void)row_tops;
+    (void)last_logits;
+    (void)timing;
+    return false;
+#endif
+}
+
+static bool metal_graph_verify_decode2_exact(
+        ds4_gpu_graph *g,
+        const ds4_model       *model,
+        const ds4_weights     *weights,
+        int                    token0,
+        int                    token1,
+        uint32_t               start,
+        int                   *top0,
+        int                   *top1,
+        float                 *logits0,
+        float                 *logits1) {
+    return metal_graph_verify_decode2_exact_impl(g,
+                                                  model,
+                                                  weights,
+                                                  token0,
+                                                  token1,
+                                                  start,
+                                                  true,
+                                                  -1,
+                                                  top0,
+                                                  top1,
+                                                  logits0,
+                                                  logits1);
+}
+
 /* Pick a raw SWA cache size for Metal.  During batched prefill it must cover
  * the previous window plus the current ubatch. */
 static uint32_t metal_graph_raw_cap_for_context(int ctx_size, uint32_t prefill_cap) {
@@ -37598,8 +40485,6 @@ static uint32_t glm53_graph_resume_prefill_min_tokens(void) {
 #define DS4_GLM_METAL_FULL_ATTN_DEFAULT_CONTEXT 4096u
 #define DS4_GLM_METAL_STREAMING_FULL_ATTN_CONTEXT 8192u
 #define DS4_GLM_METAL_FULL_ATTN_LAYER_FLUSH_CONTEXT 2048u
-#define DS4_GLM_METAL_DISPLAY_PROGRESS_LAYER_TOKENS 32u
-#define DS4_GLM_METAL_SMALL_PREFILL_STAGE_SYNC_TOKENS 0u
 #define DS4_GLM_METAL_LONG_CONTEXT_THRESHOLD 65536u
 #define DS4_GLM_METAL_LONG_CONTEXT_FULL_ATTN_CONTEXT 4096u
 #define DS4_GLM_METAL_INDEXED_PREFILL_CHUNK_TOKENS 4096u
@@ -38711,6 +41596,10 @@ struct ds4_engine {
     bool vision_ready;
     bool vision_map_ready;
     bool share_session_prefill_workspace;
+#if defined(__APPLE__) && !defined(DS4_NO_GPU)
+    void *dspark_hot_lock_addr;
+    size_t dspark_hot_lock_len;
+#endif
 #ifndef DS4_NO_GPU
     bool shared_prefill_workspace_ready;
     ds4_gpu_graph shared_prefill_workspace;
@@ -38733,8 +41622,282 @@ struct ds4_engine {
      * caller that doesn't set the option observe the prior behavior). */
     int            placement_ctx_hint;
     int            placement_session_count_hint;
+    uint32_t       live_session_count;
 };
 
+#if defined(__APPLE__) && !defined(DS4_NO_GPU)
+/* The first DSpark projection is reused by every proposal and, under Metal
+ * SSD streaming, otherwise remains a pageable no-copy view of the support
+ * GGUF.  This opt-in lock keeps only stage-0 main_norm + main_proj hot instead
+ * of pinning the full 5.6 GiB support model.  Failure is deliberately
+ * non-fatal: the existing pageable path remains correct. */
+static void ds4_engine_metal_dspark_lock_stage0_hotset(ds4_engine *e) {
+    const char *env = getenv("DS4_METAL_DSPARK_PIN_MAIN_PROJ");
+    if (!e || !env || !env[0] || strcmp(env, "0") == 0 ||
+        e->backend != DS4_BACKEND_METAL || !e->ssd_streaming || !e->dspark ||
+        e->support_kind != DS4_SUPPORT_DSPARK || e->multi_tier ||
+        e->tp.active || !e->mtp_model.map || e->dspark_weights.n_stages == 0) {
+        return;
+    }
+
+    const ds4_dspark_stage_weights *stage0 = &e->dspark_weights.stage[0];
+    const ds4_tensor *tensors[2] = {stage0->main_norm, stage0->main_proj};
+    uint64_t lo = UINT64_MAX;
+    uint64_t hi = 0;
+    for (uint32_t i = 0; i < 2; i++) {
+        const ds4_tensor *t = tensors[i];
+        if (!t || t->bytes == 0 || t->abs_offset > e->mtp_model.size ||
+            t->bytes > e->mtp_model.size - t->abs_offset) {
+            fprintf(stderr,
+                    "ds4: WARNING: Metal DSpark stage-0 hot-set tensor is "
+                    "missing or outside the support mapping; leaving it pageable\n");
+            return;
+        }
+        const uint64_t end = t->abs_offset + t->bytes;
+        if (t->abs_offset < lo) lo = t->abs_offset;
+        if (end > hi) hi = end;
+    }
+
+    const long page_long = sysconf(_SC_PAGESIZE);
+    if (page_long <= 0 || lo == UINT64_MAX || hi <= lo) return;
+    const uint64_t page = (uint64_t)page_long;
+    const uint64_t aligned_lo = (lo / page) * page;
+    if (hi > UINT64_MAX - (page - 1u)) return;
+    const uint64_t aligned_hi = ((hi + page - 1u) / page) * page;
+    if (aligned_hi <= aligned_lo || aligned_hi - aligned_lo > SIZE_MAX) return;
+    const size_t lock_len = (size_t)(aligned_hi - aligned_lo);
+    const size_t max_hotset = 128u * 1024u * 1024u;
+    if (lock_len > max_hotset) {
+        fprintf(stderr,
+                "ds4: WARNING: Metal DSpark stage-0 hot set is %.2f MiB "
+                "(limit %.2f MiB); leaving it pageable\n",
+                (double)lock_len / 1048576.0,
+                (double)max_hotset / 1048576.0);
+        return;
+    }
+
+    void *addr = (void *)(e->mtp_model.map + aligned_lo);
+    const double t0 = now_sec();
+    if (mlock(addr, lock_len) != 0) {
+        const int saved_errno = errno;
+        fprintf(stderr,
+                "ds4: WARNING: Metal DSpark could not pin the %.2f MiB "
+                "stage-0 main projection hot set: %s; leaving it pageable\n",
+                (double)lock_len / 1048576.0,
+                strerror(saved_errno));
+        return;
+    }
+    e->dspark_hot_lock_addr = addr;
+    e->dspark_hot_lock_len = lock_len;
+    fprintf(stderr,
+            "ds4: Metal DSpark pinned %.2f MiB stage-0 main projection hot "
+            "set in %.3fs\n",
+            (double)lock_len / 1048576.0,
+            now_sec() - t0);
+}
+#endif
+
+static uint64_t ds4_engine_support_model_bytes(const ds4_engine *e) {
+    if (!e || !e->mtp_model.map) return 0;
+    const bool runtime_ready =
+        e->mtp_ready ||
+        (e->support_kind == DS4_SUPPORT_DSPARK && e->dspark);
+    if (!runtime_ready || e->mtp_model.size <= e->mtp_model.tensor_data_pos) {
+        return 0;
+    }
+    return e->mtp_model.size - e->mtp_model.tensor_data_pos;
+}
+
+/*
+ * Keep the long-standing exact-count behavior unless the experimental policy
+ * is explicitly selected.  Metal SSD+DSpark is the problematic combination:
+ * target dense weights, the pageable support GGUF, graph buffers, and a
+ * numeric expert cache all compete for one unified-memory working set.  An
+ * NGB input is already handled by the manual-byte-budget path below.
+ *
+ * The support GGUF is mmap-backed and its three stages execute sequentially;
+ * charging all 5+ GiB as resident makes every cache size impossible on a
+ * 16-GiB Mac.  Reserve at most 2 GiB as its active pageable working set until
+ * hardware A/B data justifies a dynamic per-stage estimate.  This policy must
+ * never turn a previously valid explicit-count command into a startup error.
+ */
+static bool ds4_engine_cap_metal_dspark_manual_cache_count(
+        ds4_engine *e,
+        int         ctx_size) {
+    if (!e ||
+        !ds4_streaming_manual_cache_count_cap_eligible(
+                e->backend,
+                e->ssd_streaming,
+                e->dspark,
+                e->support_kind,
+                e->ssd_streaming_cache_experts,
+                e->ssd_streaming_cache_bytes)) {
+        return true;
+    }
+    if (!ds4_streaming_manual_cache_count_cap_enabled()) return true;
+
+    const uint32_t requested_count = e->ssd_streaming_cache_experts;
+    uint64_t per_expert_bytes = 0;
+    if (!ds4_streaming_routed_expert_bytes(&e->weights,
+                                           &per_expert_bytes) ||
+        per_expert_bytes == 0) {
+        fprintf(stderr,
+                "ds4: WARNING: Metal SSD+DSpark safe numeric-cache policy "
+                "could not measure routed expert size; keeping requested=%u "
+                "effective=%u slots\n",
+                requested_count,
+                requested_count);
+        return true;
+    }
+
+    uint64_t non_routed_bytes = 0;
+    if (!weights_streaming_non_routed_bytes(&e->weights,
+                                            &non_routed_bytes)) {
+        fprintf(stderr,
+                "ds4: WARNING: Metal SSD+DSpark safe numeric-cache policy "
+                "could not measure non-routed weights; keeping requested=%u "
+                "effective=%u slots\n",
+                requested_count,
+                requested_count);
+        return true;
+    }
+    const uint64_t support_model_bytes =
+        ds4_engine_support_model_bytes(e);
+    const uint64_t active_support_reserve =
+        ds4_streaming_dspark_active_support_reserve_bytes(
+                support_model_bytes);
+    const uint64_t fixed_model_bytes =
+        ds4_add_sat_u64(non_routed_bytes, active_support_reserve);
+    uint64_t safe_cache_bytes = 0;
+    const bool safe_cache_known =
+        ds4_streaming_manual_cache_safe_bytes(e->backend,
+                                              ctx_size,
+                                              e->prefill_chunk,
+                                              e->ssd_streaming,
+                                              fixed_model_bytes,
+                                              &safe_cache_bytes);
+
+    uint32_t candidate_count = 0;
+    uint64_t requested_bytes = 0;
+    uint64_t effective_bytes = 0;
+    if (!ds4_streaming_manual_cache_cap_count(
+                requested_count,
+                per_expert_bytes,
+                safe_cache_known ? safe_cache_bytes : UINT64_MAX,
+                &candidate_count,
+                &requested_bytes,
+                &effective_bytes)) {
+        fprintf(stderr,
+                "ds4: WARNING: Metal SSD+DSpark numeric expert cache %u x "
+                "%.2f MiB overflows byte accounting; keeping requested=%u "
+                "effective=%u slots (support reserve %.2f GiB)\n",
+                requested_count,
+                (double)per_expert_bytes / 1048576.0,
+                requested_count,
+                requested_count,
+                (double)active_support_reserve / 1073741824.0);
+        return true;
+    }
+
+    const uint32_t effective_count =
+        ds4_streaming_manual_cache_nonfatal_effective_count(
+                requested_count,
+                safe_cache_known,
+                candidate_count);
+
+    if (!safe_cache_known) {
+        fprintf(stderr,
+                "ds4: WARNING: Metal SSD+DSpark working-set recommendation "
+                "is unavailable; keeping requested=%u effective=%u slots "
+                "(%.2f GiB, support reserve %.2f GiB)\n",
+                requested_count,
+                requested_count,
+                (double)requested_bytes / 1073741824.0,
+                (double)active_support_reserve / 1073741824.0);
+        return true;
+    }
+    if (candidate_count == 0) {
+        fprintf(stderr,
+                "ds4: WARNING: Metal SSD+DSpark safe numeric-cache policy "
+                "found no measured room after the context/KV estimate, "
+                "target weights, and "
+                "the %.2f GiB support reserve; keeping requested=%u "
+                "effective=%u slots (%.2f GiB)\n",
+                (double)active_support_reserve / 1073741824.0,
+                requested_count,
+                requested_count,
+                (double)requested_bytes / 1073741824.0);
+        return true;
+    }
+    if (effective_count < requested_count) {
+        e->ssd_streaming_cache_experts = effective_count;
+        fprintf(stderr,
+                "ds4: WARNING: Metal SSD+DSpark safe numeric-cache policy "
+                "capped requested=%u slots (%.2f GiB) to effective=%u slots "
+                "(%.2f GiB); active support reserve %.2f GiB of %.2f GiB "
+                "mapped support\n",
+                requested_count,
+                (double)requested_bytes / 1073741824.0,
+                effective_count,
+                (double)effective_bytes / 1073741824.0,
+                (double)active_support_reserve / 1073741824.0,
+                (double)support_model_bytes / 1073741824.0);
+    } else {
+        fprintf(stderr,
+                "ds4: Metal SSD+DSpark safe numeric-cache policy kept "
+                "requested=%u effective=%u slots (%.2f GiB); active support "
+                "reserve %.2f GiB of %.2f GiB mapped support\n",
+                requested_count,
+                effective_count,
+                (double)effective_bytes / 1073741824.0,
+                (double)active_support_reserve / 1073741824.0,
+                (double)support_model_bytes / 1073741824.0);
+    }
+    return true;
+}
+
+#ifdef DS4_ROCM_BUILD
+static bool ds4_rocm_dspark_ssd_layout_supported(
+        const ds4_weights *weights,
+        uint32_t          *bad_layer,
+        uint32_t          *bad_gate_type,
+        uint32_t          *bad_down_type) {
+    if (bad_layer) *bad_layer = UINT32_MAX;
+    if (bad_gate_type) *bad_gate_type = 0;
+    if (bad_down_type) *bad_down_type = 0;
+    if (!weights) return false;
+
+    bool found_routed = false;
+    for (uint32_t il = 0; il < DS4_N_LAYER; il++) {
+        const ds4_layer_weights *layer = &weights->layer[il];
+        if (!layer->ffn_gate_exps || !layer->ffn_up_exps ||
+            !layer->ffn_down_exps) {
+            continue;
+        }
+        found_routed = true;
+        const uint32_t gate_type = layer->ffn_gate_exps->type;
+        const uint32_t up_type = layer->ffn_up_exps->type;
+        const uint32_t down_type = layer->ffn_down_exps->type;
+        const bool iq2_selected =
+            gate_type == DS4_TENSOR_IQ2_XXS &&
+            up_type == DS4_TENSOR_IQ2_XXS &&
+            (down_type == DS4_TENSOR_Q2_K ||
+             down_type == DS4_TENSOR_IQ2_XXS);
+        const bool q2_selected =
+            gate_type == DS4_TENSOR_Q2_K &&
+            up_type == DS4_TENSOR_Q2_K &&
+            down_type == DS4_TENSOR_Q2_K;
+        if (iq2_selected || q2_selected) continue;
+
+        if (bad_layer) *bad_layer = il;
+        if (bad_gate_type) *bad_gate_type = gate_type;
+        if (bad_down_type) *bad_down_type = down_type;
+        return false;
+    }
+    return found_routed;
+}
+#endif
+
 static uint64_t ds4_engine_dynamic_expert_cache_bytes(
         const ds4_engine *e) {
     if (!e || !e->ssd_streaming) return 0;
@@ -38806,6 +41969,8 @@ static void ds4_engine_print_startup_memory(
         ds4_add_sat_u64(mem.raw_bytes, mem.compressed_bytes);
     const uint64_t dynamic_expert_cache_bytes =
         ds4_engine_dynamic_expert_cache_bytes(e);
+    const uint64_t support_model_bytes =
+        ds4_engine_support_model_bytes(e);
     const uint64_t expert_reserved_bytes =
         e->ssd_streaming_prefill_headroom_bytes;
     uint64_t resident_model_bytes = e->startup_model_span_bytes;
@@ -38824,6 +41989,7 @@ static void ds4_engine_print_startup_memory(
     uint64_t total = kv_bytes;
     total = ds4_add_sat_u64(total, mem.scratch_bytes);
     total = ds4_add_sat_u64(total, resident_model_bytes);
+    total = ds4_add_sat_u64(total, support_model_bytes);
     total = ds4_add_sat_u64(total, dynamic_expert_cache_bytes);
     total = ds4_add_sat_u64(total, expert_reserved_bytes);
 
@@ -38841,6 +42007,11 @@ static void ds4_engine_print_startup_memory(
             ds4_bytes_to_gib(mem.compressed_bytes),
             ds4_bytes_to_gib(mem.scratch_bytes),
             ds4_bytes_to_gib(resident_model_bytes));
+    if (support_model_bytes != 0) {
+        fprintf(stderr,
+                " + support model %.2f GiB",
+                ds4_bytes_to_gib(support_model_bytes));
+    }
     if (dynamic_expert_cache_bytes != 0) {
         fprintf(stderr,
                 " + expert cache %.2f GiB",
@@ -41374,7 +44545,7 @@ static double glm_graph_env_double(
     return v;
 }
 
-static uint64_t glm_graph_host_memory_bytes(void) {
+static uint64_t ds4_graph_host_memory_bytes(void) {
 #if defined(__APPLE__)
     uint64_t mem = 0;
     size_t len = sizeof(mem);
@@ -41490,7 +44661,7 @@ static bool glm_graph_memory_guard_budget(
     (void)load_slice;
     (void)ssd_streaming;
 #endif
-    uint64_t budget_base = glm_graph_host_memory_bytes();
+    uint64_t budget_base = ds4_graph_host_memory_bytes();
     if (budget_base == 0) {
         budget_base = ds4_gpu_recommended_working_set_size();
     }
@@ -41901,14 +45072,6 @@ static void glm_graph_report_prefill_display_progress(
                      (int)((uint64_t)absolute_base + (uint64_t)work_total));
 }
 
-static bool glm_graph_small_prefill_stage_sync(
-        uint32_t n_tokens,
-        bool     logits_requested) {
-    return logits_requested &&
-           n_tokens > 0 &&
-           n_tokens <= DS4_GLM_METAL_SMALL_PREFILL_STAGE_SYNC_TOKENS;
-}
-
 static uint32_t glm_graph_indexed_decode_split_min_block_rows(void) {
     return 32u;
 }
@@ -41945,11 +45108,6 @@ static bool glm_graph_indexed_decode_split_group8_available(uint32_t n_selected)
 #endif
 }
 
-static bool glm_graph_prefill_stage_sync_boundary(void) {
-    if (ds4_gpu_end_commands() == 0) return false;
-    return ds4_gpu_begin_commands() != 0;
-}
-
 static bool glm_graph_indexed_prefill_attention_boundary(void) {
 #ifdef DS4_ROCM_BUILD
     /*
@@ -43126,10 +46284,7 @@ static bool glm_graph_warm_compact_indexer_store(
         return false;
     }
 
-    const bool profile = false;
-    const double t0 = 0.0;
     bool ok = ds4_gpu_begin_commands() != 0;
-    uint32_t warmed = 0;
     for (uint32_t il = g->layer_start; ok && il <= g->layer_end; il++) {
         if (!glm_graph_layer_uses_full_indexer(il)) continue;
         const ds4_layer_weights *l = &weights->layer[il];
@@ -43162,18 +46317,9 @@ static bool glm_graph_warm_compact_indexer_store(
                 DS4_ROPE_YARN_BETA_FAST,
                 DS4_ROPE_YARN_BETA_SLOW,
                 glm_graph_compact_cache_is_f16()) != 0;
-        if (ok) warmed++;
     }
     if (ok) ok = ds4_gpu_end_commands() != 0;
     else (void)ds4_gpu_synchronize();
-
-    if (profile) {
-        fprintf(stderr,
-                "ds4: GLM compact indexer warmup pos=%u layers=%u %.3f ms\n",
-                warm_pos,
-                warmed,
-                (now_sec() - t0) * 1000.0);
-    }
     return ok;
 }
 
@@ -44458,7 +47604,6 @@ static bool glm_graph_profile_router_selection_batch(
 
 static bool glm_graph_prefill_stage_boundary(
         bool        stage_profile,
-        bool        stage_sync,
         const char *part,
         const char *stage,
         uint32_t    il,
@@ -44468,7 +47613,6 @@ static bool glm_graph_prefill_stage_boundary(
     if (stage_profile) {
         return glm_graph_profile_stage(true, part, stage, il, pos0, n_tokens, stage_t0);
     }
-    if (stage_sync) return glm_graph_prefill_stage_sync_boundary();
     return true;
 }
 
@@ -45069,6 +48213,8 @@ static bool glm_graph_encode_sparse_ffn_one(
                         l,
                         il,
                         selected_event,
+                        false,
+                        false,
                         gate_out * gate_row_bytes,
                         down_out * down_row_bytes);
                 async_path_profiled = async_profile && async_load_started;
@@ -46252,7 +49398,6 @@ static bool glm_graph_encode_sparse_ffn_indexed_batch_routed_moe(
         ds4_gpu_tensor          *next,
         uint32_t                 n_tokens,
         bool                     stage_profile,
-        bool                     stage_sync,
         double                  *stage_t0) {
     const char *failed_stage = "setup";
     if (!g || !model || !l || !after_attn || !next ||
@@ -46343,7 +49488,6 @@ static bool glm_graph_encode_sparse_ffn_indexed_batch_routed_moe(
         ds4_gpu_tensor_free(logits_view);
     }
     if (ok) ok = glm_graph_prefill_stage_boundary(stage_profile,
-                                                  stage_sync,
                                                   "glm_indexed_ffn",
                                                   "router",
                                                   il,
@@ -46405,7 +49549,6 @@ static bool glm_graph_encode_sparse_ffn_indexed_batch_routed_moe(
                 !use_grouped_moe) != 0;
     }
     if (ok) ok = glm_graph_prefill_stage_boundary(stage_profile,
-                                                  stage_sync,
                                                   "glm_indexed_ffn",
                                                   "routed_moe",
                                                   il,
@@ -46555,7 +49698,6 @@ static bool glm_graph_encode_sparse_ffn_indexed_batch_routed_moe(
         ds4_gpu_tensor_free(after_attn_view);
     }
     if (ok) ok = glm_graph_prefill_stage_boundary(stage_profile,
-                                                  stage_sync,
                                                   "glm_indexed_ffn",
                                                   "shared_expert",
                                                   il,
@@ -46582,7 +49724,6 @@ static bool glm_graph_encode_sparse_ffn_indexed_batch_routed_moe(
         }
     }
     if (ok) ok = glm_graph_prefill_stage_boundary(stage_profile,
-                                                  stage_sync,
                                                   "glm_indexed_ffn",
                                                   "residual",
                                                   il,
@@ -46640,7 +49781,6 @@ static bool glm_graph_encode_ffn_batch(
         uint32_t                 n_tokens,
         bool                     full_layer_prefill,
         bool                     stage_profile,
-        bool                     stage_sync,
         double                  *stage_t0) {
     if (!g || !model || !weights || !l || !after_attn || !next || n_tokens == 0) return false;
 
@@ -46654,7 +49794,6 @@ static bool glm_graph_encode_ffn_batch(
                                              n_tokens,
                                              DS4_RMS_EPS) != 0;
     if (ok) ok = glm_graph_prefill_stage_boundary(stage_profile,
-                                                  stage_sync,
                                                   "glm_ffn",
                                                   "ffn_norm",
                                                   il,
@@ -46691,7 +49830,6 @@ static bool glm_graph_encode_ffn_batch(
                 g->glm53 ? DS4_SWIGLU_CLAMP_EXP : 0.0f);
         if (fused_gate_up) {
             ok = glm_graph_prefill_stage_boundary(stage_profile,
-                                                  stage_sync,
                                                   "glm_ffn",
                                                   "dense_gate_up_swiglu",
                                                   il,
@@ -46714,7 +49852,6 @@ static bool glm_graph_encode_ffn_batch(
                                                       g->batch_ffn_norm,
                                                       n_tokens);
             if (ok) ok = glm_graph_prefill_stage_boundary(stage_profile,
-                                                          stage_sync,
                                                           "glm_ffn",
                                                           "dense_gate_up",
                                                           il,
@@ -46728,7 +49865,6 @@ static bool glm_graph_encode_ffn_batch(
                                                g->glm53 ? DS4_SWIGLU_CLAMP_EXP : 0.0f,
                                                1.0f) != 0;
             if (ok) ok = glm_graph_prefill_stage_boundary(stage_profile,
-                                                          stage_sync,
                                                           "glm_ffn",
                                                           "dense_swiglu",
                                                           il,
@@ -46754,7 +49890,6 @@ static bool glm_graph_encode_ffn_batch(
                     (uint64_t)n_tokens * DS4_N_EMBD, il, pos0);
         }
         if (ok) ok = glm_graph_prefill_stage_boundary(stage_profile,
-                                                      stage_sync,
                                                       "glm_ffn",
                                                       "dense_down",
                                                       il,
@@ -46774,7 +49909,6 @@ static bool glm_graph_encode_ffn_batch(
                                     (uint32_t)residual_elems) != 0;
         }
         if (ok) ok = glm_graph_prefill_stage_boundary(stage_profile,
-                                                      stage_sync,
                                                       "glm_ffn",
                                                       "residual",
                                                       il,
@@ -46837,7 +49971,6 @@ static bool glm_graph_encode_ffn_batch(
         }
     }
     if (ok) ok = glm_graph_prefill_stage_boundary(stage_profile,
-                                                  stage_sync,
                                                   "glm_ffn",
                                                   "router",
                                                   il,
@@ -46905,7 +50038,6 @@ static bool glm_graph_encode_ffn_batch(
                     g->glm53 ? DS4_SWIGLU_CLAMP_EXP : 0.0f); \
             if (fused_shared) { \
                 ok = glm_graph_prefill_stage_boundary(stage_profile, \
-                                                      stage_sync, \
                                                       "glm_ffn", \
                                                       "shared_gate_up_swiglu", \
                                                       il, \
@@ -46928,7 +50060,6 @@ static bool glm_graph_encode_ffn_batch(
                                                           g->batch_ffn_norm, \
                                                           n_tokens); \
                 if (ok) ok = glm_graph_prefill_stage_boundary(stage_profile, \
-                                                              stage_sync, \
                                                               "glm_ffn", \
                                                               "shared_gate_up", \
                                                               il, \
@@ -46942,7 +50073,6 @@ static bool glm_graph_encode_ffn_batch(
                                                    g->glm53 ? DS4_SWIGLU_CLAMP_EXP : 0.0f, \
                                                    1.0f) != 0; \
                 if (ok) ok = glm_graph_prefill_stage_boundary(stage_profile, \
-                                                              stage_sync, \
                                                               "glm_ffn", \
                                                               "shared_swiglu", \
                                                               il, \
@@ -46959,7 +50089,6 @@ static bool glm_graph_encode_ffn_batch(
                                                   g->batch_shared_mid, \
                                                   n_tokens); \
         if (ok) ok = glm_graph_prefill_stage_boundary(stage_profile, \
-                                                      stage_sync, \
                                                       "glm_ffn", \
                                                       "shared_down", \
                                                       il, \
@@ -47033,7 +50162,6 @@ static bool glm_graph_encode_ffn_batch(
         if (!ok) fprintf(stderr, "ds4: GLM TP batch gate failed (layer %u)\n", il);
     }
     if (ok) ok = glm_graph_prefill_stage_boundary(stage_profile,
-                                                  stage_sync,
                                                   "glm_ffn",
                                                   "routed_moe",
                                                   il,
@@ -47078,7 +50206,6 @@ static bool glm_graph_encode_ffn_batch(
                                         (uint32_t)residual_elems) != 0;
     }
     if (ok) ok = glm_graph_prefill_stage_boundary(stage_profile,
-                                                  stage_sync,
                                                   "glm_ffn",
                                                   "residual",
                                                   il,
@@ -47945,7 +51072,6 @@ static bool glm_graph_verify_rows(
                                                 n,
                                                 false,
                                                 false,
-                                                false,
                                                 NULL);
         if (ok) {
             ds4_gpu_tensor *tmp = cur;
@@ -48225,9 +51351,7 @@ static bool glm_graph_forward_tokens(
                                               work_total,
                                               true);
 
-    const bool stage_sync =
-        glm_graph_small_prefill_stage_sync(n_tokens, logits_out != NULL);
-    const uint32_t layer_flush_interval = stage_sync ? 0u :
+    const uint32_t layer_flush_interval =
         glm_graph_full_prefill_layer_flush_interval(n_tokens,
                                                     n_rows,
                                                     logits_out != NULL);
@@ -48298,10 +51422,9 @@ static bool glm_graph_forward_tokens(
         metal_graph_stream_prefill_layer_prepare_ahead() : 1u;
     if (trace) {
         glm_graph_full_prefill_tracef(
-                "mode pos=%u tokens=%u stage_sync=%u layer_flush_interval=%u progress_flush_interval=%u drain_interval=%u",
+                "mode pos=%u tokens=%u layer_flush_interval=%u progress_flush_interval=%u drain_interval=%u",
                 pos0,
                 n_tokens,
-                stage_sync ? 1u : 0u,
                 layer_flush_interval,
                 progress_flush_interval,
                 drain_interval);
@@ -48425,8 +51548,6 @@ static bool glm_graph_forward_tokens(
 #define DS4_GLM_PROFILE_PREFILL_STAGE(part_, name_) do { \
         if (ok && layer_stage_profile) { \
             ok = metal_graph_layer_stage_profile_boundary((part_), (name_), il, pos0, n_tokens, &layer_stage_t0); \
-        } else if (ok && stage_sync) { \
-            ok = glm_graph_prefill_stage_sync_boundary(); \
         } \
     } while (0)
     for (uint32_t il = g->layer_start; ok && il <= g->layer_end; il++) {
@@ -49044,7 +52165,6 @@ static bool glm_graph_forward_tokens(
                                             n_tokens,
                                             full_layer_prefill,
                                             layer_stage_profile,
-                                            stage_sync,
                                             layer_stage_profile ? &layer_stage_t0 : NULL);
         }
         if (ok && g->glm53) {
@@ -49575,9 +52695,7 @@ static bool glm_graph_forward_indexed_tokens(
     const bool use_batch_indexer_weights_proj = true;
     const bool use_batch_attn_out_proj = true;
     const bool use_batch_ffn = glm_graph_indexed_prefill_batch_ffn();
-    const bool stage_sync =
-        glm_graph_small_prefill_stage_sync(n_tokens, logits_out != NULL);
-    const uint32_t layer_flush_interval = stage_sync ? 0u :
+    const uint32_t layer_flush_interval =
         glm_graph_full_prefill_layer_flush_interval(n_tokens,
                                                     n_tokens,
                                                     logits_out != NULL);
@@ -49710,8 +52828,6 @@ static bool glm_graph_forward_indexed_tokens(
         } \
         if (ok && layer_stage_profile) { \
             ok = metal_graph_layer_stage_profile_boundary((part_), (name_), il, pos0, n_tokens, &layer_stage_t0); \
-        } else if (ok && stage_sync) { \
-            ok = glm_graph_prefill_stage_sync_boundary(); \
         } \
     } while (0)
     ds4_gpu_tensor *last_indexer_selected = NULL;
@@ -50680,7 +53796,6 @@ static bool glm_graph_forward_indexed_tokens(
                                             n_tokens,
                                             false,
                                             layer_stage_profile,
-                                            stage_sync,
                                             layer_stage_profile ? &layer_stage_t0 : NULL);
         } else if (ok) {
             const bool use_batch_ffn_norm =
@@ -50712,7 +53827,6 @@ static bool glm_graph_forward_indexed_tokens(
                         next,
                         n_tokens,
                         layer_stage_profile,
-                        stage_sync,
                         layer_stage_profile ? &layer_stage_t0 : NULL);
             } else for (uint32_t t = 0; ok && t < n_tokens; t++) {
                 ds4_gpu_tensor *after_attn_view =
@@ -51534,13 +54648,8 @@ static bool glm_graph_forward_token(
         }
     }
 
-    const bool decode_output_profile = false;
     const bool merge_indexed_output =
-        logits_out != NULL && use_indexed_attention && !decode_output_profile;
-    double decode_output_stage_t0 = decode_output_profile ? now_sec() : 0.0;
-    const bool decode_flush_profile = false;
-    uint32_t decode_flush_layer0 = 0;
-    double decode_flush_stage_t0 = decode_flush_profile ? now_sec() : 0.0;
+        logits_out != NULL && use_indexed_attention;
 
     const bool static_decode_map =
         !input_hc &&
@@ -52455,24 +55564,7 @@ static bool glm_graph_forward_token(
             decode_layer_flush_interval != 0 &&
             il < g->layer_end &&
             (slice_layer_done % decode_layer_flush_interval) == 0) {
-            if (decode_flush_profile) {
-                ok = ds4_gpu_flush_commands() != 0;
-                if (ok) ok = ds4_gpu_synchronize() != 0;
-                if (ok) {
-                    const double now = now_sec();
-                    fprintf(stderr,
-                            "ds4: GLM decode layer flush pos=%u layers=%u..%u %.3f ms\n",
-                            pos,
-                            decode_flush_layer0,
-                            il,
-                            (now - decode_flush_stage_t0) * 1000.0);
-                    decode_flush_layer0 = il + 1u;
-                    decode_flush_stage_t0 = now;
-                    ok = ds4_gpu_begin_commands() != 0;
-                }
-            } else {
-                ok = ds4_gpu_flush_commands() != 0;
-            }
+            ok = ds4_gpu_flush_commands() != 0;
         }
         if (streaming_decode_sync_each_layer) {
             if (ok) ok = ds4_gpu_end_commands() != 0;
@@ -52501,14 +55593,6 @@ static bool glm_graph_forward_token(
     } else if (!ok) {
         (void)ds4_gpu_synchronize();
     }
-    if (decode_output_profile) {
-        const double now = now_sec();
-        fprintf(stderr,
-                "ds4: GLM decode output profile pos=%u layers=%.3f ms\n",
-                pos,
-                (now - decode_output_stage_t0) * 1000.0);
-        decode_output_stage_t0 = now;
-    }
     if (ok && output_hc && !defer_completion) {
         ok = ds4_gpu_tensor_read(g->glm53 ? g->hc_cur : g->cur,
                                  0,
@@ -52519,23 +55603,6 @@ static bool glm_graph_forward_token(
     }
     if (ok && logits_out && !defer_completion) {
         if (use_indexed_attention) {
-            if (!merge_indexed_output) {
-                if (g->ssd_streaming && !static_decode_map) {
-                    ok = glm_graph_stream_map_output(g, model, weights);
-                }
-                if (ok) ok = glm_graph_begin_commands_if_needed();
-                if (ok) ok = glm_graph_encode_output_head(g, model, weights);
-                if (ok) ok = glm_graph_end_commands_if_active();
-                else (void)ds4_gpu_synchronize();
-                if (decode_output_profile) {
-                    const double now = now_sec();
-                    fprintf(stderr,
-                            "ds4: GLM decode output profile pos=%u output_head=%.3f ms\n",
-                            pos,
-                            (now - decode_output_stage_t0) * 1000.0);
-                    decode_output_stage_t0 = now;
-                }
-            }
             if (ok) {
                 if (glm_debug_hidden_dump_layer() < 0)
                     glm_debug_dump_hidden_row(g->cur, 0);
@@ -52543,13 +55610,6 @@ static bool glm_graph_forward_token(
                                          0,
                                          logits_out,
                                          (uint64_t)DS4_N_VOCAB * sizeof(float)) != 0;
-                if (decode_output_profile) {
-                    const double now = now_sec();
-                    fprintf(stderr,
-                            "ds4: GLM decode output profile pos=%u logits_read=%.3f ms\n",
-                            pos,
-                            (now - decode_output_stage_t0) * 1000.0);
-                }
             }
         } else {
             if (g->ssd_streaming && !static_decode_map) {
@@ -52564,13 +55624,6 @@ static bool glm_graph_forward_token(
                 weights,
                 g->glm53 ? g->hc_cur : g->cur,
                 logits_out);
-            if (decode_output_profile) {
-                const double now = now_sec();
-                fprintf(stderr,
-                        "ds4: GLM decode output profile pos=%u fallback_output=%.3f ms\n",
-                        pos,
-                        (now - decode_output_stage_t0) * 1000.0);
-            }
         }
     }
     if (ok &&
@@ -52597,482 +55650,6 @@ static bool glm_graph_forward_token(
 #undef DS4_GLM_FT_FAIL
 }
 
-static int glm_metal_first_token_logits(
-        const ds4_model   *model,
-        const ds4_weights *weights,
-        int                token,
-        float             *logits_out) {
-    if (!model || !weights || !logits_out) return 1;
-    if (token < 0 || token >= (int)DS4_N_VOCAB) {
-        fprintf(stderr, "ds4: GLM token %d is outside vocab\n", token);
-        return 1;
-    }
-    if (!weights->token_embd || weights->token_embd->type != DS4_TENSOR_Q8_0 ||
-        !weights->output_norm || weights->output_norm->type != DS4_TENSOR_F32 ||
-        !weights->output || weights->output->type != DS4_TENSOR_Q8_0 ||
-        weights->output_norm->dim[0] != DS4_N_EMBD ||
-        weights->output->dim[0] != DS4_N_EMBD ||
-        weights->output->dim[1] != DS4_N_VOCAB) {
-        fprintf(stderr, "ds4: GLM Metal first-token path found unexpected embedding/output layout\n");
-        return 1;
-    }
-    if (DS4_N_LAYER <= DS4_N_NEXTN_PREDICT) {
-        fprintf(stderr, "ds4: GLM Metal first-token path has no normal transformer layers\n");
-        return 1;
-    }
-
-    const uint32_t normal_layers = DS4_N_LAYER - DS4_N_NEXTN_PREDICT;
-    const uint64_t heads_dim = (uint64_t)DS4_N_HEAD * DS4_N_VALUE_MLA;
-    uint64_t kv_raw_dim = 0;
-    uint64_t dense_hidden_max = DS4_N_FF_EXP;
-    bool generic_routed_moe = false;
-    for (uint32_t il = 0; il < normal_layers; il++) {
-        const ds4_layer_weights *l = &weights->layer[il];
-        if (l->attn_kv_a_mqa && l->attn_kv_a_mqa->dim[1] > kv_raw_dim) {
-            kv_raw_dim = l->attn_kv_a_mqa->dim[1];
-        }
-        if (il < DS4_N_LEADING_DENSE && l->ffn_gate &&
-            l->ffn_gate->dim[1] > dense_hidden_max) {
-            dense_hidden_max = l->ffn_gate->dim[1];
-        }
-        if (glm_graph_layer_uses_generic_routed_moe(l)) generic_routed_moe = true;
-    }
-    if (kv_raw_dim < DS4_N_KV_LORA) {
-        fprintf(stderr, "ds4: GLM Metal first-token path found no valid KV projection\n");
-        return 1;
-    }
-
-    const uint64_t emb_bytes = (uint64_t)DS4_N_EMBD * sizeof(float);
-    const uint64_t sparse_mid_elems = (uint64_t)DS4_N_EXPERT_USED * DS4_N_FF_EXP;
-    const uint64_t ffn_mid_elems =
-        dense_hidden_max > sparse_mid_elems ? dense_hidden_max : sparse_mid_elems;
-    const uint64_t routed_mid_bytes =
-        (uint64_t)DS4_N_EXPERT_USED * DS4_N_FF_EXP * sizeof(float);
-    const uint64_t routed_down_bytes =
-        (uint64_t)DS4_N_EXPERT_USED * DS4_N_EMBD * sizeof(float);
-    const uint64_t logits_bytes = (uint64_t)DS4_N_VOCAB * sizeof(float);
-
-    ds4_gpu_tensor *cur = NULL;
-    ds4_gpu_tensor *attn_norm = NULL;
-    ds4_gpu_tensor *kv_raw = NULL;
-    ds4_gpu_tensor *kv_norm = NULL;
-    ds4_gpu_tensor *heads = NULL;
-    ds4_gpu_tensor *attn_out = NULL;
-    ds4_gpu_tensor *after_attn = NULL;
-    ds4_gpu_tensor *ffn_norm = NULL;
-    ds4_gpu_tensor *ffn_gate = NULL;
-    ds4_gpu_tensor *ffn_up = NULL;
-    ds4_gpu_tensor *ffn_mid = NULL;
-    ds4_gpu_tensor *routed_gate = NULL;
-    ds4_gpu_tensor *routed_up = NULL;
-    ds4_gpu_tensor *routed_down = NULL;
-    ds4_gpu_tensor *ffn_out = NULL;
-    ds4_gpu_tensor *ffn_sum = NULL;
-    ds4_gpu_tensor *next = NULL;
-    ds4_gpu_tensor *router_logits = NULL;
-    ds4_gpu_tensor *router_probs = NULL;
-    ds4_gpu_tensor *router_selected = NULL;
-    ds4_gpu_tensor *router_weights = NULL;
-    ds4_gpu_tensor *logits = NULL;
-
-    int ok = 1;
-#define DS4_GLM_FIRST_ALLOC_TENSOR(var, bytes_) \
-    do { \
-        (var) = ds4_gpu_tensor_alloc((bytes_)); \
-        if (!(var)) { \
-            fprintf(stderr, "ds4: GLM Metal first-token path could not allocate %s\n", #var); \
-            ok = 0; \
-        } \
-    } while (0)
-
-    DS4_GLM_FIRST_ALLOC_TENSOR(cur, emb_bytes);
-    DS4_GLM_FIRST_ALLOC_TENSOR(attn_norm, emb_bytes);
-    DS4_GLM_FIRST_ALLOC_TENSOR(kv_raw, kv_raw_dim * sizeof(float));
-    DS4_GLM_FIRST_ALLOC_TENSOR(kv_norm, (uint64_t)DS4_N_KV_LORA * sizeof(float));
-    DS4_GLM_FIRST_ALLOC_TENSOR(heads, heads_dim * sizeof(float));
-    DS4_GLM_FIRST_ALLOC_TENSOR(attn_out, emb_bytes);
-    DS4_GLM_FIRST_ALLOC_TENSOR(after_attn, emb_bytes);
-    DS4_GLM_FIRST_ALLOC_TENSOR(ffn_norm, emb_bytes);
-    DS4_GLM_FIRST_ALLOC_TENSOR(ffn_gate, dense_hidden_max * sizeof(float));
-    DS4_GLM_FIRST_ALLOC_TENSOR(ffn_up, dense_hidden_max * sizeof(float));
-    DS4_GLM_FIRST_ALLOC_TENSOR(ffn_mid, ffn_mid_elems * sizeof(float));
-    if (generic_routed_moe) {
-        DS4_GLM_FIRST_ALLOC_TENSOR(routed_gate, routed_mid_bytes);
-        DS4_GLM_FIRST_ALLOC_TENSOR(routed_up, routed_mid_bytes);
-        DS4_GLM_FIRST_ALLOC_TENSOR(routed_down, routed_down_bytes);
-    }
-    DS4_GLM_FIRST_ALLOC_TENSOR(ffn_out, emb_bytes);
-    DS4_GLM_FIRST_ALLOC_TENSOR(ffn_sum, emb_bytes);
-    DS4_GLM_FIRST_ALLOC_TENSOR(next, emb_bytes);
-    DS4_GLM_FIRST_ALLOC_TENSOR(router_logits, (uint64_t)DS4_N_EXPERT * sizeof(float));
-    DS4_GLM_FIRST_ALLOC_TENSOR(router_probs, (uint64_t)DS4_N_EXPERT * sizeof(float));
-    DS4_GLM_FIRST_ALLOC_TENSOR(router_selected, (uint64_t)DS4_N_EXPERT_USED * sizeof(int32_t));
-    DS4_GLM_FIRST_ALLOC_TENSOR(router_weights, (uint64_t)DS4_N_EXPERT_USED * sizeof(float));
-    DS4_GLM_FIRST_ALLOC_TENSOR(logits, logits_bytes);
-#undef DS4_GLM_FIRST_ALLOC_TENSOR
-
-    if (ok) {
-        ok = ds4_gpu_embed_token_q8_0_tensor(cur,
-                                             model->map,
-                                             model->size,
-                                             weights->token_embd->abs_offset,
-                                             DS4_N_VOCAB,
-                                             (uint32_t)token,
-                                             DS4_N_EMBD);
-    }
-    for (uint32_t il = 0; ok && il < normal_layers; il++) {
-        const ds4_layer_weights *gl = &weights->layer[il];
-        const uint64_t gl_kv_raw_dim = gl->attn_kv_a_mqa ? gl->attn_kv_a_mqa->dim[1] : 0;
-        if (!gl->attn_norm ||
-            !gl->attn_kv_a_mqa ||
-            !gl->attn_kv_a_norm ||
-            !gl->attn_v_b ||
-            !gl->attn_output ||
-            !gl->ffn_norm ||
-            gl_kv_raw_dim < DS4_N_KV_LORA ||
-            gl_kv_raw_dim > kv_raw_dim ||
-            gl->attn_kv_a_mqa->type != DS4_TENSOR_Q8_0 ||
-            gl->attn_kv_a_mqa->dim[0] != DS4_N_EMBD ||
-            gl->attn_v_b->type != DS4_TENSOR_Q8_0 ||
-            gl->attn_v_b->dim[0] != DS4_N_KV_LORA ||
-            gl->attn_v_b->dim[1] != DS4_N_VALUE_MLA ||
-            gl->attn_v_b->dim[2] != DS4_N_HEAD ||
-            gl->attn_output->type != DS4_TENSOR_Q8_0 ||
-            gl->attn_output->dim[0] != heads_dim ||
-            gl->attn_output->dim[1] != DS4_N_EMBD) {
-            fprintf(stderr,
-                    "ds4: GLM Metal first-token path found unexpected attention layout in layer %u\n",
-                    il);
-            ok = 0;
-            break;
-        }
-
-        if (ok) ok = ds4_gpu_rms_norm_weight_tensor(attn_norm, cur,
-                                                    model->map, model->size,
-                                                    gl->attn_norm->abs_offset,
-                                                    DS4_N_EMBD, DS4_RMS_EPS);
-        if (ok) ok = ds4_gpu_matmul_q8_0_tensor(kv_raw,
-                                                model->map,
-                                                model->size,
-                                                gl->attn_kv_a_mqa->abs_offset,
-                                                DS4_N_EMBD,
-                                                gl_kv_raw_dim,
-                                                attn_norm,
-                                                1);
-        if (ok) ok = ds4_gpu_rms_norm_weight_tensor(kv_norm, kv_raw,
-                                                    model->map, model->size,
-                                                    gl->attn_kv_a_norm->abs_offset,
-                                                    DS4_N_KV_LORA, DS4_RMS_EPS);
-        if (ok) ok = ds4_gpu_matmul_q8_0_tensor(heads,
-                                                model->map,
-                                                model->size,
-                                                gl->attn_v_b->abs_offset,
-                                                DS4_N_KV_LORA,
-                                                heads_dim,
-                                                kv_norm,
-                                                1);
-        if (ok) ok = ds4_gpu_matmul_q8_0_tensor(attn_out,
-                                                model->map,
-                                                model->size,
-                                                gl->attn_output->abs_offset,
-                                                heads_dim,
-                                                DS4_N_EMBD,
-                                                heads,
-                                                1);
-        if (ok) ok = ds4_gpu_add_tensor(after_attn, cur, attn_out, DS4_N_EMBD);
-        if (ok) ok = ds4_gpu_rms_norm_weight_tensor(ffn_norm, after_attn,
-                                                    model->map, model->size,
-                                                    gl->ffn_norm->abs_offset,
-                                                    DS4_N_EMBD, DS4_RMS_EPS);
-        if (il < DS4_N_LEADING_DENSE) {
-            const uint64_t gl_ffn_hidden = gl->ffn_gate ? gl->ffn_gate->dim[1] : 0;
-            if (!gl->ffn_gate ||
-                !gl->ffn_up ||
-                !gl->ffn_down ||
-                gl->ffn_gate->type != DS4_TENSOR_Q8_0 ||
-                gl->ffn_up->type != DS4_TENSOR_Q8_0 ||
-                gl->ffn_down->type != DS4_TENSOR_Q8_0 ||
-                gl->ffn_gate->dim[0] != DS4_N_EMBD ||
-                gl->ffn_up->dim[0] != DS4_N_EMBD ||
-                gl->ffn_up->dim[1] != gl_ffn_hidden ||
-                gl->ffn_down->dim[0] != gl_ffn_hidden ||
-                gl->ffn_down->dim[1] != DS4_N_EMBD ||
-                gl_ffn_hidden > dense_hidden_max) {
-                fprintf(stderr,
-                        "ds4: GLM Metal first-token path found unexpected dense FFN layout in layer %u\n",
-                        il);
-                ok = 0;
-                break;
-            }
-            if (ok) ok = ds4_gpu_matmul_q8_0_tensor(ffn_gate,
-                                                    model->map,
-                                                    model->size,
-                                                    gl->ffn_gate->abs_offset,
-                                                    DS4_N_EMBD,
-                                                    gl_ffn_hidden,
-                                                    ffn_norm,
-                                                    1);
-            if (ok) ok = ds4_gpu_matmul_q8_0_tensor(ffn_up,
-                                                    model->map,
-                                                    model->size,
-                                                    gl->ffn_up->abs_offset,
-                                                    DS4_N_EMBD,
-                                                    gl_ffn_hidden,
-                                                    ffn_norm,
-                                                    1);
-            if (ok) ok = ds4_gpu_swiglu_tensor(ffn_mid, ffn_gate, ffn_up,
-                                               (uint32_t)gl_ffn_hidden, 0.0f, 1.0f);
-            if (ok) ok = ds4_gpu_matmul_q8_0_tensor(ffn_out,
-                                                    model->map,
-                                                    model->size,
-                                                    gl->ffn_down->abs_offset,
-                                                    gl_ffn_hidden,
-                                                    DS4_N_EMBD,
-                                                    ffn_mid,
-                                                    1);
-            if (ok) ok = ds4_gpu_add_tensor(next, after_attn, ffn_out, DS4_N_EMBD);
-        } else {
-            const uint32_t gl_gate_type = gl->ffn_gate_exps ? gl->ffn_gate_exps->type : 0;
-            const uint32_t gl_up_type = gl->ffn_up_exps ? gl->ffn_up_exps->type : 0;
-            const bool gl_gate_pair_supported =
-                glm_graph_gate_pair_type_supported(gl_gate_type, gl_up_type);
-            uint64_t gate_in = 0, gate_out = 0, gate_row_bytes = 0;
-            uint64_t up_in = 0, up_out = 0, up_row_bytes = 0;
-            uint64_t down_in = 0, down_out = 0, down_row_bytes = 0;
-
-            if (!gl->ffn_gate_inp ||
-                !gl->ffn_exp_probs_b ||
-                !gl->ffn_gate_exps ||
-                !gl->ffn_up_exps ||
-                !gl->ffn_down_exps ||
-                !gl->ffn_gate_shexp ||
-                !gl->ffn_up_shexp ||
-                !gl->ffn_down_shexp ||
-                gl->ffn_gate_inp->type != DS4_TENSOR_F32 ||
-                gl->ffn_gate_inp->dim[0] != DS4_N_EMBD ||
-                gl->ffn_gate_inp->dim[1] != DS4_N_EXPERT ||
-                gl->ffn_exp_probs_b->type != DS4_TENSOR_F32 ||
-                gl->ffn_exp_probs_b->dim[0] != DS4_N_EXPERT ||
-                !gl_gate_pair_supported ||
-                !glm_graph_down_type_supported(gl->ffn_down_exps->type) ||
-                gl->ffn_gate_shexp->type != DS4_TENSOR_Q8_0 ||
-                gl->ffn_up_shexp->type != DS4_TENSOR_Q8_0 ||
-                gl->ffn_down_shexp->type != DS4_TENSOR_Q8_0 ||
-                gl->ffn_gate_shexp->dim[0] != DS4_N_EMBD ||
-                gl->ffn_gate_shexp->dim[1] != DS4_N_FF_EXP ||
-                gl->ffn_up_shexp->dim[0] != DS4_N_EMBD ||
-                gl->ffn_up_shexp->dim[1] != DS4_N_FF_EXP ||
-                gl->ffn_down_shexp->dim[0] != DS4_N_FF_EXP ||
-                gl->ffn_down_shexp->dim[1] != DS4_N_EMBD ||
-                sparse_mid_elems > ffn_mid_elems) {
-                fprintf(stderr,
-                        "ds4: GLM Metal first-token path found unexpected sparse FFN layout in layer %u\n",
-                        il);
-                ok = 0;
-                break;
-            }
-
-            (void)tensor_expert_bytes(model, gl->ffn_gate_exps, 0,
-                                      &gate_in, &gate_out, &gate_row_bytes);
-            (void)tensor_expert_bytes(model, gl->ffn_up_exps, 0,
-                                      &up_in, &up_out, &up_row_bytes);
-            (void)tensor_expert_bytes(model, gl->ffn_down_exps, 0,
-                                      &down_in, &down_out, &down_row_bytes);
-            if (gate_in != DS4_N_EMBD ||
-                up_in != DS4_N_EMBD ||
-                down_in != DS4_N_FF_EXP ||
-                gate_out != DS4_N_FF_EXP ||
-                up_out != DS4_N_FF_EXP ||
-                down_out != DS4_N_EMBD) {
-                fprintf(stderr,
-                        "ds4: GLM Metal first-token path found unexpected expert strides in layer %u\n",
-                        il);
-                ok = 0;
-                break;
-            }
-
-            if (ok) ok = ds4_gpu_matmul_f32_tensor(router_logits,
-                                                   model->map,
-                                                   model->size,
-                                                   gl->ffn_gate_inp->abs_offset,
-                                                   DS4_N_EMBD,
-                                                   DS4_N_EXPERT,
-                                                   ffn_norm,
-                                                   1);
-            if (ok) ok = ds4_gpu_glm_router_select_tensor(router_selected,
-                                                          router_weights,
-                                                          router_probs,
-                                                          model->map,
-                                                          model->size,
-                                                          gl->ffn_exp_probs_b->abs_offset,
-                                                          router_logits,
-                                                          DS4_N_EXPERT,
-                                                          DS4_N_EXPERT_USED,
-                                                          DS4_EXPERT_WEIGHT_SCALE);
-            if (ok) {
-                const ds4_gpu_stream_expert_table table = {
-                    .model_map = model->map,
-                    .model_size = model->size,
-                    .layer = il,
-                    .n_total_expert = DS4_N_EXPERT,
-                    .gate_offset = gl->ffn_gate_exps->abs_offset,
-                    .up_offset = gl->ffn_up_exps->abs_offset,
-                    .down_offset = gl->ffn_down_exps->abs_offset,
-                    .gate_expert_bytes = gate_out * gate_row_bytes,
-                    .down_expert_bytes = down_out * down_row_bytes,
-                };
-                ok = ds4_gpu_glm_stream_expert_cache_begin_selected_load_tensor(
-                        &table,
-                        router_selected,
-                        DS4_N_EXPERT_USED) != 0;
-            }
-            ds4_glm_gpu_graph route_g = {
-                .routed_gate = routed_gate,
-                .routed_up = routed_up,
-                .routed_down = routed_down,
-                .ssd_streaming = false,
-                .glm53 = ds4_model_is_glm53(),
-            };
-            if (ok) ok = glm_graph_routed_moe_one_dispatch(
-                    &route_g,
-                    model,
-                    gl,
-                    il,
-                    ffn_out,
-                    ffn_mid,
-                    gate_out * gate_row_bytes,
-                    gate_row_bytes,
-                    up_out * up_row_bytes,
-                    up_row_bytes,
-                    down_out * down_row_bytes,
-                    down_row_bytes,
-                    router_selected,
-                    router_weights,
-                    ffn_norm,
-                    false);
-            if (ok) ok = ds4_gpu_shared_gate_up_swiglu_q8_0_tensor(
-                    ffn_gate,
-                    ffn_up,
-                    ffn_mid,
-                    model->map,
-                    model->size,
-                    gl->ffn_gate_shexp->abs_offset,
-                    gl->ffn_up_shexp->abs_offset,
-                    DS4_N_EMBD,
-                    DS4_N_FF_EXP,
-                    ffn_norm,
-                    DS4_SWIGLU_CLAMP_EXP);
-            if (ok) ok = ds4_gpu_matmul_q8_0_tensor(ffn_sum,
-                                                    model->map,
-                                                    model->size,
-                                                    gl->ffn_down_shexp->abs_offset,
-                                                    DS4_N_FF_EXP,
-                                                    DS4_N_EMBD,
-                                                    ffn_mid,
-                                                    1);
-            if (ok) ok = ds4_gpu_add_tensor(attn_out, ffn_out, ffn_sum, DS4_N_EMBD);
-            if (ok) ok = ds4_gpu_add_tensor(next, after_attn, attn_out, DS4_N_EMBD);
-        }
-
-        if (ok) {
-            ds4_gpu_tensor *tmp = cur;
-            cur = next;
-            next = tmp;
-        }
-    }
-    if (ok) ok = ds4_gpu_rms_norm_weight_tensor(ffn_norm, cur,
-                                                model->map, model->size,
-                                                weights->output_norm->abs_offset,
-                                                DS4_N_EMBD, DS4_RMS_EPS);
-    if (ok) ok = ds4_gpu_matmul_q8_0_tensor(logits,
-                                            model->map,
-                                            model->size,
-                                            weights->output->abs_offset,
-                                            DS4_N_EMBD,
-                                            DS4_N_VOCAB,
-                                            ffn_norm,
-                                            1);
-    if (ok) ok = ds4_gpu_tensor_read(logits, 0, logits_out, logits_bytes) != 0;
-
-    ds4_gpu_tensor_free(router_weights);
-    ds4_gpu_tensor_free(router_selected);
-    ds4_gpu_tensor_free(router_probs);
-    ds4_gpu_tensor_free(router_logits);
-    ds4_gpu_tensor_free(logits);
-    ds4_gpu_tensor_free(next);
-    ds4_gpu_tensor_free(ffn_sum);
-    ds4_gpu_tensor_free(ffn_out);
-    ds4_gpu_tensor_free(routed_down);
-    ds4_gpu_tensor_free(routed_up);
-    ds4_gpu_tensor_free(routed_gate);
-    ds4_gpu_tensor_free(ffn_mid);
-    ds4_gpu_tensor_free(ffn_up);
-    ds4_gpu_tensor_free(ffn_gate);
-    ds4_gpu_tensor_free(ffn_norm);
-    ds4_gpu_tensor_free(after_attn);
-    ds4_gpu_tensor_free(attn_out);
-    ds4_gpu_tensor_free(heads);
-    ds4_gpu_tensor_free(kv_norm);
-    ds4_gpu_tensor_free(kv_raw);
-    ds4_gpu_tensor_free(attn_norm);
-    ds4_gpu_tensor_free(cur);
-    return ok ? 0 : 1;
-}
-
-static DS4_MAYBE_UNUSED int generate_glm_metal_first_token(
-        const ds4_model   * model,
-        const ds4_vocab   * vocab,
-        const ds4_weights * weights,
-        const token_vec   * prompt,
-        int                 n_predict,
-        int                 ctx_size,
-        ds4_token_emit_fn   emit,
-        ds4_generation_done_fn done,
-        void              * emit_ud) {
-    fprintf(stderr, "ds4: using GLM Metal first-token generation path\n");
-
-    if (prompt->len != 1 || prompt->len > ctx_size) {
-        fprintf(stderr,
-                "ds4: GLM Metal generation currently supports exactly one prompt token; "
-                "multi-token prefill needs the GLM KV/DSA graph\n");
-        return 1;
-    }
-    if (n_predict <= 0) {
-        if (done) done(emit_ud);
-        return 0;
-    }
-    if (n_predict > 1) {
-        fprintf(stderr,
-                "ds4: GLM Metal generation currently emits only the first generated token; "
-                "stopping after one token\n");
-    }
-
-    float *logits = xmalloc((size_t)DS4_N_VOCAB * sizeof(logits[0]));
-    const double t0 = now_sec();
-    const int rc = glm_metal_first_token_logits(model, weights, prompt->v[0], logits);
-    const double t1 = now_sec();
-    if (rc != 0) {
-        free(logits);
-        return 1;
-    }
-
-    if (getenv("DS4_TRACE_TOP") != NULL) {
-        print_top_logits(stderr, "GLM first-token", vocab, logits, DS4_N_VOCAB, 10);
-    }
-    const int token = sample_argmax(logits, DS4_N_VOCAB);
-    if (!vocab_token_is_generation_stop(vocab, token) && emit) emit(emit_ud, token);
-    if (done) done(emit_ud);
-
-    const double eval_s = t1 - t0;
-    ds4_log(stderr,
-            DS4_LOG_TIMING,
-            "ds4: GLM first-token eval: %.2f t/s\n",
-            eval_s > 0.0 ? 1.0 / eval_s : 0.0);
-
-    free(logits);
-    return 0;
-}
-
 static int generate_glm_metal_argmax(
         const ds4_model   * model,
         const ds4_vocab   * vocab,
@@ -53270,6 +55847,12 @@ static int generate_glm_metal_argmax(
     return 0;
 }
 
+static int ds4_prepare_q4_attn_q_b_sidecars(
+        const ds4_model *model,
+        const ds4_weights *weights,
+        uint32_t max_batch_rows,
+        uint64_t working_set_reserve_bytes);
+
 /* Metal generation entry point.  The model runs as one local whole-graph
  * pipeline: graph prefill followed by graph decode steps.  Streaming PRO may
  * use decode-style prefill for short prompts. */
@@ -53286,6 +55869,7 @@ static int generate_metal_graph_raw_swa(
         uint32_t            ssd_streaming_preload_experts,
         uint64_t            ssd_streaming_cache_bytes,
         uint64_t            ssd_streaming_prefill_headroom_bytes,
+        uint64_t            q4_sidecar_working_set_reserve_bytes,
         int                 power_percent,
         uint32_t            prefill_chunk,
         const char        * directional_steering_file,
@@ -53365,9 +55949,40 @@ static int generate_metal_graph_raw_swa(
         metal_graph_free(&g);
         return 1;
     }
+    /* This frontend knows the real prompt width.  Let the backend prepare
+     * either its reusable transient scratch or explicit resident sidecars
+     * after graph allocation is visible, but before warmup and the measured
+     * prefill window. */
+    const uint32_t q4_sidecar_rows =
+        (uint32_t)prompt->len < prefill_cap ?
+            (uint32_t)prompt->len : prefill_cap;
+    if (ds4_prepare_q4_attn_q_b_sidecars(
+            model, weights, q4_sidecar_rows,
+            q4_sidecar_working_set_reserve_bytes) < 0) {
+        fprintf(stderr,
+                "ds4: required GPU Q4 attn_q_b F16 acceleration preflight "
+                "could not be completed\n");
+        metal_graph_free(&g);
+        return 1;
+    }
     const bool memory_report = getenv("DS4_METAL_MEMORY_REPORT") != NULL;
     if (memory_report) ds4_gpu_print_memory_report("after graph alloc");
 
+    /* Match the session frontend's bounded first-submission warmup.  Without
+     * this call the legacy greedy frontend reaches the warmup from inside
+     * metal_graph_prefill_layer_major() and uses the whole prompt width.  The
+     * scratch result is overwritten by the real prefill, so warming thousands
+     * of rows is redundant work as well as making its timing incomparable to
+     * the session/DSpark frontend. */
+    const uint32_t warmup_rows = prefill_cap < 32u ? prefill_cap : 32u;
+    if (!metal_graph_warmup_prefill_kernels(&g,
+                                            model,
+                                            weights,
+                                            warmup_rows)) {
+        metal_graph_free(&g);
+        return 1;
+    }
+
     float *logits = xmalloc((size_t)DS4_N_VOCAB * sizeof(logits[0]));
     const bool trace_top = getenv("DS4_TRACE_TOP") != NULL;
     const bool token_timing = getenv("DS4_TOKEN_TIMING") != NULL;
@@ -53406,7 +56021,25 @@ static int generate_metal_graph_raw_swa(
     int pos = prompt->len;
     int n_generated = 0;
     int n_decode_eval = 0;
+    /* Greedy decode needs only the winning id after each graph evaluation.
+     * Resident and SSD-mapped paths can both reduce on the device and transfer
+     * one int32 instead of DS4_N_VOCAB floats; diagnostics retain full logits.
+     * CUDA and ROCm already have a dedicated top-1 kernel, while Metal selects
+     * its two-stage decode reduction. */
+    const bool greedy_top1_readback =
+        !quality &&
+        g.tp_world < 2u &&
+        !trace_top &&
+        !token_timing &&
+        !graph_power_throttle_enabled(&g) &&
+        getenv("DS4_METAL_GRAPH_TOKEN_PROFILE") == NULL &&
+        getenv("DS4_ROCM_GRAPH_TOKEN_PROFILE") == NULL &&
+        getenv("DS4_DISABLE_GREEDY_TOP1_READBACK") == NULL;
     const double t_decode0 = now_sec();
+    /* Both arms need the initial CPU selection from prefill logits.  Keep it
+     * inside the same decode timing window so short rollback A/B runs have an
+     * identical perimeter. */
+    int next_token = sample_argmax(logits, DS4_N_VOCAB);
     for (int i = 0; i < n_predict && pos < ctx_size; i++) {
         if (trace_top) {
             char label[64];
@@ -53414,7 +56047,7 @@ static int generate_metal_graph_raw_swa(
             print_top_logits(stderr, label, vocab, logits, DS4_N_VOCAB, 10);
         }
 
-        int token = sample_argmax(logits, DS4_N_VOCAB);
+        const int token = next_token;
         if (vocab_token_is_generation_stop(vocab, token)) break;
 
         if (emit) emit(emit_ud, token);
@@ -53426,17 +56059,33 @@ static int generate_metal_graph_raw_swa(
         }
 
         const double t_eval0 = token_timing ? now_sec() : 0.0;
-        ok = metal_graph_eval_token_raw_swa(&g,
-                                            model,
-                                            weights,
-                                            (uint32_t)token,
-                                            (uint32_t)pos,
-                                            logits);
+        if (greedy_top1_readback) {
+            ok = metal_graph_eval_token_raw_swa_top(&g,
+                                                    model,
+                                                    weights,
+                                                    token,
+                                                    (uint32_t)pos,
+                                                    &next_token,
+                                                    NULL,
+                                                    false,
+                                                    NULL,
+                                                    false);
+        } else {
+            ok = metal_graph_eval_token_raw_swa(&g,
+                                                model,
+                                                weights,
+                                                (uint32_t)token,
+                                                (uint32_t)pos,
+                                                logits);
+        }
         if (!ok) break;
         if (token_timing) {
             const double t_eval1 = now_sec();
             fprintf(stderr, "ds4: gpu decode eval %d took %.3f ms\n", n_decode_eval + 1, (t_eval1 - t_eval0) * 1000.0);
         }
+        if (!greedy_top1_readback) {
+            next_token = sample_argmax(logits, DS4_N_VOCAB);
+        }
         n_decode_eval++;
         pos++;
     }
@@ -54109,14 +56758,61 @@ typedef struct ds4_dspark_spec_stats {
     uint64_t direct_full_commits;
     uint64_t direct_partial_commits;
     uint64_t replay_fallbacks;
+    uint64_t proposer_capped;
+    uint64_t proposer_scheduled_rows;
+    uint64_t exact2_attempts;
+    uint64_t exact2_full_accepts;
+    uint64_t exact2_partial_accepts;
+    uint64_t exact2_fallbacks;
+    uint64_t cuda_exactn_attempts;
+    uint64_t cuda_exactn_full_accepts;
+    uint64_t cuda_exactn_fallbacks;
+    uint64_t cuda_exactn_partial_fallbacks;
+    uint64_t cuda_exactn_error_fallbacks;
+    uint64_t cuda_exactn_rows;
+    uint64_t cuda_exactn_partial_replays;
+    uint64_t cuda_exactn_legacy_verify_skips;
+    uint64_t cuda_exactn_batch_head_attempts;
+    uint64_t cuda_exactn_batch_head_uses;
+    uint64_t cuda_exactn_batch_head_fallbacks;
+    uint64_t cuda_exactn_graph_attempts;
+    uint64_t cuda_exactn_graph_uses;
+    uint64_t cuda_exactn_graph_captures;
+    uint64_t cuda_exactn_graph_replays;
+    uint64_t cuda_exactn_graph_warms;
+    uint64_t cuda_exactn_graph_no_slots;
+    uint64_t cuda_exactn_graph_failures;
+    uint64_t cuda_device_proposer_attempts;
+    uint64_t cuda_device_proposer_uses;
+    uint64_t cuda_device_proposer_fallbacks;
+    uint64_t cuda_device_proposer_policy_mismatches;
+    uint64_t metal_device_proposer_attempts;
+    uint64_t metal_device_proposer_uses;
+    uint64_t metal_device_proposer_fallbacks;
+    uint64_t metal_device_proposer_policy_mismatches;
+    uint64_t exactn_union_attempts;
+    uint64_t exactn_union_full_accepts;
+    uint64_t exactn_union_fallbacks;
+    uint64_t exactn_union_partial_fallbacks;
+    uint64_t exactn_union_error_fallbacks;
+    uint64_t exactn_union_partial_replays;
+    uint64_t exactn_union_legacy_verify_skips;
+    uint64_t metal_exactn_batch_head_attempts;
+    uint64_t metal_exactn_batch_head_uses;
+    uint64_t metal_exactn_batch_head_fallbacks;
+    uint64_t exactn_attempts;
+    uint64_t exactn_full_accepts;
+    uint64_t exactn_partial_accepts;
+    uint64_t exactn_fallbacks;
+    uint64_t exactn_partial_fallbacks;
+    uint64_t exactn_error_fallbacks;
+    uint64_t exactn_boundary_rows;
     uint64_t first_misses;
     uint64_t no_draft;
     uint64_t no_room;
     uint64_t invalid_draft;
     uint64_t draft_len_hist[DS4_DSPARK_MAX_BLOCK_SIZE + 1u];
     uint64_t accepted_len_hist[DS4_DSPARK_MAX_BLOCK_SIZE + 1u];
-    uint64_t scheduler_skips;
-    uint64_t tail_skips;
     uint64_t verifier_unavailable;
     uint64_t verifier_errors;
     double target_ms;
@@ -54137,7 +56833,19 @@ typedef struct ds4_dspark_spec_stats {
     double verify_layer_ms;
     double verify_head_ms;
     double verify_read_ms;
+    double cuda_exactn_ms;
+    double cuda_exactn_setup_ms;
+    double cuda_exactn_layer_ms;
+    double cuda_exactn_head_ms;
+    double cuda_exactn_read_ms;
+    double cuda_exactn_restore_ms;
+    double cuda_exactn_legacy_verify_ms;
+    double cuda_exactn_partial_replay_ms;
+    double exactn_union_partial_replay_ms;
     uint64_t verifier_fused_head;
+    uint64_t metal_acceptance_only_attempts;
+    uint64_t metal_acceptance_only_rows_saved;
+    uint64_t metal_replay_headless_tokens;
     double replay_ms;
     double total_ms;
 } ds4_dspark_spec_stats;
@@ -54197,26 +56905,11 @@ struct ds4_session {
 #ifndef DS4_NO_GPU
     int dspark_draft_tokens[DS4_DSPARK_MAX_BLOCK_SIZE];
     uint32_t dspark_draft_len;
-    uint32_t dspark_sched_cycles;
-    uint32_t dspark_sched_accepted;
-    uint32_t dspark_sched_no_draft;
-    uint32_t dspark_sched_skip;
-    uint32_t dspark_sched_lifetime_accepted;
-    double dspark_sched_life_extra_ms;
-    double dspark_sched_life_saved_ms;
-    double dspark_sched_extra_ms;
-    double dspark_sched_saved_ms;
     double dspark_last_target_eval_ms;
-    double dspark_last_propose_ms;
-    float dspark_last_confidence0;
     float dspark_sample_temperature;
     uint64_t *dspark_sample_rng;
     bool dspark_draft_valid;
     bool dspark_stochastic_draft;
-    bool dspark_sched_skipped_cycle;
-    bool dspark_sched_long_accept_seen;
-    bool dspark_sched_bypass;
-    bool dspark_last_confidence0_valid;
     ds4_dspark_spec_stats dspark_stats;
 #endif
     uint64_t mtp_probe_total;
@@ -54229,11 +56922,18 @@ struct ds4_session {
     void *cancel_ud;
     uint32_t prefill_cap;
     int ctx_size;
+    bool engine_session_counted;
+#ifndef DS4_NO_GPU
+    uint64_t q4_attn_q_b_f16_sidecars_generation;
+    uint32_t q4_attn_q_b_f16_prepared_rows;
+#endif
     bool checkpoint_valid;
     bool mtp_draft_valid;
     bool greedy_splitkv_anchor_valid;
 };
 
+enum { DS4_METAL_SESSION_STREAMS = 8 };
+
 #ifndef DS4_NO_GPU
 static bool ds4_dspark_stats_enabled(void);
 
@@ -54268,279 +56968,342 @@ static bool ds4_session_dspark_seed_batch_enabled(
     return ds4_session_dspark_rocm_gfx1151_fast_path(s);
 }
 
-static bool ds4_dspark_scheduler_enabled(const ds4_session *s) {
-    const char *env = getenv("DS4_DSPARK_SCHEDULER");
-    if (env && env[0]) return strcmp(env, "0") != 0;
+/* Exact-2 is currently a resident, single-GPU CUDA experiment.  Keep its hard
+ * backend eligibility in one place; proposal and verification widths remain
+ * independently controllable for attribution tests. */
+static bool ds4_session_cuda_dspark_exact2_enabled(
+        const ds4_session *s) {
+#if !defined(__APPLE__) && !defined(DS4_ROCM_BUILD)
+    return s && s->engine &&
+           s->engine->backend == DS4_BACKEND_CUDA &&
+           !s->engine->multi_tier &&
+           !s->engine->tp.active &&
+           !s->graph.ssd_streaming &&
+           s->graph.placement == NULL &&
+           s->graph.prefill_cap >= 3u &&
+           metal_graph_tp_env_flag("DS4_CUDA_DSPARK_EXACT2", false);
+#else
     (void)s;
-    return true;
+    return false;
+#endif
 }
 
-static uint32_t ds4_dspark_scheduler_window(const ds4_session *s) {
-    const uint32_t fallback =
-        ds4_session_dspark_rocm_gfx1151_fast_path(s) ? 16u : 4u;
-    uint32_t v = ds4_dspark_env_u32("DS4_DSPARK_SCHEDULER_WINDOW", fallback);
-    return v ? v : 4;
+/* Resident single-GPU exact-N is deliberately independent of exact-2 so the
+ * proposal width and verifier state transition can be attributed separately
+ * on CUDA hardware.  It is opt-in and has an explicit emergency kill switch;
+ * neither variable changes Metal or ROCm behavior. */
+static bool ds4_session_cuda_dspark_exactn_enabled(
+        const ds4_session *s) {
+#if !defined(__APPLE__) && !defined(DS4_ROCM_BUILD)
+    return s && s->engine &&
+           s->engine->backend == DS4_BACKEND_CUDA &&
+           !s->engine->multi_tier &&
+           !s->engine->tp.active &&
+           !s->graph.ssd_streaming &&
+           s->graph.placement == NULL &&
+           s->graph.prefill_cap >= 3u &&
+           metal_graph_tp_env_flag("DS4_CUDA_DSPARK_EXACTN", false) &&
+           !metal_graph_tp_env_flag(
+                   "DS4_CUDA_DISABLE_DSPARK_EXACTN", false);
+#else
+    (void)s;
+    return false;
+#endif
 }
 
-static uint32_t ds4_dspark_scheduler_skip_cycles(void) {
-    const uint32_t fallback =
-        ds4_dspark_rocm_gfx1151_fast_path() ? 4u : 2u;
-    return ds4_dspark_env_u32("DS4_DSPARK_SCHEDULER_SKIP", fallback);
+/* Metal exact-2 reuses the same decode-exact body as CUDA, but starts as an
+ * SSD-only opt-in until its selected-expert barriers and throughput are
+ * measured on real Apple hardware. */
+static bool ds4_session_metal_dspark_exact2_enabled(
+        const ds4_session *s) {
+#if defined(__APPLE__)
+    return s && s->engine &&
+           s->engine->backend == DS4_BACKEND_METAL &&
+           !s->engine->multi_tier &&
+           !s->engine->tp.active &&
+           s->graph.ssd_streaming &&
+           s->graph.placement == NULL &&
+           s->graph.prefill_cap >= 3u &&
+           metal_graph_tp_env_flag("DS4_METAL_DSPARK_EXACT2", false);
+#else
+    (void)s;
+    return false;
+#endif
 }
 
-static uint32_t ds4_dspark_scheduler_slow_skip_cycles(void) {
-    return ds4_dspark_env_u32("DS4_DSPARK_SCHEDULER_SLOW_SKIP", 4);
+/* Correctness oracle for a future exact Metal microbatch.  It deliberately
+ * advances each row through the ordinary one-token SSD decode entry point,
+ * whose command completion and logits readback form a hard boundary between
+ * rows.  Only a fully accepted block is committed directly; partial blocks
+ * are rolled back and left to the established verifier/replay path. */
+static bool ds4_session_metal_dspark_exactn_enabled(
+        const ds4_session *s) {
+#if defined(__APPLE__)
+    return s && s->engine &&
+           s->engine->backend == DS4_BACKEND_METAL &&
+           !s->engine->multi_tier &&
+           !s->engine->tp.active &&
+           s->graph.ssd_streaming &&
+           s->graph.placement == NULL &&
+           metal_graph_tp_env_flag("DS4_METAL_DSPARK_EXACTN", false);
+#else
+    (void)s;
+    return false;
+#endif
 }
 
-static uint32_t ds4_dspark_scheduler_min_avg_milli(void) {
-    const uint32_t fallback =
-        ds4_dspark_rocm_gfx1151_fast_path() ? 4000u : 1500u;
-    return ds4_dspark_env_u32(
-            "DS4_DSPARK_SCHEDULER_MIN_AVG_MILLI", fallback);
+/* Fast exact-N experiment: keep the canonical one-token tape, but split every
+ * layer at the router so all speculative rows share one immutable selected-
+ * expert union.  It stays independent of the boundary oracle until both state
+ * and output comparisons pass on real Metal hardware. */
+static bool ds4_session_metal_dspark_exactn_union_enabled(
+        const ds4_session *s) {
+#if defined(__APPLE__)
+    return s && s->engine &&
+           s->engine->backend == DS4_BACKEND_METAL &&
+           !s->engine->multi_tier &&
+           !s->engine->tp.active &&
+           s->graph.ssd_streaming &&
+           s->graph.placement == NULL &&
+           metal_graph_tp_env_flag(
+                   "DS4_METAL_DSPARK_EXACTN_UNION", false);
+#else
+    (void)s;
+    return false;
+#endif
 }
 
-static uint32_t ds4_dspark_scheduler_max_ms_per_accept_milli(void) {
-    return ds4_dspark_env_u32(
-            "DS4_DSPARK_SCHEDULER_MAX_MS_PER_ACCEPT_MILLI", 0);
-}
+/* Keep the proposer width independently controllable from the verifier so a
+ * CUDA A/B can attribute the gain.  An explicit zero keeps the checkpoint's
+ * native width; otherwise exact-2 defaults to two rows and exact-N to the
+ * largest resident width it can consume (at most five). */
+static uint32_t ds4_session_cuda_dspark_proposer_block_cap(
+        const ds4_session *s,
+        uint32_t           native_block_size) {
+#if !defined(__APPLE__) && !defined(DS4_ROCM_BUILD)
+    if (!s || !s->engine || native_block_size == 0 ||
+        s->engine->backend != DS4_BACKEND_CUDA ||
+        s->engine->multi_tier || s->engine->tp.active ||
+        s->graph.ssd_streaming || s->graph.placement != NULL) {
+        return native_block_size;
+    }
 
-static uint32_t ds4_dspark_scheduler_max_extra_saved_ratio_milli(void) {
-    return ds4_dspark_env_u32(
-            "DS4_DSPARK_SCHEDULER_MAX_EXTRA_SAVED_RATIO_MILLI", 0);
-}
+    const char *env = getenv("DS4_CUDA_DSPARK_PROPOSER_BLOCK_MAX");
+    if (env && env[0]) {
+        char *end = NULL;
+        errno = 0;
+        const unsigned long value = strtoul(env, &end, 10);
+        if (end != env && *end == '\0' && errno == 0 &&
+            value <= UINT32_MAX) {
+            if (value == 0) return native_block_size;
+            uint32_t cap = (uint32_t)value;
+            if (cap > DS4_DSPARK_MAX_BLOCK_SIZE) {
+                cap = DS4_DSPARK_MAX_BLOCK_SIZE;
+            }
+            return cap < native_block_size ? cap : native_block_size;
+        }
+        return native_block_size;
+    }
 
-static uint32_t ds4_dspark_scheduler_break_even_window(void) {
-    return ds4_dspark_env_u32("DS4_DSPARK_SCHEDULER_BREAK_EVEN_WINDOW", 0);
-}
+    if (ds4_session_cuda_dspark_exactn_enabled(s)) {
+        /* The support stage carries the current target row plus every draft,
+         * so a D-row proposal needs D+1 rows of prefill workspace. */
+        uint32_t cap = s->graph.prefill_cap - 1u;
+        if (cap > 5u) cap = 5u;
+        return cap < native_block_size ? cap : native_block_size;
+    }
 
-static uint32_t ds4_dspark_scheduler_no_draft_skip_cycles(void) {
-    return ds4_dspark_env_u32("DS4_DSPARK_SCHEDULER_NO_DRAFT_SKIP", 3);
+    const uint32_t verify_cap = ds4_dspark_env_u32(
+            "DS4_DSPARK_SSD_VERIFY_BLOCK_MAX", 0);
+    if (s->graph.prefill_cap >= 3u &&
+        ds4_session_cuda_dspark_exact2_enabled(s) &&
+        (verify_cap == 0u || verify_cap == 2u)) {
+        return native_block_size > 2u ? 2u : native_block_size;
+    }
+#else
+    (void)s;
+#endif
+    return native_block_size;
+}
+
+/* A tiny SSD verifier may touch top-k experts for every speculative row.  On
+ * low-memory Metal systems the useful upper bound is the number of complete
+ * top-k rows that fit in the configured target expert cache.  The Metal
+ * proposer normally inherits this limit below, while its explicit zero
+ * override preserves the old full-proposal/capped-verifier A/B path. */
+static uint32_t ds4_session_dspark_verify_block_cap(const ds4_session *s) {
+    uint32_t cap = ds4_dspark_env_u32(
+            "DS4_DSPARK_SSD_VERIFY_BLOCK_MAX", 0);
+    if (cap == 0 && ds4_session_cuda_dspark_exactn_enabled(s)) {
+        cap = s->graph.prefill_cap;
+        if (cap > 5u) cap = 5u;
+    }
+    /* Exact-2 defaults verification to two rows.  The independent proposer
+     * cap may preserve the checkpoint's native proposal width for A/B tests;
+     * an explicit verifier cap still takes precedence. */
+    if (cap == 0 &&
+        !ds4_session_metal_dspark_exactn_enabled(s) &&
+        (ds4_session_cuda_dspark_exact2_enabled(s) ||
+         ds4_session_metal_dspark_exact2_enabled(s))) {
+        cap = 2;
+    }
+    if (cap == 0 && s && s->engine &&
+        s->engine->backend == DS4_BACKEND_METAL &&
+        s->graph.ssd_streaming) {
+        const uint32_t slots =
+            ds4_gpu_stream_expert_cache_configured_count();
+        cap = DS4_N_EXPERT_USED != 0 ? slots / DS4_N_EXPERT_USED : 0;
+        if (cap == 0) cap = 1;
+    }
+    if (cap == 0 || cap > DS4_DSPARK_MAX_BLOCK_SIZE) {
+        cap = DS4_DSPARK_MAX_BLOCK_SIZE;
+    }
+    return cap;
 }
 
-static uint32_t ds4_dspark_scheduler_short_accept_no_draft_skip_cycles(void) {
-    return ds4_dspark_env_u32("DS4_DSPARK_SCHEDULER_SHORT_ACCEPT_NO_DRAFT_SKIP", 4);
+/* On single-device Metal, schedule only rows the target verifier can consume.
+ * The automatic SSD cap is cache_slots / top_k (so 16 slots schedule two rows,
+ * while 30/32 slots retain a native five-row DSpark block).  An explicit
+ * DS4_METAL_DSPARK_PROPOSER_BLOCK_MAX overrides that policy; zero is the A/B
+ * kill switch and restores the checkpoint's native proposal width. */
+static uint32_t ds4_session_metal_dspark_proposer_block_cap(
+        const ds4_session *s,
+        uint32_t           native_block_size) {
+#if defined(__APPLE__)
+    if (!s || !s->engine || native_block_size == 0 ||
+        s->engine->backend != DS4_BACKEND_METAL ||
+        s->engine->multi_tier || s->engine->tp.active ||
+        s->graph.placement != NULL) {
+        return native_block_size;
+    }
+
+    const char *env = getenv("DS4_METAL_DSPARK_PROPOSER_BLOCK_MAX");
+    uint32_t cap = 0;
+    if (env && env[0]) {
+        char *end = NULL;
+        errno = 0;
+        const unsigned long value = strtoul(env, &end, 10);
+        if (end == env || *end != '\0' || errno != 0 ||
+            value > UINT32_MAX) {
+            return native_block_size;
+        }
+        if (value == 0) return native_block_size;
+        cap = (uint32_t)value;
+        if (cap > DS4_DSPARK_MAX_BLOCK_SIZE) {
+            cap = DS4_DSPARK_MAX_BLOCK_SIZE;
+        }
+    } else {
+        cap = ds4_session_dspark_verify_block_cap(s);
+    }
+    return cap < native_block_size ? cap : native_block_size;
+#else
+    (void)s;
+    return native_block_size;
+#endif
 }
 
-static uint32_t ds4_dspark_scheduler_cold_low_confidence_skip_cycles(void) {
-    return ds4_dspark_env_u32("DS4_DSPARK_SCHEDULER_COLD_LOW_CONFIDENCE_SKIP", 7);
+/* Intermediate accepted tokens only advance target state; their logits are
+ * discarded.  Single-device Metal can therefore omit those output heads and
+ * readbacks.  Keep an explicit switch for byte-for-byte A/B diagnostics. */
+static bool ds4_session_metal_dspark_headless_replay_enabled(
+        const ds4_session *s) {
+#if defined(__APPLE__)
+    return s && s->engine &&
+           s->engine->backend == DS4_BACKEND_METAL &&
+           !s->engine->multi_tier && !s->engine->tp.active &&
+           s->graph.placement == NULL &&
+           metal_graph_tp_env_flag(
+                   "DS4_METAL_DSPARK_HEADLESS_REPLAY", true);
+#else
+    (void)s;
+    return false;
+#endif
 }
 
-static uint32_t ds4_dspark_scheduler_tail_min_tokens(void) {
-    return ds4_dspark_env_u32("DS4_DSPARK_SCHEDULER_TAIL_MIN_TOKENS", 10);
+/* The base target logits have already accepted draft[0].  A single-device
+ * Metal cycle that unconditionally rolls verification back can therefore
+ * evaluate only draft[0..N-2] and still obtain every remaining acceptance
+ * top. Keep the contraction opt-in: on the M1 Pro SSD path the smaller batch
+ * did not beat the legacy N-row verifier despite doing less row work. */
+static bool ds4_session_metal_dspark_acceptance_only_verify_enabled(
+        const ds4_session *s) {
+#if defined(__APPLE__)
+    return s && s->engine &&
+           s->engine->backend == DS4_BACKEND_METAL &&
+           !s->engine->multi_tier && !s->engine->tp.active &&
+           s->graph.placement == NULL &&
+           metal_graph_tp_env_flag(
+                   "DS4_METAL_DSPARK_ACCEPTANCE_ONLY_VERIFY", false);
+#else
+    (void)s;
+    return false;
+#endif
 }
 
-static float ds4_dspark_scheduler_cold_low_confidence_threshold(void) {
-    return (float)ds4_dspark_env_u32("DS4_DSPARK_SCHEDULER_COLD_LOW_CONFIDENCE_MILLI", 500) / 1000.0f;
+/* Tiny-batch verification writes every future raw row before evaluating the
+ * block.  A deliberately narrow raw ring must not wrap onto rows still in the
+ * visible SWA window.  Default padded rings satisfy this invariant; custom
+ * DS4_METAL_GRAPH_RAW_CAP values may not. */
+static bool ds4_session_dspark_batch_raw_safe(
+        const ds4_session *s,
+        uint32_t           draft_n) {
+    if (!s || draft_n <= 1u) return true;
+    const uint64_t start = (uint64_t)s->checkpoint.len;
+    const uint64_t raw_cap = s->graph.raw_cap;
+    const uint64_t raw_window = s->graph.raw_window;
+    if (raw_cap == 0) return false;
+    if (start + draft_n <= raw_cap) return true;
+    return raw_cap >= raw_window && draft_n <= raw_cap - raw_window;
+}
+
+/* Experimental exact verifier that advances two draft tokens with the
+ * ordinary one-token kernels, layer by layer. CUDA remains resident-only;
+ * the Metal caller installs its complete SSD target map before entering. */
+static bool ds4_session_dspark_exact2_requested(
+        const ds4_session *s,
+        uint32_t           draft_n) {
+    return draft_n == 2u &&
+           (ds4_session_cuda_dspark_exact2_enabled(s) ||
+            ds4_session_metal_dspark_exact2_enabled(s));
 }
 
-/* Timing-sensitive scheduling changes which arithmetic path advances a token.
- * Keep it opt-in so greedy DSpark output is reproducible across runs. */
-static bool ds4_dspark_scheduler_timing_enabled(void) {
-    return ds4_dspark_scheduler_max_ms_per_accept_milli() != 0 ||
-           ds4_dspark_scheduler_max_extra_saved_ratio_milli() != 0;
+static bool ds4_session_cuda_dspark_exactn_requested(
+        const ds4_session *s,
+        uint32_t           draft_n) {
+    return draft_n >= 2u &&
+           draft_n <= DS4_DSPARK_MAX_BLOCK_SIZE &&
+           draft_n <= 5u &&
+           s && draft_n <= s->graph.prefill_cap &&
+           ds4_session_cuda_dspark_exactn_enabled(s);
 }
 
-static void ds4_session_dspark_scheduler_reset(ds4_session *s) {
-    if (!s) return;
-    s->dspark_sched_cycles = 0;
-    s->dspark_sched_accepted = 0;
-    s->dspark_sched_no_draft = 0;
-    s->dspark_sched_extra_ms = 0.0;
-    s->dspark_sched_saved_ms = 0.0;
+static bool ds4_session_metal_dspark_exactn_requested(
+        const ds4_session *s,
+        uint32_t           draft_n) {
+    return draft_n >= 2u &&
+           draft_n <= DS4_DSPARK_MAX_BLOCK_SIZE &&
+           draft_n <= 5u &&
+           ds4_session_metal_dspark_exactn_enabled(s);
 }
 
-static void ds4_session_dspark_scheduler_begin_request(ds4_session *s) {
-    if (!s) return;
-    ds4_session_dspark_scheduler_reset(s);
-    s->dspark_sched_skip = 0;
-    s->dspark_sched_lifetime_accepted = 0;
-    s->dspark_sched_life_extra_ms = 0.0;
-    s->dspark_sched_life_saved_ms = 0.0;
-    s->dspark_sched_skipped_cycle = false;
-    s->dspark_sched_long_accept_seen = false;
-    s->dspark_sched_bypass = false;
-}
-
-static bool ds4_session_dspark_scheduler_should_skip(ds4_session *s) {
-    if (!s || !ds4_dspark_scheduler_enabled(s)) return false;
-    s->dspark_sched_skipped_cycle = false;
-    if (s->dspark_sched_bypass) {
-        s->dspark_sched_skipped_cycle = true;
-        s->dspark_stats.scheduler_skips++;
-        return true;
-    }
-    if (s->dspark_sched_skip == 0) return false;
-    s->dspark_sched_skip--;
-    s->dspark_sched_skipped_cycle = true;
-    s->dspark_stats.scheduler_skips++;
-    if (getenv("DS4_DSPARK_SPEC_LOG") != NULL) {
-        fprintf(stderr,
-                "ds4: DSpark scheduler skip remaining=%u\n",
-                s->dspark_sched_skip);
-    }
-    return true;
+static bool ds4_session_metal_dspark_exactn_union_requested(
+        const ds4_session *s,
+        uint32_t           draft_n) {
+    return draft_n >= 2u &&
+           draft_n <= DS4_DSPARK_MAX_BLOCK_SIZE &&
+           draft_n <= DS4_METAL_EXACTN_UNION_MAX_ROWS &&
+           ds4_session_metal_dspark_exactn_union_enabled(s);
 }
 
-static void ds4_session_dspark_scheduler_note(
+static void ds4_session_dspark_stats_note_saved(
         ds4_session *s,
-        uint32_t     accepted_drafts,
-        bool         no_draft,
-        double       extra_ms) {
-    if (!s || !ds4_dspark_scheduler_enabled(s)) return;
-    if (s->dspark_sched_skipped_cycle) {
-        s->dspark_sched_skipped_cycle = false;
-        return;
-    }
-
-    s->dspark_sched_cycles++;
-    s->dspark_sched_accepted += accepted_drafts;
-    if (accepted_drafts != 0) {
-        if (s->dspark_sched_lifetime_accepted <=
-            UINT32_MAX - accepted_drafts) {
-            s->dspark_sched_lifetime_accepted += accepted_drafts;
-        } else {
-            s->dspark_sched_lifetime_accepted = UINT32_MAX;
-        }
-        if (accepted_drafts > 2u) {
-            s->dspark_sched_long_accept_seen = true;
-        }
-    }
-    if (no_draft) s->dspark_sched_no_draft++;
-    if (extra_ms > 0.0 && isfinite(extra_ms)) {
-        s->dspark_sched_extra_ms += extra_ms;
-    }
-    if (accepted_drafts != 0 &&
-        s->dspark_last_target_eval_ms > 0.0 &&
+        uint32_t     accepted_drafts) {
+    if (!s || accepted_drafts == 0 || !ds4_dspark_stats_enabled()) return;
+    if (s->dspark_last_target_eval_ms > 0.0 &&
         isfinite(s->dspark_last_target_eval_ms)) {
-        const double saved_ms =
+        s->dspark_stats.saved_ms +=
             s->dspark_last_target_eval_ms * (double)accepted_drafts;
-        s->dspark_sched_saved_ms += saved_ms;
-        if (ds4_dspark_stats_enabled()) {
-            s->dspark_stats.saved_ms += saved_ms;
-        }
-    }
-
-    const uint32_t no_draft_skip =
-        ds4_dspark_scheduler_no_draft_skip_cycles();
-    if (no_draft && no_draft_skip != 0) {
-        uint32_t skip = no_draft_skip;
-        if (s->dspark_sched_lifetime_accepted != 0 &&
-            !s->dspark_sched_long_accept_seen) {
-            const uint32_t short_accept_skip =
-                ds4_dspark_scheduler_short_accept_no_draft_skip_cycles();
-            if (skip < short_accept_skip) skip = short_accept_skip;
-        } else if (s->dspark_sched_lifetime_accepted == 0 &&
-                   s->dspark_last_confidence0_valid &&
-                   s->dspark_last_confidence0 <=
-                       ds4_dspark_scheduler_cold_low_confidence_threshold()) {
-            const uint32_t cold_low_conf_skip =
-                ds4_dspark_scheduler_cold_low_confidence_skip_cycles();
-            if (skip < cold_low_conf_skip) skip = cold_low_conf_skip;
-        }
-        if (s->dspark_sched_skip < skip) {
-            s->dspark_sched_skip = skip;
-        }
-        if (getenv("DS4_DSPARK_SPEC_LOG") != NULL) {
-            fprintf(stderr,
-                    "ds4: DSpark scheduler no-draft pause skip=%u "
-                    "accepted_total=%u long_accept=%d confidence0=%s%.3f\n",
-                    s->dspark_sched_skip,
-                    s->dspark_sched_lifetime_accepted,
-                    s->dspark_sched_long_accept_seen ? 1 : 0,
-                    s->dspark_last_confidence0_valid ? "" : "n/a:",
-                    s->dspark_last_confidence0);
-        }
-    }
-
-    const uint32_t window = ds4_dspark_scheduler_window(s);
-    const uint32_t break_even_window =
-        ds4_dspark_scheduler_break_even_window();
-
-    const uint32_t max_extra_saved_ratio_milli =
-        ds4_dspark_scheduler_max_extra_saved_ratio_milli();
-    const bool measured_unprofitable =
-        max_extra_saved_ratio_milli != 0 &&
-        s->dspark_sched_accepted != 0 &&
-        s->dspark_sched_saved_ms > 0.0 &&
-        s->dspark_sched_extra_ms * 1000.0 >
-            s->dspark_sched_saved_ms *
-            (double)max_extra_saved_ratio_milli;
-
-    if (break_even_window != 0 &&
-        s->dspark_sched_cycles >= break_even_window &&
-        measured_unprofitable) {
-        s->dspark_sched_skip = ds4_dspark_scheduler_slow_skip_cycles();
-        if (getenv("DS4_DSPARK_SPEC_LOG") != NULL) {
-            fprintf(stderr,
-                    "ds4: DSpark scheduler break-even pause cycles=%u "
-                    "accepted=%u saved=%.3fms extra=%.3fms skip=%u\n",
-                    s->dspark_sched_cycles,
-                    s->dspark_sched_accepted,
-                    s->dspark_sched_saved_ms,
-                    s->dspark_sched_extra_ms,
-                    s->dspark_sched_skip);
-        }
-        ds4_session_dspark_scheduler_reset(s);
-        return;
-    }
-
-    if (s->dspark_sched_cycles < window) return;
-
-    const uint64_t avg_milli =
-        ((uint64_t)s->dspark_sched_accepted * 1000ull) /
-        (uint64_t)s->dspark_sched_cycles;
-    const uint32_t min_avg_milli =
-        ds4_dspark_scheduler_min_avg_milli();
-    const bool low_accept = avg_milli < min_avg_milli;
-    const bool many_no_draft =
-        s->dspark_sched_no_draft * 2u >= s->dspark_sched_cycles;
-    const uint32_t max_ms_per_accept_milli =
-        ds4_dspark_scheduler_max_ms_per_accept_milli();
-    const double extra_per_accept_ms =
-        s->dspark_sched_accepted != 0 ?
-        s->dspark_sched_extra_ms / (double)s->dspark_sched_accepted : 0.0;
-    const bool slow_accept =
-        max_ms_per_accept_milli != 0 &&
-        s->dspark_sched_accepted != 0 &&
-        extra_per_accept_ms * 1000.0 > (double)max_ms_per_accept_milli;
-    if (low_accept || many_no_draft || slow_accept || measured_unprofitable) {
-        if (ds4_session_dspark_rocm_gfx1151_fast_path(s)) {
-            s->dspark_sched_bypass = true;
-            s->dspark_sched_skip = 0;
-            if (getenv("DS4_DSPARK_SPEC_LOG") != NULL) {
-                fprintf(stderr,
-                        "ds4: DSpark scheduler bypass accepted=%u avg=%.3f "
-                        "no_draft=%u\n",
-                        s->dspark_sched_accepted,
-                        (double)avg_milli / 1000.0,
-                        s->dspark_sched_no_draft);
-            }
-            ds4_session_dspark_scheduler_reset(s);
-            return;
-        }
-        s->dspark_sched_skip = ds4_dspark_scheduler_skip_cycles();
-        if (many_no_draft || slow_accept || measured_unprofitable) {
-            const uint32_t slow_skip = ds4_dspark_scheduler_slow_skip_cycles();
-            if (s->dspark_sched_skip < slow_skip) {
-                s->dspark_sched_skip = slow_skip;
-            }
-        }
-        if (getenv("DS4_DSPARK_SPEC_LOG") != NULL) {
-            fprintf(stderr,
-                    "ds4: DSpark scheduler pause cycles=%u accepted=%u "
-                    "avg=%.3f no_draft=%u extra_per_accept=%.3fms "
-                    "saved=%.3fms extra=%.3fms skip=%u\n",
-                    s->dspark_sched_cycles,
-                    s->dspark_sched_accepted,
-                    (double)avg_milli / 1000.0,
-                    s->dspark_sched_no_draft,
-                    extra_per_accept_ms,
-                    s->dspark_sched_saved_ms,
-                    s->dspark_sched_extra_ms,
-                    s->dspark_sched_skip);
-        }
     }
-    ds4_session_dspark_scheduler_reset(s);
 }
 #endif
 
@@ -56054,6 +58817,30 @@ static void spec_frontier_free(ds4_spec_frontier *f) {
     memset(f, 0, sizeof(*f));
 }
 
+/* Frontier snapshots are a burst of independent, same-device D2D copies.
+ * On resident single-GPU CUDA they can share the default stream and one final
+ * end_commands fence; retaining the synchronous primitive everywhere else is
+ * important because a single final device fence cannot cover multi-tier
+ * streams. */
+static bool spec_frontier_tensor_copy(
+        ds4_session          *s,
+        ds4_gpu_tensor       *dst,
+        const ds4_gpu_tensor *src,
+        uint64_t              bytes) {
+#if !defined(__APPLE__) && !defined(DS4_ROCM_BUILD)
+    if (s && s->engine &&
+        s->engine->backend == DS4_BACKEND_CUDA &&
+        !s->engine->multi_tier && !s->engine->tp.active &&
+        s->graph.placement == NULL) {
+        if (ds4_gpu_set_current_device(0) != 0) return false;
+        return ds4_gpu_tensor_copy_async(dst, src, bytes) != 0;
+    }
+#else
+    (void)s;
+#endif
+    return ds4_gpu_tensor_copy(dst, 0, src, 0, bytes) != 0;
+}
+
 static bool spec_frontier_snapshot(ds4_spec_frontier *f, ds4_session *s) {
     memset(f, 0, sizeof(*f));
     ds4_gpu_graph *g = &s->graph;
@@ -56070,17 +58857,21 @@ static bool spec_frontier_snapshot(ds4_spec_frontier *f, ds4_session *s) {
         const uint32_t ratio = ds4_layer_compress_ratio(il);
         if (ratio == 0) continue;
         const uint64_t ab = ds4_gpu_tensor_bytes(g->layer_attn_state_kv[il]);
-        ok = ds4_gpu_tensor_copy(g->spec_attn_state_kv[il], 0,
-                                   g->layer_attn_state_kv[il], 0, ab) != 0 &&
-             ds4_gpu_tensor_copy(g->spec_attn_state_score[il], 0,
-                                   g->layer_attn_state_score[il], 0, ab) != 0;
+        ok = spec_frontier_tensor_copy(s,
+                                       g->spec_attn_state_kv[il],
+                                       g->layer_attn_state_kv[il], ab) &&
+             spec_frontier_tensor_copy(s,
+                                       g->spec_attn_state_score[il],
+                                       g->layer_attn_state_score[il], ab);
         if (ratio == 4) {
             const uint64_t ib = ds4_gpu_tensor_bytes(g->layer_index_state_kv[il]);
             ok = ok &&
-                 ds4_gpu_tensor_copy(g->spec_index_state_kv[il], 0,
-                                       g->layer_index_state_kv[il], 0, ib) != 0 &&
-                 ds4_gpu_tensor_copy(g->spec_index_state_score[il], 0,
-                                       g->layer_index_state_score[il], 0, ib) != 0;
+                 spec_frontier_tensor_copy(s,
+                                           g->spec_index_state_kv[il],
+                                           g->layer_index_state_kv[il], ib) &&
+                 spec_frontier_tensor_copy(s,
+                                           g->spec_index_state_score[il],
+                                           g->layer_index_state_score[il], ib);
         }
     }
     if (ok) ok = ds4_gpu_end_commands() != 0;
@@ -56112,16 +58903,20 @@ static bool spec_frontier_restore(ds4_spec_frontier *f, ds4_session *s) {
         const uint32_t ratio = ds4_layer_compress_ratio(il);
         if (ratio == 0) continue;
         const uint64_t ab = ds4_gpu_tensor_bytes(g->layer_attn_state_kv[il]);
-        ok = ds4_gpu_tensor_copy(g->layer_attn_state_kv[il], 0,
-                                   g->spec_attn_state_kv[il], 0, ab) != 0 &&
-             ds4_gpu_tensor_copy(g->layer_attn_state_score[il], 0,
-                                   g->spec_attn_state_score[il], 0, ab) != 0;
+        ok = spec_frontier_tensor_copy(s,
+                                       g->layer_attn_state_kv[il],
+                                       g->spec_attn_state_kv[il], ab) &&
+             spec_frontier_tensor_copy(s,
+                                       g->layer_attn_state_score[il],
+                                       g->spec_attn_state_score[il], ab);
         if (ok && ratio == 4) {
             const uint64_t ib = ds4_gpu_tensor_bytes(g->layer_index_state_kv[il]);
-            ok = ds4_gpu_tensor_copy(g->layer_index_state_kv[il], 0,
-                                       g->spec_index_state_kv[il], 0, ib) != 0 &&
-                 ds4_gpu_tensor_copy(g->layer_index_state_score[il], 0,
-                                       g->spec_index_state_score[il], 0, ib) != 0;
+            ok = spec_frontier_tensor_copy(s,
+                                           g->layer_index_state_kv[il],
+                                           g->spec_index_state_kv[il], ib) &&
+                 spec_frontier_tensor_copy(s,
+                                           g->layer_index_state_score[il],
+                                           g->spec_index_state_score[il], ib);
         }
     }
     if (ok) ok = ds4_gpu_end_commands() != 0;
@@ -58463,20 +61258,31 @@ int ds4_engine_generate_argmax(
             ds4_session_free(s);
             return rc;
         }
-        return generate_metal_graph_raw_swa(model, vocab, weights, prompt,
-                                            n_predict, ctx_size, e->quality,
-                                            e->ssd_streaming,
-                                            e->ssd_streaming_cold,
-                                            e->ssd_streaming_preload_experts,
-                                            e->ssd_streaming_cache_bytes,
-                                            e->ssd_streaming_prefill_headroom_bytes,
-                                            e->power_percent,
-                                            e->prefill_chunk,
-                                            e->directional_steering_file,
-                                            e->directional_steering_attn_scale,
-                                            e->directional_steering_ffn_scale,
-                                            emit, done, emit_ud,
-                                            progress, progress_ud);
+        const int rc = generate_metal_graph_raw_swa(
+            model, vocab, weights, prompt, n_predict, ctx_size, e->quality,
+            e->ssd_streaming, e->ssd_streaming_cold,
+            e->ssd_streaming_preload_experts,
+            e->ssd_streaming_cache_bytes,
+            e->ssd_streaming_prefill_headroom_bytes,
+            ds4_engine_streaming_transient_guard_bytes(e),
+            e->power_percent,
+            e->prefill_chunk, e->directional_steering_file,
+            e->directional_steering_attn_scale,
+            e->directional_steering_ffn_scale, emit, done, emit_ud,
+            progress, progress_ud);
+        /* The legacy frontend owns a temporary graph rather than a session.
+         * Do not leave explicitly requested resident sidecars pinned after
+         * return.  Each backend may either retain reusable scratch for its
+         * lifecycle or release it at this quiescent boundary. */
+        if (__atomic_load_n(
+                &e->live_session_count, __ATOMIC_RELAXED) == 0u &&
+            !ds4_gpu_release_q4_attn_q_b_f16_sidecars()) {
+            fprintf(stderr,
+                    "ds4: WARNING: could not release resident %s Q4 "
+                    "attn_q_b F16 sidecars after legacy generation\n",
+                    ds4_backend_name(e->backend));
+        }
+        return rc;
 #else
         fprintf(stderr, "ds4: %s generation requested but this build has no graph backend support\n",
                 ds4_backend_name(e->backend));
@@ -58544,187 +61350,6 @@ static int glm_metal_compare_i32_list(
 }
 
 #ifndef DS4_NO_GPU
-static void glm_metal_q8_diag_fill_input(float *x, uint64_t n_tok, uint64_t in_dim) {
-    for (uint64_t t = 0; t < n_tok; t++) {
-        for (uint64_t i = 0; i < in_dim; i++) {
-            uint32_t s = (uint32_t)(0x9e3779b9u ^ (uint32_t)(t * 0x85ebca6bu) ^
-                                    (uint32_t)(i * 0xc2b2ae35u));
-            s ^= s >> 16;
-            s *= 0x7feb352du;
-            s ^= s >> 15;
-            s *= 0x846ca68bu;
-            s ^= s >> 16;
-            const float centered = ((float)(int32_t)(s & 0xffffu) - 32768.0f) / 32768.0f;
-            const float scale = 0.25f + 0.015625f * (float)((i + 3u * t) & 31u);
-            x[t * in_dim + i] = centered * scale;
-        }
-    }
-}
-
-static void glm_metal_q8_diag_reference(
-        float            *out,
-        const ds4_model  *model,
-        const ds4_tensor *w,
-        const float      *x,
-        uint64_t          n_tok) {
-    const uint64_t out_dim = w->elements / w->dim[0];
-    for (uint64_t t = 0; t < n_tok; t++) {
-        matvec_q8_0_f32_ref(out + t * out_dim,
-                            model,
-                            w,
-                            x + t * w->dim[0]);
-    }
-}
-
-static int glm_metal_graph_test_q8_prefill_one(
-        ds4_engine       *e,
-        const char       *name,
-        const ds4_tensor *w,
-        int               strict) {
-    if (!w) return 1;
-    if (w->type != DS4_TENSOR_Q8_0 || w->ndim < 2 || w->dim[0] == 0) {
-        fprintf(stderr, "ds4: GLM Q8 prefill diagnostic found unexpected %s layout\n",
-                name);
-        return 0;
-    }
-
-    const ds4_model *model = &e->model;
-    const uint64_t in_dim = w->dim[0];
-    const uint64_t out_dim = w->elements / in_dim;
-    const uint32_t cases[] = { 16u, 17u, 31u, 32u };
-    int ok = 1;
-
-    printf("  q8_prefill_diag: %s ndim=%u in=%llu out=%llu strict=%d\n",
-           name,
-           w->ndim,
-           (unsigned long long)in_dim,
-           (unsigned long long)out_dim,
-           strict);
-
-    for (uint32_t ci = 0; ci < sizeof(cases) / sizeof(cases[0]); ci++) {
-        const uint32_t n_tok = cases[ci];
-        if (in_dim > UINT64_MAX / n_tok / sizeof(float) ||
-            out_dim > UINT64_MAX / n_tok / sizeof(float) ||
-            out_dim > UINT32_MAX / n_tok) {
-            fprintf(stderr, "ds4: GLM Q8 prefill diagnostic size overflow in %s token case %u\n",
-                    name,
-                    n_tok);
-            ok = 0;
-            if (strict) break;
-            continue;
-        }
-
-        const uint64_t x_bytes = (uint64_t)n_tok * in_dim * sizeof(float);
-        const uint64_t out_bytes = (uint64_t)n_tok * out_dim * sizeof(float);
-        if (x_bytes > SIZE_MAX || out_bytes > SIZE_MAX) {
-            fprintf(stderr, "ds4: GLM Q8 prefill diagnostic host allocation is too large in %s token case %u\n",
-                    name,
-                    n_tok);
-            ok = 0;
-            if (strict) break;
-            continue;
-        }
-
-        char label[128];
-        snprintf(label, sizeof(label), "q8_prefill_diag_%s_tok%u", name, n_tok);
-
-        float *x_host = xmalloc((size_t)x_bytes);
-        float *cpu_out = xmalloc((size_t)out_bytes);
-        float *gpu_out = xmalloc((size_t)out_bytes);
-        ds4_gpu_tensor *x_gpu = ds4_gpu_tensor_alloc(x_bytes);
-        ds4_gpu_tensor *out_gpu = ds4_gpu_tensor_alloc(out_bytes);
-        int case_ok = x_gpu && out_gpu;
-
-        if (!case_ok) {
-            fprintf(stderr, "ds4: GLM Q8 prefill diagnostic could not allocate %s token case %u\n",
-                    name,
-                    n_tok);
-        }
-        if (case_ok) {
-            glm_metal_q8_diag_fill_input(x_host, n_tok, in_dim);
-            glm_metal_q8_diag_reference(cpu_out, model, w, x_host, n_tok);
-            case_ok = ds4_gpu_tensor_write(x_gpu, 0, x_host, x_bytes) != 0;
-        }
-        if (case_ok) {
-            case_ok = ds4_gpu_matmul_q8_0_tensor(out_gpu,
-                                                 model->map,
-                                                 model->size,
-                                                 w->abs_offset,
-                                                 in_dim,
-                                                 out_dim,
-                                                 x_gpu,
-                                                 n_tok);
-        }
-        if (case_ok) {
-            case_ok = ds4_gpu_tensor_read(out_gpu, 0, gpu_out, out_bytes) != 0;
-        }
-        if (case_ok) {
-            case_ok = glm_metal_compare_f32(label,
-                                            cpu_out,
-                                            gpu_out,
-                                            (uint32_t)((uint64_t)n_tok * out_dim),
-                                            strict ? 5.0e-2f : 3.0e38f);
-        }
-
-        ds4_gpu_tensor_free(out_gpu);
-        ds4_gpu_tensor_free(x_gpu);
-        free(gpu_out);
-        free(cpu_out);
-        free(x_host);
-
-        if (!case_ok) {
-            ok = 0;
-            if (strict) break;
-        }
-    }
-
-    return ok || !strict;
-}
-
-static int glm_metal_graph_test_q8_prefill(
-        ds4_engine              *e,
-        const ds4_layer_weights *layer) {
-    return 1;
-
-    if (!layer) {
-        fprintf(stderr, "ds4: GLM Q8 prefill diagnostic requires layer weights\n");
-        return 0;
-    }
-
-    const int strict = 0;
-    int ok = 1;
-    const ds4_layer_weights *sparse =
-        DS4_N_LEADING_DENSE < DS4_N_LAYER ? &e->weights.layer[DS4_N_LEADING_DENSE] : NULL;
-
-#define DS4_GLM_Q8_DIAG_ONE(label, tensor)                                      \
-    do {                                                                        \
-        if (!glm_metal_graph_test_q8_prefill_one(e, label, tensor, strict)) {   \
-            ok = 0;                                                             \
-            if (strict) goto done;                                              \
-        }                                                                       \
-    } while (0)
-
-    DS4_GLM_Q8_DIAG_ONE("layer0.attn_q_a", layer->attn_q_a);
-    DS4_GLM_Q8_DIAG_ONE("layer0.attn_q_b", layer->attn_q_b);
-    DS4_GLM_Q8_DIAG_ONE("layer0.attn_kv_a_mqa", layer->attn_kv_a_mqa);
-    DS4_GLM_Q8_DIAG_ONE("layer0.attn_k_b", layer->attn_k_b);
-    DS4_GLM_Q8_DIAG_ONE("layer0.attn_v_b", layer->attn_v_b);
-    DS4_GLM_Q8_DIAG_ONE("layer0.attn_output", layer->attn_output);
-    DS4_GLM_Q8_DIAG_ONE("layer0.ffn_gate", layer->ffn_gate);
-    DS4_GLM_Q8_DIAG_ONE("layer0.ffn_up", layer->ffn_up);
-    DS4_GLM_Q8_DIAG_ONE("layer0.ffn_down", layer->ffn_down);
-
-    if (sparse) {
-        DS4_GLM_Q8_DIAG_ONE("layer3.ffn_gate_shexp", sparse->ffn_gate_shexp);
-        DS4_GLM_Q8_DIAG_ONE("layer3.ffn_up_shexp", sparse->ffn_up_shexp);
-        DS4_GLM_Q8_DIAG_ONE("layer3.ffn_down_shexp", sparse->ffn_down_shexp);
-    }
-
-done:
-#undef DS4_GLM_Q8_DIAG_ONE
-    return strict ? ok : 1;
-}
-
 static int glm_metal_graph_test_multitok_attention(
         ds4_engine              *e,
         const ds4_tokens        *prompt,
@@ -59632,8 +62257,6 @@ static int glm_metal_graph_test(ds4_engine *e, const ds4_tokens *prompt) {
     }
     if (ok) ok = glm_metal_graph_test_multitok_attention(e, prompt, layer);
     if (ok) ok = glm_metal_graph_test_decode_attention(e, prompt, layer);
-    if (ok) ok = glm_metal_graph_test_q8_prefill(e, layer);
-
     if (ok) {
         memcpy(cpu_cur, cpu_emb, emb_bytes);
         for (uint32_t il = 0; il < sparse_il; il++) {
@@ -60436,6 +63059,10 @@ static bool ds4_engine_configure_streaming_auto_cache(ds4_engine *e) {
                 "ds4: SSD streaming auto cache could not measure non-routed model weights\n");
         return false;
     }
+    const uint64_t support_model_bytes =
+        ds4_engine_support_model_bytes(e);
+    const uint64_t fixed_model_bytes =
+        ds4_add_sat_u64(non_routed_bytes, support_model_bytes);
 
     uint64_t per_expert_bytes = 0;
     if (!ds4_streaming_routed_expert_bytes(&e->weights, &per_expert_bytes)) {
@@ -60466,7 +63093,7 @@ static bool ds4_engine_configure_streaming_auto_cache(ds4_engine *e) {
     }
     ds4_ssd_cache_plan plan;
     if (!ds4_ssd_auto_cache_plan(recommended,
-                                 non_routed_bytes,
+                                 fixed_model_bytes,
                                  per_expert_bytes,
                                  max_model_experts,
                                  &plan)) {
@@ -60529,8 +63156,8 @@ static bool ds4_engine_configure_streaming_auto_cache(ds4_engine *e) {
         }
         uint64_t active_model_bytes =
             glm_graph_streaming_active_model_bytes(&e->weights);
-        if (non_routed_bytes > active_model_bytes) {
-            active_model_bytes = non_routed_bytes;
+        if (fixed_model_bytes > active_model_bytes) {
+            active_model_bytes = fixed_model_bytes;
         }
         const double fraction = glm_graph_env_double(
                 "DS4_GLM_MEMORY_GUARD_FRACTION", 0.99, 0.50, 1.00);
@@ -60594,6 +63221,11 @@ static bool ds4_engine_configure_streaming_auto_cache(ds4_engine *e) {
     fprintf(stderr,
             "ds4:   non-routed weights: %.2f GiB\n",
             (double)non_routed_bytes / 1073741824.0);
+    if (support_model_bytes != 0) {
+        fprintf(stderr,
+                "ds4:   support model weights: %.2f GiB\n",
+                (double)support_model_bytes / 1073741824.0);
+    }
     fprintf(stderr,
             "ds4:   routed expert size: %.2f MiB\n",
             (double)per_expert_bytes / 1048576.0);
@@ -60625,9 +63257,9 @@ static bool ds4_engine_configure_streaming_auto_cache(ds4_engine *e) {
                 e->placement_ctx_hint > 0 ? e->placement_ctx_hint : 4096);
     }
 #endif
-    if (plan.model_target_bytes <= non_routed_bytes) {
+    if (plan.model_target_bytes <= fixed_model_bytes) {
         fprintf(stderr,
-                "ds4:   note: non-routed weights already fill the 80%% target; keeping a one-expert cache\n");
+                "ds4:   note: fixed model weights already fill the 80%% target; keeping a one-expert cache\n");
     }
     return true;
 #endif
@@ -62462,6 +65094,62 @@ int ds4_test_glm_memory_guard_disabled(void) {
     return glm_graph_memory_guard_disabled() ? 1 : 0;
 }
 
+int ds4_test_dspark_runtime_policy(ds4_backend backend,
+                                   ds4_distributed_role distributed_role) {
+    return (int)ds4_dspark_runtime_policy_for(backend, distributed_role);
+}
+
+bool ds4_test_streaming_manual_cache_cap_count(
+        uint32_t  requested_count,
+        uint64_t  per_expert_bytes,
+        uint64_t  safe_cache_bytes,
+        uint32_t *effective_count_out,
+        uint64_t *requested_bytes_out,
+        uint64_t *effective_bytes_out) {
+    return ds4_streaming_manual_cache_cap_count(requested_count,
+                                                per_expert_bytes,
+                                                safe_cache_bytes,
+                                                effective_count_out,
+                                                requested_bytes_out,
+                                                effective_bytes_out);
+}
+
+bool ds4_test_streaming_manual_cache_count_cap_enabled(void) {
+    return ds4_streaming_manual_cache_count_cap_enabled();
+}
+
+bool ds4_test_streaming_manual_cache_count_cap_eligible(
+        ds4_backend backend,
+        bool        ssd_streaming,
+        bool        dspark_enabled,
+        bool        support_is_dspark,
+        uint32_t    cache_experts,
+        uint64_t    cache_bytes) {
+    return ds4_streaming_manual_cache_count_cap_eligible(
+            backend,
+            ssd_streaming,
+            dspark_enabled,
+            support_is_dspark ? DS4_SUPPORT_DSPARK : DS4_SUPPORT_NONE,
+            cache_experts,
+            cache_bytes);
+}
+
+uint64_t ds4_test_streaming_dspark_active_support_reserve_bytes(
+        uint64_t support_model_bytes) {
+    return ds4_streaming_dspark_active_support_reserve_bytes(
+            support_model_bytes);
+}
+
+uint32_t ds4_test_streaming_manual_cache_nonfatal_effective_count(
+        uint32_t requested_count,
+        bool     safe_cache_known,
+        uint32_t candidate_count) {
+    return ds4_streaming_manual_cache_nonfatal_effective_count(
+            requested_count,
+            safe_cache_known,
+            candidate_count);
+}
+
 static int ds4_test_make_engine(
         ds4_engine *eng,
         const ds4_test_fake_tensor *tensors,
@@ -62788,6 +65476,27 @@ static int ds4_engine_open_internal(ds4_engine **out,
         *out = NULL;
         return 1;
     }
+    if (opt->dspark && !opt->inspect_only) {
+        const ds4_dspark_runtime_policy policy =
+            ds4_dspark_runtime_policy_for(opt->backend,
+                                          opt->distributed.role);
+        if (policy == DS4_DSPARK_RUNTIME_UNSUPPORTED_CPU) {
+            fprintf(stderr,
+                    "ds4: --dspark requires a Metal, CUDA, or ROCm graph "
+                    "backend; CPU speculative decode is not implemented\n");
+            free(e);
+            *out = NULL;
+            return 1;
+        }
+        if (policy == DS4_DSPARK_RUNTIME_UNSUPPORTED_DISTRIBUTED) {
+            fprintf(stderr,
+                    "ds4: --dspark is not yet compatible with distributed "
+                    "inference\n");
+            free(e);
+            *out = NULL;
+            return 1;
+        }
+    }
     if ((opt->directional_steering_attn != 0.0f || opt->directional_steering_ffn != 0.0f) &&
         (!opt->directional_steering_file || !opt->directional_steering_file[0]))
     {
@@ -63087,24 +65796,6 @@ static int ds4_engine_open_internal(ds4_engine **out,
         *out = NULL;
         return 1;
     }
-    if (e->ssd_streaming && e->ssd_streaming_cache_bytes != 0) {
-        const uint64_t requested_cache_bytes = e->ssd_streaming_cache_bytes;
-        const uint64_t safe_cache_bytes =
-            ds4_streaming_manual_cache_safe_bytes(e->backend,
-                                                  opt->context_size,
-                                                  e->prefill_chunk,
-                                                  e->ssd_streaming);
-        if (safe_cache_bytes != 0 &&
-            e->ssd_streaming_cache_bytes > safe_cache_bytes) {
-            e->ssd_streaming_cache_bytes = safe_cache_bytes;
-            fprintf(stderr,
-                    "ds4: %s SSD streaming cache budget %.2f GiB capped to %.2f GiB "
-                    "to stay below the graph working-set pressure budget\n",
-                    ds4_backend_name(e->backend),
-                    (double)requested_cache_bytes / 1073741824.0,
-                    (double)e->ssd_streaming_cache_bytes / 1073741824.0);
-        }
-    }
     if (opt->inspect_only) {
         if (opt->mtp_path && opt->mtp_path[0] &&
             opt->distributed.role == DS4_DISTRIBUTED_NONE) {
@@ -63209,12 +65900,6 @@ static int ds4_engine_open_internal(ds4_engine **out,
     }
     if (opt->mtp_path && opt->mtp_path[0] &&
         opt->distributed.role == DS4_DISTRIBUTED_NONE) {
-        if (e->ssd_streaming) {
-            fprintf(stderr, "ds4: --ssd-streaming is not compatible with --mtp-model yet\n");
-            ds4_engine_close(e);
-            *out = NULL;
-            return 1;
-        }
         model_open(&e->mtp_model, opt->mtp_path, graph_backend, true);
         ds4_dspark_summary dspark = {0};
         e->support_kind =
@@ -63224,7 +65909,26 @@ static int ds4_engine_open_internal(ds4_engine **out,
             *out = NULL;
             return 1;
         }
+        if (e->dspark && e->support_kind != DS4_SUPPORT_DSPARK) {
+            fprintf(stderr,
+                    "ds4: --dspark requires a DSpark support GGUF; %s was "
+                    "detected as %s\n",
+                    opt->mtp_path,
+                    support_kind_name(e->support_kind));
+            ds4_engine_close(e);
+            *out = NULL;
+            return 1;
+        }
         if (e->support_kind == DS4_SUPPORT_MTP_LEGACY) {
+            if (e->ssd_streaming) {
+                fprintf(stderr,
+                        "ds4: --ssd-streaming is not compatible with the "
+                        "legacy MTP support model; use DSpark or disable "
+                        "streaming\n");
+                ds4_engine_close(e);
+                *out = NULL;
+                return 1;
+            }
             if (opt->tp.role != DS4_TP_NONE) {
                 fprintf(stderr,
                         "ds4: legacy MTP support is ignored under tensor parallelism; "
@@ -63240,6 +65944,15 @@ static int ds4_engine_open_internal(ds4_engine **out,
                         e->mtp_draft_tokens);
             }
         } else if (e->support_kind == DS4_SUPPORT_DSPARK) {
+            if (e->ssd_streaming && e->dspark &&
+                opt->tp.role != DS4_TP_NONE) {
+                fprintf(stderr,
+                        "ds4: DSpark with --ssd-streaming is not yet "
+                        "supported with tensor parallelism\n");
+                ds4_engine_close(e);
+                *out = NULL;
+                return 1;
+            }
             dspark_weights_bind_optional(&e->dspark_weights,
                                          &e->mtp_model,
                                          &dspark);
@@ -63273,6 +65986,87 @@ static int ds4_engine_open_internal(ds4_engine **out,
             return 1;
         }
     }
+#if defined(__APPLE__) && !defined(DS4_NO_GPU)
+    /* A 4096-row graph workspace consumes several GiB on Flash and competes
+     * directly with the file-backed target/support working sets.  When the
+     * user did not request a chunk size, keep DSpark+SSD viable on small Macs
+     * by choosing a decode-oriented workspace.  Explicit CLI/env choices win;
+     * set DS4_DSPARK_LOW_MEMORY_PREFILL_CHUNK=0 to disable this policy. */
+    if (e->backend == DS4_BACKEND_METAL &&
+        e->ssd_streaming && e->dspark &&
+        e->support_kind == DS4_SUPPORT_DSPARK &&
+        opt->prefill_chunk == 0 &&
+        getenv("DS4_METAL_PREFILL_CHUNK") == NULL) {
+        const uint64_t host_bytes = ds4_graph_host_memory_bytes();
+        if (host_bytes != 0 && host_bytes <= 24ull * 1024ull * 1024ull * 1024ull) {
+            const uint32_t low_memory_chunk = ds4_dspark_env_u32(
+                    "DS4_DSPARK_LOW_MEMORY_PREFILL_CHUNK", 128u);
+            if (low_memory_chunk != 0) {
+                e->prefill_chunk = low_memory_chunk;
+                fprintf(stderr,
+                        "ds4: Metal SSD+DSpark low-memory prefill chunk set "
+                        "to %u (override with --prefill-chunk or "
+                        "DS4_DSPARK_LOW_MEMORY_PREFILL_CHUNK=0)\n",
+                        e->prefill_chunk);
+            }
+        }
+    }
+#endif
+    if (!ds4_engine_cap_metal_dspark_manual_cache_count(e,
+                                                        opt->context_size)) {
+        ds4_engine_close(e);
+        *out = NULL;
+        return 1;
+    }
+    if (e->ssd_streaming && e->ssd_streaming_cache_bytes != 0) {
+        const uint64_t requested_cache_bytes = e->ssd_streaming_cache_bytes;
+        uint64_t non_routed_bytes = 0;
+        const bool non_routed_known =
+            weights_streaming_non_routed_bytes(&e->weights,
+                                               &non_routed_bytes);
+        if (ds4_backend_uses_graph(e->backend) && !non_routed_known) {
+            fprintf(stderr,
+                    "ds4: %s SSD streaming could not measure non-routed "
+                    "weights for the manual cache safety cap\n",
+                    ds4_backend_name(e->backend));
+            ds4_engine_close(e);
+            *out = NULL;
+            return 1;
+        }
+        const uint64_t support_model_bytes =
+            ds4_engine_support_model_bytes(e);
+        const uint64_t fixed_model_bytes =
+            ds4_add_sat_u64(non_routed_bytes, support_model_bytes);
+        uint64_t safe_cache_bytes = 0;
+        const bool safe_cache_known =
+            ds4_streaming_manual_cache_safe_bytes(e->backend,
+                                                  opt->context_size,
+                                                  e->prefill_chunk,
+                                                  e->ssd_streaming,
+                                                  fixed_model_bytes,
+                                                  &safe_cache_bytes);
+        if (safe_cache_known && safe_cache_bytes == 0) {
+            fprintf(stderr,
+                    "ds4: %s SSD streaming has no safe room for the requested "
+                    "expert cache after the context/KV estimate and fixed "
+                    "model weights\n",
+                    ds4_backend_name(e->backend));
+            ds4_engine_close(e);
+            *out = NULL;
+            return 1;
+        }
+        if (safe_cache_known &&
+            e->ssd_streaming_cache_bytes > safe_cache_bytes) {
+            e->ssd_streaming_cache_bytes = safe_cache_bytes;
+            fprintf(stderr,
+                    "ds4: %s SSD streaming cache budget %.2f GiB capped to %.2f GiB "
+                    "to stay below the context-estimated working-set pressure "
+                    "budget\n",
+                    ds4_backend_name(e->backend),
+                    (double)requested_cache_bytes / 1073741824.0,
+                    (double)e->ssd_streaming_cache_bytes / 1073741824.0);
+        }
+    }
 
 #ifndef DS4_NO_GPU
     if (e->backend == DS4_BACKEND_CUDA) {
@@ -63295,13 +66089,17 @@ static int ds4_engine_open_internal(ds4_engine **out,
      * GPU budget, so let the residency set pin them — that is what makes
      * the shard actually resident.  Without the sysctl, fall back to lazy
      * faulting (slow but functional). */
-    if (graph_backend && tp_shard && glm_graph_wired_limit_bytes() == 0) {
+    const bool skip_model_residency =
+        graph_backend && tp_shard && glm_graph_wired_limit_bytes() == 0;
+    if (skip_model_residency) {
         fprintf(stderr,
                 "ds4: iogpu.wired_limit_mb is 0 -- TP expert shard will page "
                 "lazily; raise it (e.g. sudo sysctl iogpu.wired_limit_mb=120000) "
                 "for full residency\n");
-        ds4_gpu_model_residency_skip(1);
     }
+    /* This is process-global backend state, so every engine open must reset it
+     * after a previous TP/no-wired-limit engine (including failed startup). */
+    ds4_gpu_model_residency_skip(skip_model_residency ? 1 : 0);
     if (graph_backend) {
         if (e->multi_tier) {
             /* Wave-2 multi-tier branch.
@@ -63387,6 +66185,50 @@ static int ds4_engine_open_internal(ds4_engine **out,
                 load_output ||
                     (load_output_optional && weights_have_output_head(&e->weights)),
                 opt->context_size);
+#ifdef DS4_ROCM_BUILD
+        if (e->backend == DS4_BACKEND_CUDA &&
+            e->ssd_streaming && e->dspark &&
+            e->support_kind == DS4_SUPPORT_DSPARK) {
+            uint32_t bad_layer = UINT32_MAX;
+            uint32_t bad_gate_type = 0;
+            uint32_t bad_down_type = 0;
+            if (!ds4_rocm_dspark_ssd_layout_supported(
+                        &e->weights,
+                        &bad_layer,
+                        &bad_gate_type,
+                        &bad_down_type)) {
+                fprintf(stderr,
+                        "ds4: ROCm DSpark with SSD streaming currently "
+                        "requires IQ2_XXS/Q2_K or Q2_K routed experts; "
+                        "layer %u uses gate=%s down=%s\n",
+                        bad_layer,
+                        tensor_type_name(bad_gate_type),
+                        tensor_type_name(bad_down_type));
+                ds4_engine_close(e);
+                *out = NULL;
+                return 1;
+            }
+            uint64_t min_verify_experts =
+                (uint64_t)e->dspark_weights.block_size *
+                (uint64_t)DS4_N_EXPERT_USED;
+            if (min_verify_experts > DS4_N_EXPERT) {
+                min_verify_experts = DS4_N_EXPERT;
+            }
+            if (e->ssd_streaming_cache_experts < min_verify_experts) {
+                fprintf(stderr,
+                        "ds4: ROCm DSpark verification with SSD streaming "
+                        "needs at least %llu cached experts (%u draft rows x "
+                        "%u experts); configured %u\n",
+                        (unsigned long long)min_verify_experts,
+                        e->dspark_weights.block_size,
+                        DS4_N_EXPERT_USED,
+                        e->ssd_streaming_cache_experts);
+                ds4_engine_close(e);
+                *out = NULL;
+                return 1;
+            }
+        }
+#endif
         if (!ds4_engine_glm_streaming_memory_guard(
                     e,
                     load_slice,
@@ -63666,32 +66508,6 @@ static int ds4_engine_open_internal(ds4_engine **out,
         const bool support_model_runtime_ready =
             e->mtp_ready ||
             (e->support_kind == DS4_SUPPORT_DSPARK && e->dspark);
-        bool support_uses_secondary_rocm_cache = false;
-#ifdef DS4_ROCM_BUILD
-        /* The generic range cache is already keyed by model map. Keep the
-         * resident target ranges when adding DSpark support on gfx1151. */
-        support_uses_secondary_rocm_cache =
-            e->support_kind == DS4_SUPPORT_DSPARK &&
-            ds4_gpu_dspark_gfx1151_fast_path() != 0;
-#endif
-        if (support_model_runtime_ready &&
-            !support_uses_secondary_rocm_cache &&
-            !ds4_gpu_set_model_map_range(e->mtp_model.map,
-                                           e->mtp_model.size,
-                                           e->mtp_model.tensor_data_pos,
-                                           e->mtp_model.size - e->mtp_model.tensor_data_pos,
-                                           e->mtp_model.max_tensor_bytes))
-        {
-            fprintf(stderr,
-                    "ds4: %s failed to map support model views; aborting startup. "
-                    "This is commonly caused by insufficient memory or accelerator VM budget.\n",
-                    ds4_backend_name(e->backend));
-            free(load_offsets);
-            free(load_sizes);
-            ds4_engine_close(e);
-            *out = NULL;
-            return 1;
-        }
         if (!ds4_engine_preload_pro_q4_expert_tables(e,
                                                      load_slice,
                                                      load_layer_start,
@@ -63716,19 +66532,31 @@ static int ds4_engine_open_internal(ds4_engine **out,
         }
         free(load_offsets);
         free(load_sizes);
-        /* Also apply explicit optional Q8 preload settings to the runtime
-         * support model when loaded. */
+        /* The support GGUF is deliberately fully resident. Backends prepare
+         * it without replacing the active target mapping: Metal installs a
+         * second view set, CUDA a host-base-keyed range, and ROCm a persistent
+         * device image that survives target layer remaps. */
         if (support_model_runtime_ready) {
             (void)ds4_gpu_set_model_fd_for_map(e->mtp_model.fd, e->mtp_model.map);
-            if (!accelerator_cache_model_tensors(e->backend, &e->mtp_model,
-                                                 NULL, NULL, 0)) {
-                fprintf(stderr, "ds4: %s failed to prepare optional support model cache\n",
+            const int support_ok = ds4_gpu_prepare_support_model(
+                    e->mtp_model.map,
+                    e->mtp_model.size,
+                    e->mtp_model.tensor_data_pos,
+                    e->mtp_model.size - e->mtp_model.tensor_data_pos,
+                    e->mtp_model.max_tensor_bytes);
+            (void)ds4_gpu_set_model_fd_for_map(e->model.fd, e->model.map);
+            if (!support_ok) {
+                fprintf(stderr,
+                        "ds4: %s failed to prepare the resident support model; "
+                        "check accelerator memory and model-cache limits\n",
                         ds4_backend_name(e->backend));
                 ds4_engine_close(e);
                 *out = NULL;
                 return 1;
             }
-            (void)ds4_gpu_set_model_fd_for_map(e->model.fd, e->model.map);
+#if defined(__APPLE__) && !defined(DS4_NO_GPU)
+            ds4_engine_metal_dspark_lock_stage0_hotset(e);
+#endif
         }
         if (e->vision_ready) {
 #if defined(__APPLE__)
@@ -64378,6 +67206,12 @@ int ds4_engine_tp_bind(ds4_engine *e, struct ds4_tp *tp, char *err, size_t errle
         snprintf(err, errlen, "tensor parallelism requires the Metal backend");
         return 0;
     }
+    if (e->ssd_streaming && e->dspark &&
+        e->support_kind == DS4_SUPPORT_DSPARK) {
+        snprintf(err, errlen,
+                 "DSpark with SSD streaming does not yet support Metal tensor parallelism");
+        return 0;
+    }
     if (e->tp.active) {
         snprintf(err, errlen, "tensor parallelism already bound");
         return 0;
@@ -64483,16 +67317,30 @@ void ds4_engine_close(ds4_engine *e) {
     weights_free(&e->weights);
     vocab_free(&e->vocab);
     ds4_threads_shutdown();
-    if (e->mtp_model.map) model_close(&e->mtp_model);
-    if (e->vision_model.map) model_close(&e->vision_model);
-    model_close(&e->model);
 #ifndef DS4_NO_GPU
     if (e->shared_prefill_workspace_ready) {
         metal_graph_free_prefill_workspace(&e->shared_prefill_workspace);
         e->shared_prefill_workspace_ready = false;
     }
+    /* Model views are no-copy aliases of the mmap regions.  Release and wait
+     * for the backend before munmap, especially when main and DSpark support
+     * views coexist in the Metal registry. */
     ds4_gpu_cleanup();
 #endif
+#if defined(__APPLE__) && !defined(DS4_NO_GPU)
+    if (e->dspark_hot_lock_addr && e->dspark_hot_lock_len != 0) {
+        if (munlock(e->dspark_hot_lock_addr, e->dspark_hot_lock_len) != 0) {
+            fprintf(stderr,
+                    "ds4: WARNING: Metal DSpark stage-0 hot-set unlock failed: %s\n",
+                    strerror(errno));
+        }
+        e->dspark_hot_lock_addr = NULL;
+        e->dspark_hot_lock_len = 0;
+    }
+#endif
+    if (e->mtp_model.map) model_close(&e->mtp_model);
+    if (e->vision_model.map) model_close(&e->vision_model);
+    model_close(&e->model);
     ds4_ssd_memory_lock_release(&e->simulated_memory);
     ds4_release_instance_lock();
     free(e->directional_steering_dirs);
@@ -64554,15 +67402,63 @@ static void ds4_session_print_dspark_stats(const ds4_session *s) {
             "ds4: DSpark stats cycles=%llu first_tokens=%llu proposed=%llu "
             "accepted_draft=%llu accept_rate=%.2f%% avg_accept=%.3f "
             "full=%llu partial=%llu direct_full=%llu direct_partial=%llu "
-            "replay_fallbacks=%llu miss_first=%llu no_draft=%llu "
-            "no_room=%llu invalid=%llu scheduler_skips=%llu "
-            "tail_skips=%llu verifier_unavailable=%llu errors=%llu time_ms propose=%.3f "
+            "replay_fallbacks=%llu prop_capped=%llu "
+            "prop_scheduled_rows=%llu "
+            "exact2_attempt=%llu exact2_full=%llu exact2_partial=%llu "
+            "exact2_fallback=%llu cuda_exactn_attempt=%llu "
+            "cuda_exactn_full=%llu cuda_exactn_fallback=%llu "
+            "cuda_exactn_partial_fallback=%llu "
+            "cuda_exactn_error_fallback=%llu cuda_exactn_rows=%llu "
+            "cuda_exactn_partial_replay=%llu "
+            "cuda_exactn_verify_skip=%llu "
+            "cuda_exactn_batch_head_attempt=%llu "
+            "cuda_exactn_batch_head_use=%llu "
+            "cuda_exactn_batch_head_fallback=%llu "
+            "cuda_exactn_graph_attempt=%llu "
+            "cuda_exactn_graph_use=%llu "
+            "cuda_exactn_graph_capture=%llu "
+            "cuda_exactn_graph_replay=%llu "
+            "cuda_exactn_graph_warm=%llu "
+            "cuda_exactn_graph_no_slot=%llu "
+            "cuda_exactn_graph_failure=%llu "
+            "cuda_device_proposer_attempt=%llu "
+            "cuda_device_proposer_use=%llu "
+            "cuda_device_proposer_fallback=%llu "
+            "cuda_device_proposer_policy_mismatch=%llu "
+            "metal_device_proposer_attempt=%llu "
+            "metal_device_proposer_use=%llu "
+            "metal_device_proposer_fallback=%llu "
+            "metal_device_proposer_policy_mismatch=%llu "
+            "exactn_union_attempt=%llu "
+            "exactn_union_full=%llu exactn_union_fallback=%llu "
+            "exactn_union_partial_fallback=%llu "
+            "exactn_union_error_fallback=%llu "
+            "exactn_union_partial_replay=%llu "
+            "exactn_union_verify_skip=%llu "
+            "metal_exactn_batch_head_attempt=%llu "
+            "metal_exactn_batch_head_use=%llu "
+            "metal_exactn_batch_head_fallback=%llu "
+            "exactn_attempt=%llu exactn_full=%llu "
+            "exactn_partial=%llu "
+            "exactn_fallback=%llu exactn_partial_fallback=%llu "
+            "exactn_error_fallback=%llu exactn_boundary_rows=%llu "
+            "miss_first=%llu no_draft=%llu "
+            "no_room=%llu invalid=%llu verifier_unavailable=%llu "
+            "errors=%llu time_ms propose=%.3f "
             "prop_stage0=%.3f prop_setup=%.3f prop_cache=%.3f "
             "prop_chain=%.3f prop_hidden=%.3f prop_conf0=%.3f "
             "prop_logits=%.3f prop_markov=%.3f prop_confidence=%.3f "
             "snapshot=%.3f verify=%.3f verify_upload=%.3f "
             "verify_layer=%.3f verify_head=%.3f verify_read=%.3f "
-            "verify_fused_head=%llu replay=%.3f spec_total=%.3f "
+            "cuda_exactn_ms=%.3f cuda_exactn_setup=%.3f "
+            "cuda_exactn_layer=%.3f cuda_exactn_head=%.3f "
+            "cuda_exactn_read=%.3f cuda_exactn_restore=%.3f "
+            "cuda_exactn_legacy_verify=%.3f "
+            "cuda_exactn_partial_replay_ms=%.3f "
+            "exactn_union_partial_replay_ms=%.3f "
+            "verify_fused_head=%llu metal_accept_only=%llu "
+            "metal_verify_rows_saved=%llu metal_replay_headless=%llu "
+            "replay=%.3f spec_total=%.3f "
             "target=%.3f saved=%.3f net_saved=%.3f "
             "draft_len_hist=%s accepted_len_hist=%s\n",
             (unsigned long long)st->cycles,
@@ -64576,12 +67472,59 @@ static void ds4_session_print_dspark_stats(const ds4_session *s) {
             (unsigned long long)st->direct_full_commits,
             (unsigned long long)st->direct_partial_commits,
             (unsigned long long)st->replay_fallbacks,
+            (unsigned long long)st->proposer_capped,
+            (unsigned long long)st->proposer_scheduled_rows,
+            (unsigned long long)st->exact2_attempts,
+            (unsigned long long)st->exact2_full_accepts,
+            (unsigned long long)st->exact2_partial_accepts,
+            (unsigned long long)st->exact2_fallbacks,
+            (unsigned long long)st->cuda_exactn_attempts,
+            (unsigned long long)st->cuda_exactn_full_accepts,
+            (unsigned long long)st->cuda_exactn_fallbacks,
+            (unsigned long long)st->cuda_exactn_partial_fallbacks,
+            (unsigned long long)st->cuda_exactn_error_fallbacks,
+            (unsigned long long)st->cuda_exactn_rows,
+            (unsigned long long)st->cuda_exactn_partial_replays,
+            (unsigned long long)st->cuda_exactn_legacy_verify_skips,
+            (unsigned long long)st->cuda_exactn_batch_head_attempts,
+            (unsigned long long)st->cuda_exactn_batch_head_uses,
+            (unsigned long long)st->cuda_exactn_batch_head_fallbacks,
+            (unsigned long long)st->cuda_exactn_graph_attempts,
+            (unsigned long long)st->cuda_exactn_graph_uses,
+            (unsigned long long)st->cuda_exactn_graph_captures,
+            (unsigned long long)st->cuda_exactn_graph_replays,
+            (unsigned long long)st->cuda_exactn_graph_warms,
+            (unsigned long long)st->cuda_exactn_graph_no_slots,
+            (unsigned long long)st->cuda_exactn_graph_failures,
+            (unsigned long long)st->cuda_device_proposer_attempts,
+            (unsigned long long)st->cuda_device_proposer_uses,
+            (unsigned long long)st->cuda_device_proposer_fallbacks,
+            (unsigned long long)st->cuda_device_proposer_policy_mismatches,
+            (unsigned long long)st->metal_device_proposer_attempts,
+            (unsigned long long)st->metal_device_proposer_uses,
+            (unsigned long long)st->metal_device_proposer_fallbacks,
+            (unsigned long long)st->metal_device_proposer_policy_mismatches,
+            (unsigned long long)st->exactn_union_attempts,
+            (unsigned long long)st->exactn_union_full_accepts,
+            (unsigned long long)st->exactn_union_fallbacks,
+            (unsigned long long)st->exactn_union_partial_fallbacks,
+            (unsigned long long)st->exactn_union_error_fallbacks,
+            (unsigned long long)st->exactn_union_partial_replays,
+            (unsigned long long)st->exactn_union_legacy_verify_skips,
+            (unsigned long long)st->metal_exactn_batch_head_attempts,
+            (unsigned long long)st->metal_exactn_batch_head_uses,
+            (unsigned long long)st->metal_exactn_batch_head_fallbacks,
+            (unsigned long long)st->exactn_attempts,
+            (unsigned long long)st->exactn_full_accepts,
+            (unsigned long long)st->exactn_partial_accepts,
+            (unsigned long long)st->exactn_fallbacks,
+            (unsigned long long)st->exactn_partial_fallbacks,
+            (unsigned long long)st->exactn_error_fallbacks,
+            (unsigned long long)st->exactn_boundary_rows,
             (unsigned long long)st->first_misses,
             (unsigned long long)st->no_draft,
             (unsigned long long)st->no_room,
             (unsigned long long)st->invalid_draft,
-            (unsigned long long)st->scheduler_skips,
-            (unsigned long long)st->tail_skips,
             (unsigned long long)st->verifier_unavailable,
             (unsigned long long)st->verifier_errors,
             st->propose_ms,
@@ -64600,7 +67543,19 @@ static void ds4_session_print_dspark_stats(const ds4_session *s) {
             st->verify_layer_ms,
             st->verify_head_ms,
             st->verify_read_ms,
+            st->cuda_exactn_ms,
+            st->cuda_exactn_setup_ms,
+            st->cuda_exactn_layer_ms,
+            st->cuda_exactn_head_ms,
+            st->cuda_exactn_read_ms,
+            st->cuda_exactn_restore_ms,
+            st->cuda_exactn_legacy_verify_ms,
+            st->cuda_exactn_partial_replay_ms,
+            st->exactn_union_partial_replay_ms,
             (unsigned long long)st->verifier_fused_head,
+            (unsigned long long)st->metal_acceptance_only_attempts,
+            (unsigned long long)st->metal_acceptance_only_rows_saved,
+            (unsigned long long)st->metal_replay_headless_tokens,
             st->replay_ms,
             st->total_ms,
             st->target_ms,
@@ -64632,6 +67587,113 @@ static int ds4_session_tp_register(ds4_session *s) {
     return 1;
 }
 
+#ifndef DS4_NO_GPU
+static int ds4_prepare_q4_attn_q_b_sidecars(
+        const ds4_model *model,
+        const ds4_weights *weights,
+        uint32_t max_batch_rows,
+        uint64_t working_set_reserve_bytes) {
+    if (!model || !weights) return 0;
+    ds4_gpu_q4_attn_q_b_f16_sidecar_desc descs[DS4_MAX_LAYER];
+    uint32_t count = 0;
+    const uint64_t q_dim = (uint64_t)DS4_N_HEAD * DS4_N_HEAD_DIM;
+    for (uint32_t il = 0; il < (uint32_t)DS4_N_LAYER; il++) {
+        const ds4_tensor *tensor = weights->layer[il].attn_q_b;
+        if (!tensor || tensor->type != DS4_TENSOR_Q4_K ||
+            tensor->ndim != 2u || tensor->dim[0] != DS4_N_LORA_Q ||
+            tensor->dim[1] != q_dim) {
+            continue;
+        }
+        descs[count++] = (ds4_gpu_q4_attn_q_b_f16_sidecar_desc) {
+            .weight_offset = tensor->abs_offset,
+            .weight_bytes = tensor->bytes,
+            .in_dim = tensor->dim[0],
+            .out_dim = tensor->dim[1],
+            .weight_type = tensor->type,
+            .layer = il,
+        };
+    }
+    if (count == 0u) return 1;
+
+    uint64_t prepared_bytes = 0;
+    return ds4_gpu_prepare_q4_attn_q_b_f16_sidecars(
+        model->map, model->size, descs, count, max_batch_rows,
+        working_set_reserve_bytes, &prepared_bytes);
+}
+
+static int ds4_session_prepare_q4_attn_q_b_sidecars(
+        ds4_session *s,
+        uint32_t max_batch_rows,
+        bool reserve_future_sessions) {
+    if (!s || !s->engine ||
+        !ds4_backend_uses_graph(s->engine->backend) ||
+        ds4_session_is_cpu(s) || ds4_session_is_glm(s)) {
+        return 1;
+    }
+    const uint64_t cache_generation =
+        ds4_gpu_q4_attn_q_b_f16_cache_generation();
+    if (s->q4_attn_q_b_f16_sidecars_generation ==
+            cache_generation &&
+        max_batch_rows <= s->q4_attn_q_b_f16_prepared_rows) {
+        return 1;
+    }
+    ds4_engine *e = s->engine;
+
+    uint64_t future_session_bytes = 0;
+    const uint32_t session_count = engine_placement_session_count(e);
+    const uint32_t live_sessions = __atomic_load_n(
+        &e->live_session_count, __ATOMIC_RELAXED);
+    uint32_t including_current = live_sessions;
+    /* Eager preparation runs before session registration, while the ordinary
+     * prompt-aware preflight runs after it.  Count the current session only
+     * in the former case instead of under-reserving one future graph. */
+    if (!s->engine_session_counted && including_current != UINT32_MAX) {
+        including_current++;
+    }
+    const uint32_t remaining =
+        session_count > including_current ?
+            session_count - including_current : 0u;
+    if (reserve_future_sessions && remaining != 0u) {
+        const ds4_context_memory memory =
+            ds4_context_memory_estimate_with_prefill_mode(
+                e->backend, s->ctx_size, e->prefill_chunk,
+                e->ssd_streaming);
+        future_session_bytes =
+            memory.total_bytes > UINT64_MAX / remaining
+                ? UINT64_MAX
+                : memory.total_bytes * remaining;
+    }
+
+    const uint64_t streaming_reserve_bytes =
+        ds4_engine_streaming_transient_guard_bytes(e);
+    const uint64_t working_set_reserve_bytes =
+        ds4_add_sat_u64(future_session_bytes, streaming_reserve_bytes);
+    const int rc = ds4_prepare_q4_attn_q_b_sidecars(
+        &e->model, &e->weights, max_batch_rows,
+        working_set_reserve_bytes);
+    if (rc < 0) {
+        fprintf(stderr,
+                "ds4: required %s Q4 attn_q_b F16 acceleration preflight "
+                "could not be completed\n",
+                ds4_backend_name(e->backend));
+        return 0;
+    }
+    if (rc > 0) {
+        s->q4_attn_q_b_f16_sidecars_generation =
+            ds4_gpu_q4_attn_q_b_f16_cache_generation();
+        s->q4_attn_q_b_f16_prepared_rows = max_batch_rows;
+    }
+    return 1;
+}
+#endif
+
+static void ds4_session_mark_engine_counted(ds4_session *s) {
+    if (!s || !s->engine || s->engine_session_counted) return;
+    __atomic_add_fetch(
+        &s->engine->live_session_count, 1u, __ATOMIC_RELAXED);
+    s->engine_session_counted = true;
+}
+
 int ds4_session_create(ds4_session **out, ds4_engine *e, int ctx_size) {
     if (!out || !e || ctx_size <= 0) return 1;
     if (e->backend == DS4_BACKEND_CPU) {
@@ -64656,6 +67718,7 @@ int ds4_session_create(ds4_session **out, ds4_engine *e, int ctx_size) {
             ds4_session_free(s);
             return 1;
         }
+        ds4_session_mark_engine_counted(s);
         *out = s;
         return 0;
     }
@@ -64782,6 +67845,7 @@ int ds4_session_create(ds4_session **out, ds4_engine *e, int ctx_size) {
             ds4_session_free(s);
             return 1;
         }
+        ds4_session_mark_engine_counted(s);
         *out = s;
         return 0;
     }
@@ -64797,13 +67861,32 @@ int ds4_session_create(ds4_session **out, ds4_engine *e, int ctx_size) {
     const bool need_spec_verifier =
         e->mtp_ready ||
         (e->support_kind == DS4_SUPPORT_DSPARK && e->dspark) ||
-        e->tp.active; /* TP worker mirrors the leader's verify blocks */
+        e->tp.active /* TP worker mirrors the leader's verify blocks */
+#ifdef DS4_TEST_HOOKS
+        || getenv("DS4_TEST_METAL_EXACTN_ORACLE") != NULL
+#endif
+        ;
     const int *placement = e->multi_tier ? e->placement : NULL;
     const ds4_gpu_graph *shared_prefill_workspace =
         e->share_session_prefill_workspace &&
         e->shared_prefill_workspace_ready
             ? &e->shared_prefill_workspace
             : NULL;
+    if (ds4_backend_uses_graph(e->backend) &&
+        __atomic_load_n(
+            &e->live_session_count, __ATOMIC_RELAXED) != 0u) {
+        /* The graph estimator intentionally omits several large prefill
+         * workspaces.  Evict conservatively before dynamic session growth;
+         * sessions created together at startup see an empty cache and pay
+         * nothing.  Cache generations make existing sessions re-prewarm. */
+        if (!ds4_gpu_make_room_for_q4_attn_q_b_f16_session()) {
+            fprintf(stderr,
+                    "ds4: could not make room for the %s session graph\n",
+                    ds4_backend_name(e->backend));
+            free(s);
+            return 1;
+        }
+    }
     s->graph.dspark_exec_tier = e->multi_tier ? e->dspark_exec_tier : 0;
     if (!metal_graph_alloc_raw_cap(&s->graph, &e->weights, shape_layer,
                                    raw_cap, (uint32_t)ctx_size, s->prefill_cap,
@@ -64878,7 +67961,10 @@ int ds4_session_create(ds4_session **out, ds4_engine *e, int ctx_size) {
         free(s);
         return 1;
     }
-    if (e->support_kind == DS4_SUPPORT_DSPARK) {
+    /* Loading a DSpark support GGUF is not, by itself, a request to run the
+     * speculative runtime.  Avoid allocating capture buffers and reducing
+     * every prompt row unless --dspark is actually enabled. */
+    if (e->support_kind == DS4_SUPPORT_DSPARK && e->dspark) {
         if (!metal_graph_configure_dspark_capture(&s->graph,
                                                   &e->dspark_weights)) {
             fprintf(stderr, "ds4: failed to configure DSpark target-hidden capture\n");
@@ -64897,6 +67983,21 @@ int ds4_session_create(ds4_session **out, ds4_engine *e, int ctx_size) {
             fprintf(stderr, "\n");
         }
     }
+    /* Remote workers do not necessarily enter the local sync preflight before
+     * their first mirrored/layer-slice batch, so keep eager preparation for
+     * TP/distributed sessions.  Local sessions defer until the real prompt is
+     * known, avoiding a 2.69 GiB decode-only allocation. */
+    if ((e->tp.active ||
+         e->distributed.role != DS4_DISTRIBUTED_NONE) &&
+        !ds4_session_prepare_q4_attn_q_b_sidecars(
+            s, s->prefill_cap, true)) {
+        if (__atomic_load_n(&e->live_session_count, __ATOMIC_RELAXED) == 0u) {
+            (void)ds4_gpu_release_q4_attn_q_b_f16_sidecars();
+        }
+        metal_graph_free(&s->graph);
+        free(s);
+        return 1;
+    }
     s->logits = xmalloc((size_t)DS4_N_VOCAB * sizeof(s->logits[0]));
     s->sample_probs =
         xmalloc((size_t)DS4_N_VOCAB * sizeof(s->sample_probs[0]));
@@ -64933,6 +68034,10 @@ int ds4_session_create(ds4_session **out, ds4_engine *e, int ctx_size) {
             fprintf(stderr,
                     "ds4: failed to create distributed coordinator session: %s\n",
                     err[0] ? err : "unknown error");
+            if (__atomic_load_n(
+                    &e->live_session_count, __ATOMIC_RELAXED) == 0u) {
+                (void)ds4_gpu_release_q4_attn_q_b_f16_sidecars();
+            }
             metal_graph_free(&s->graph);
             free(s->logits);
             free(s->sample_probs);
@@ -64948,6 +68053,7 @@ int ds4_session_create(ds4_session **out, ds4_engine *e, int ctx_size) {
         ds4_session_free(s);
         return 1;
     }
+    ds4_session_mark_engine_counted(s);
     *out = s;
     return 0;
 #endif
@@ -64955,6 +68061,31 @@ int ds4_session_create(ds4_session **out, ds4_engine *e, int ctx_size) {
 
 void ds4_session_free(ds4_session *s) {
     if (!s) return;
+#ifndef DS4_NO_GPU
+    bool release_gpu_q4_sidecars = false;
+#endif
+    if (s->engine && s->engine_session_counted) {
+        const uint32_t remaining_sessions = __atomic_sub_fetch(
+            &s->engine->live_session_count, 1u, __ATOMIC_RELAXED);
+#ifndef DS4_NO_GPU
+        release_gpu_q4_sidecars =
+            remaining_sessions == 0u &&
+            ds4_backend_uses_graph(s->engine->backend);
+#else
+        (void)remaining_sessions;
+#endif
+        s->engine_session_counted = false;
+    }
+#ifndef DS4_NO_GPU
+    else if (s->engine && ds4_backend_uses_graph(s->engine->backend) &&
+             __atomic_load_n(
+                 &s->engine->live_session_count, __ATOMIC_RELAXED) == 0u) {
+        /* An eager TP/distributed prewarm may have succeeded before session
+         * registration failed, so this uncounted session can still own the
+         * process-global sidecars. */
+        release_gpu_q4_sidecars = true;
+    }
+#endif
     if (ds4_session_tp_leader(s) && s->tp_session_id != 0 &&
         !ds4_tp_failed(s->engine->tp.ctx)) {
         char err[256] = "";
@@ -64977,6 +68108,13 @@ void ds4_session_free(ds4_session *s) {
     }
 #ifndef DS4_NO_GPU
     else {
+        if (release_gpu_q4_sidecars &&
+            !ds4_gpu_release_q4_attn_q_b_f16_sidecars()) {
+            fprintf(stderr,
+                    "ds4: WARNING: could not release resident %s Q4 "
+                    "attn_q_b F16 sidecars at session teardown\n",
+                    ds4_backend_name(s->engine->backend));
+        }
         if (ds4_session_is_glm(s)) {
             glm_graph_free(&s->glm_graph);
         } else {
@@ -66299,6 +69437,68 @@ static bool ds4_session_store_vision_identities(ds4_session *s) {
     return true;
 }
 
+/* Prepare one-shot resources for the actual work the next sync will emit,
+ * without changing checkpoint/KV state.  Timed frontends may call this before
+ * starting their clock; ds4_session_sync() repeats it as an idempotent safety
+ * net for API users that do not. */
+int ds4_session_prepare_sync(ds4_session *s,
+                             const ds4_tokens *prompt,
+                             char *err,
+                             size_t errlen) {
+    if (!s || !prompt) {
+        if (err && errlen) snprintf(err, errlen, "missing session or prompt");
+        return 1;
+    }
+    if (prompt->len <= 0) {
+        if (err && errlen) snprintf(err, errlen, "empty prompt");
+        return 1;
+    }
+    if (prompt->len >= s->ctx_size) {
+        if (err && errlen) {
+            snprintf(err, errlen,
+                     "prompt length %d exceeds context %d "
+                     "(one token of generation room is required)",
+                     prompt->len, s->ctx_size);
+        }
+        return 1;
+    }
+
+#ifndef DS4_NO_GPU
+    if (!s->engine || !ds4_backend_uses_graph(s->engine->backend) ||
+        ds4_session_is_cpu(s) || ds4_session_is_glm(s)) {
+        return 0;
+    }
+
+    uint32_t start = 0u;
+    uint32_t rows = (uint32_t)prompt->len;
+    if (s->checkpoint_valid &&
+        prompt->len >= s->checkpoint.len &&
+        ds4_tokens_starts_with(prompt, &s->checkpoint)) {
+        start = (uint32_t)s->checkpoint.len;
+        rows = (uint32_t)(prompt->len - s->checkpoint.len);
+        if (rows < metal_graph_resume_prefill_min_tokens()) return 0;
+    }
+    if (rows == 0u) return 0;
+
+    const uint32_t max_batch_rows = metal_graph_prefill_max_chunk_rows(
+        &s->graph, start, rows);
+    if (max_batch_rows == 0u) return 0;
+    if (!ds4_session_prepare_q4_attn_q_b_sidecars(
+            s, max_batch_rows, false)) {
+        if (err && errlen) {
+            snprintf(err, errlen,
+                     "required %s Q4 attn_q_b F16 acceleration preflight failed",
+                     ds4_backend_name(s->engine->backend));
+        }
+        return 1;
+    }
+#else
+    (void)err;
+    (void)errlen;
+#endif
+    return 0;
+}
+
 /* Under tensor parallelism the leader mirrors every public sync/eval to the
  * worker before doing the work itself, so both engines execute the same
  * graph sequence and the per-layer gates pair up.  The worker acks a sync
@@ -66309,9 +69509,7 @@ int ds4_session_sync(ds4_session *s, const ds4_tokens *prompt, char *err, size_t
                      s, s->sync_images, s->sync_image_count)) {
         ds4_session_invalidate(s);
     }
-#ifndef DS4_NO_GPU
-    ds4_session_dspark_scheduler_begin_request(s);
-#endif
+    if (ds4_session_prepare_sync(s, prompt, err, errlen) != 0) return 1;
     const bool mirror = ds4_session_tp_leader(s);
     if (mirror && prompt && prompt->len > 0) {
         if (s->sync_image_count > UINT32_MAX) {
@@ -67726,11 +70924,7 @@ static bool ds4_session_prepare_dspark_draft_impl(ds4_session *s,
         s->engine->dspark_confidence_threshold < 0.8f ?
             0.8f : s->engine->dspark_confidence_threshold;
     const bool stats_enabled = ds4_dspark_stats_enabled();
-    const bool scheduler_enabled = ds4_dspark_scheduler_enabled(s);
-    const bool time_enabled =
-        stats_enabled ||
-        (scheduler_enabled && ds4_dspark_scheduler_timing_enabled());
-    const double stats_t0 = time_enabled ? now_sec() : 0.0;
+    const double stats_t0 = stats_enabled ? now_sec() : 0.0;
 #define DS4_DSPARK_PROP_T0() (stats_enabled ? now_sec() : 0.0)
 #define DS4_DSPARK_PROP_ADD(field_, t0_) do {                              \
         if (stats_enabled) {                                                \
@@ -67740,28 +70934,32 @@ static bool ds4_session_prepare_dspark_draft_impl(ds4_session *s,
     s->dspark_draft_valid = false;
     s->dspark_draft_len = 0;
     s->dspark_stochastic_draft = false;
-    s->dspark_last_confidence0 = 0.0f;
-    s->dspark_last_confidence0_valid = false;
-    if (scheduler_enabled) s->dspark_last_propose_ms = 0.0;
-    if (enabled && !fake_argmax_enabled &&
-        ds4_session_dspark_scheduler_should_skip(s)) {
-        (void)metal_graph_dspark_ring_maintain(&s->graph,
-                                               &s->engine->mtp_model,
-                                               &s->engine->dspark_weights,
-                                               pos);
-        const double propose_ms =
-            time_enabled ? (now_sec() - stats_t0) * 1000.0 : 0.0;
-        if (scheduler_enabled) s->dspark_last_propose_ms = propose_ms;
-        if (stats_enabled) {
-            s->dspark_stats.propose_ms += propose_ms;
-        }
-        return false;
-    }
     if (probe_log || enabled) {
         const bool capture_ok = ds4_session_dspark_capture_current(s);
         const bool batch_capture_ok =
             ds4_session_dspark_capture_batch_current(s);
-        const ds4_dspark_weights *dw = &s->engine->dspark_weights;
+        const ds4_dspark_weights *native_dw = &s->engine->dspark_weights;
+        ds4_dspark_weights capped_dw;
+        const ds4_dspark_weights *dw = native_dw;
+        /* Backends choose their proposal width independently.  CUDA exact-2
+         * keeps its isolated A/B control; Metal defaults to the effective
+         * verifier/cache budget.  Buffers retain their native capacity. */
+        uint32_t proposer_cap = native_dw->block_size;
+        if (enabled) {
+            proposer_cap = ds4_session_cuda_dspark_proposer_block_cap(
+                    s, proposer_cap);
+            proposer_cap = ds4_session_metal_dspark_proposer_block_cap(
+                    s, proposer_cap);
+        }
+        if (proposer_cap < native_dw->block_size) {
+            capped_dw = *native_dw;
+            capped_dw.block_size = proposer_cap;
+            dw = &capped_dw;
+            if (stats_enabled) s->dspark_stats.proposer_capped++;
+        }
+        if (stats_enabled) {
+            s->dspark_stats.proposer_scheduled_rows += dw->block_size;
+        }
         const uint32_t verify_cap = ds4_dspark_env_u32(
             "DS4_DSPARK_VERIFY_CAP",
             ds4_session_dspark_rocm_gfx1151_fast_path(s) ? 5u : 0u);
@@ -67969,8 +71167,49 @@ static bool ds4_session_prepare_dspark_draft_impl(ds4_session *s,
             markov_ready && !probe_log && confidence_threshold > 0.0f;
         if (lazy_runtime_confidence) {
             const double markov_t0 = DS4_DSPARK_PROP_T0();
-            markov_ok =
-                dspark_apply_markov_confidence_lazy_runtime(
+            bool device_attempted = false;
+            bool device_policy_mismatch = false;
+            markov_ok = dspark_apply_markov_confidence_device_runtime(
+                    &s->graph,
+                    &s->engine->mtp_model,
+                    dw,
+                    token,
+                    confidence_threshold,
+                    markov_proposal,
+                    &markov_proposal_len,
+                    &confidence_len,
+                    &confidence_prefix_len,
+                    reuse_confidence0_markov,
+                    &confidence0,
+                    &device_attempted,
+                    &device_policy_mismatch);
+            if (stats_enabled && device_attempted) {
+#if defined(__APPLE__)
+                s->dspark_stats.metal_device_proposer_attempts++;
+                if (markov_ok) {
+                    s->dspark_stats.metal_device_proposer_uses++;
+                } else {
+                    s->dspark_stats.metal_device_proposer_fallbacks++;
+                    if (device_policy_mismatch) {
+                        s->dspark_stats
+                            .metal_device_proposer_policy_mismatches++;
+                    }
+                }
+#else
+                s->dspark_stats.cuda_device_proposer_attempts++;
+                if (markov_ok) {
+                    s->dspark_stats.cuda_device_proposer_uses++;
+                } else {
+                    s->dspark_stats.cuda_device_proposer_fallbacks++;
+                    if (device_policy_mismatch) {
+                        s->dspark_stats
+                            .cuda_device_proposer_policy_mismatches++;
+                    }
+                }
+#endif
+            }
+            if (!markov_ok) {
+                markov_ok = dspark_apply_markov_confidence_lazy_runtime(
                         &s->graph,
                         &s->engine->mtp_model,
                         dw,
@@ -67986,6 +71225,7 @@ static bool ds4_session_prepare_dspark_draft_impl(ds4_session *s,
                         &confidence_prefix_len,
                         reuse_confidence0_markov,
                         &confidence0);
+            }
             DS4_DSPARK_PROP_ADD(propose_markov_ms, markov_t0);
             confidence_ok = markov_ok;
         } else if (markov_ready) {
@@ -68125,10 +71365,6 @@ static bool ds4_session_prepare_dspark_draft_impl(ds4_session *s,
             s->dspark_stochastic_draft =
                 s->dspark_draft_valid && stochastic_requested;
         }
-        if (confidence_ok && confidence_len != 0) {
-            s->dspark_last_confidence0 = confidence0;
-            s->dspark_last_confidence0_valid = true;
-        }
         bool fake_argmax_ok = false;
         if (!s->dspark_draft_valid && fake_argmax_enabled) {
             s->dspark_draft_tokens[0] = sample_argmax(s->logits, DS4_N_VOCAB);
@@ -68250,10 +71486,8 @@ static bool ds4_session_prepare_dspark_draft_impl(ds4_session *s,
                     dw->metadata_errors);
         }
     }
-    if (time_enabled) {
-        const double propose_ms = (now_sec() - stats_t0) * 1000.0;
-        if (scheduler_enabled) s->dspark_last_propose_ms = propose_ms;
-        if (stats_enabled) s->dspark_stats.propose_ms += propose_ms;
+    if (stats_enabled) {
+        s->dspark_stats.propose_ms += (now_sec() - stats_t0) * 1000.0;
     }
 #undef DS4_DSPARK_PROP_ADD
 #undef DS4_DSPARK_PROP_T0
@@ -68273,7 +71507,17 @@ static bool ds4_session_prepare_dspark_draft(ds4_session *s,
         if (ds4_gpu_set_current_device(exec_tier) != 0) return false;
         g->active_tier = exec_tier;
     }
-    const bool ok = ds4_session_prepare_dspark_draft_impl(s, token, pos);
+    bool ok = true;
+    if (g->ssd_streaming) {
+        ok = metal_graph_stream_map_decode_static_all(&s->engine->model,
+                                                       &s->engine->weights);
+        if (ok) {
+            g->streaming_static_decode_map_current =
+                metal_graph_stream_decode_static_map_enabled() &&
+                metal_graph_stream_decode_static_map_state_cache_enabled();
+        }
+    }
+    if (ok) ok = ds4_session_prepare_dspark_draft_impl(s, token, pos);
     if (exec_tier != saved_tier) {
         g->active_tier = saved_tier;
         if (ds4_gpu_set_current_device(saved_tier) != 0) return false;
@@ -68436,9 +71680,7 @@ static int ds4_session_eval_internal(ds4_session *s, int token, bool probe_mtp,
     }
     const bool dspark_target_timing =
         e->support_kind == DS4_SUPPORT_DSPARK &&
-        (ds4_dspark_stats_enabled() ||
-         (ds4_dspark_scheduler_enabled(s) &&
-          ds4_dspark_scheduler_timing_enabled()));
+        ds4_dspark_stats_enabled();
     const double target_t0 = dspark_target_timing ? now_sec() : 0.0;
     if (!metal_graph_eval_token_raw_swa(&s->graph, &e->model, &e->weights,
                                         (uint32_t)token,
@@ -68452,9 +71694,7 @@ static int ds4_session_eval_internal(ds4_session *s, int token, bool probe_mtp,
     if (dspark_target_timing) {
         const double target_ms = (now_sec() - target_t0) * 1000.0;
         s->dspark_last_target_eval_ms = target_ms;
-        if (ds4_dspark_stats_enabled()) {
-            s->dspark_stats.target_ms += target_ms;
-        }
+        s->dspark_stats.target_ms += target_ms;
     }
     token_vec_push(&s->checkpoint, token);
     s->checkpoint_valid = true;
@@ -68513,6 +71753,232 @@ static int ds4_session_eval_probe_tp(ds4_session *s, int token, bool probe_mtp,
     return rc;
 }
 
+#if !defined(DS4_NO_GPU) && defined(__APPLE__)
+/* Small fail-closed policy shared by runtime admission and its test hook.
+ * Detailed graph/layout checks remain below; this layer owns the externally
+ * controllable arm and the state classes that must never reach multi-queue
+ * execution. */
+static bool ds4_q4_stream_overlap_policy(
+        int  count,
+        bool resident,
+        bool ssd_streaming,
+        bool quality) {
+    const bool enabled = metal_graph_tp_env_flag(
+            "DS4_METAL_ENABLE_Q4_STREAM_OVERLAP", false);
+    const bool disabled = metal_graph_tp_env_flag(
+            "DS4_METAL_DISABLE_Q4_STREAM_OVERLAP", false);
+    return enabled && !disabled &&
+           count >= 2 && count <= DS4_METAL_SESSION_STREAMS &&
+           resident && !ssd_streaming && !quality;
+}
+
+#ifdef DS4_TEST_HOOKS
+int ds4_test_q4_stream_overlap_policy(
+        int  count,
+        bool resident,
+        bool ssd_streaming,
+        bool quality) {
+    return ds4_q4_stream_overlap_policy(count,
+                                        resident,
+                                        ssd_streaming,
+                                        quality) ? 1 : 0;
+}
+#endif
+#endif
+
+#if !defined(DS4_NO_GPU) && defined(__APPLE__)
+static bool metal_graph_mixed_workspace_compatible(
+        const ds4_gpu_graph *owner,
+        const ds4_gpu_graph *member);
+
+static bool ds4_engine_has_q4_stream_overlap_weights(const ds4_engine *e) {
+    if (!e) return false;
+    for (uint32_t il = 0; il < DS4_N_LAYER; il++) {
+        const ds4_layer_weights *layer = &e->weights.layer[il];
+        const bool aproj_q4 =
+            layer->attn_q_a && layer->attn_kv &&
+            layer->attn_q_a->type == DS4_TENSOR_Q4_K &&
+            layer->attn_kv->type == DS4_TENSOR_Q4_K;
+        const bool routed_q4 =
+            layer->ffn_gate_exps && layer->ffn_up_exps &&
+            layer->ffn_down_exps &&
+            layer->ffn_gate_exps->type == DS4_TENSOR_Q4_K &&
+            layer->ffn_up_exps->type == DS4_TENSOR_Q4_K &&
+            layer->ffn_down_exps->type == DS4_TENSOR_Q4_K;
+        if (aproj_q4 || routed_q4) return true;
+    }
+    return false;
+}
+
+static bool ds4_session_q4_stream_overlap_eligible(ds4_session *s) {
+    if (!s || !s->engine) return false;
+    const ds4_engine *e = s->engine;
+    const ds4_gpu_graph *g = &s->graph;
+    return e->backend == DS4_BACKEND_METAL &&
+           e->support_kind == DS4_SUPPORT_NONE &&
+           !e->tp.active && !e->multi_tier && !e->ssd_streaming &&
+           !s->distributed && !ds4_session_is_cpu(s) &&
+           !ds4_session_is_glm(s) && !ds4_session_cancelled(s) &&
+           s->checkpoint_valid &&
+           !g->ssd_streaming && !g->placement && !g->quality &&
+           g->tp_world < 2 && !g->materialize_ffn_out &&
+           !g->decode_stage_profile &&
+           !g->decode_index_stage_profile &&
+           !g->output_stage_profile && !g_expert_profile.active &&
+           !metal_graph_hc_norm_fusion_check_enabled() &&
+           !metal_graph_use_reference_shared_down_hc() &&
+           !metal_graph_use_pro_q4_cpu_router() &&
+           !metal_graph_use_q4_selected_shared_overlap(g) &&
+           !metal_graph_q4_non_streaming_opt_in_enabled() &&
+           !metal_graph_directional_steering_attn_enabled(g) &&
+           !metal_graph_directional_steering_ffn_enabled(g) &&
+           getenv("DS4_METAL_GRAPH_DUMP_PREFIX") == NULL &&
+           getenv("DS4_METAL_DECODE_STAGE_PROFILE") == NULL &&
+           getenv("DS4_METAL_LAYER_STAGE_PROFILE") == NULL &&
+           getenv("DS4_METAL_ATTN_OUT_STAGE_PROFILE") == NULL &&
+           getenv("DS4_METAL_FLASH_ATTN_STAGE_PROFILE") == NULL &&
+           getenv("DS4_METAL_MOE_ONE_STAGE_PROFILE") == NULL &&
+           getenv("DS4_METAL_MOE_STAGE_PROFILE") == NULL &&
+           ds4_engine_has_q4_stream_overlap_weights(e);
+}
+
+static bool ds4_q4_ssd_session_union_required(void) {
+    if (metal_graph_tp_env_flag(
+            "DS4_METAL_DISABLE_Q4_SSD_SESSION_UNION", false)) {
+        return false;
+    }
+    return metal_graph_tp_env_flag(
+            "DS4_METAL_REQUIRE_Q4_SSD_SESSION_UNION", false);
+}
+
+static bool ds4_q4_ssd_session_union_requested(void) {
+    const bool disabled = metal_graph_tp_env_flag(
+            "DS4_METAL_DISABLE_Q4_SSD_SESSION_UNION", false);
+    const bool enabled = metal_graph_tp_env_flag(
+            "DS4_METAL_ENABLE_Q4_SSD_SESSION_UNION", false) ||
+        ds4_q4_ssd_session_union_required();
+    return enabled && !disabled;
+}
+
+static bool ds4_q4_ssd_session_union_workspace_compatible(
+        const ds4_gpu_graph *owner,
+        const ds4_gpu_graph *member) {
+    if (!owner || !member ||
+        !metal_graph_mixed_workspace_compatible(owner, member)) {
+        return false;
+    }
+    const int t = owner->active_tier;
+    return t == 0 && member->active_tier == t &&
+           owner->batch_after_attn_hc_by_tier[t] ==
+               member->batch_after_attn_hc_by_tier[t] &&
+           owner->batch_ffn_cur_by_tier[t] ==
+               member->batch_ffn_cur_by_tier[t] &&
+           owner->batch_ffn_norm_by_tier[t] ==
+               member->batch_ffn_norm_by_tier[t] &&
+           owner->batch_hc_split_by_tier[t] ==
+               member->batch_hc_split_by_tier[t] &&
+           owner->batch_router_selected_by_tier[t] ==
+               member->batch_router_selected_by_tier[t] &&
+           owner->batch_router_weights_by_tier[t] ==
+               member->batch_router_weights_by_tier[t];
+}
+
+static bool ds4_engine_q4_ssd_session_union_layout_supported(
+        const ds4_engine    *e,
+        const ds4_gpu_graph *g) {
+    if (!e || !g || DS4_MODEL_FAMILY != DS4_MODEL_FAMILY_DEEPSEEK4 ||
+        DS4_N_LAYER == 0u || DS4_N_EXPERT_USED != 6u ||
+        DS4_N_EXPERT < 128u || DS4_N_EXPERT > 384u ||
+        !e->weights.output ||
+        e->weights.output->type != DS4_TENSOR_Q8_0) {
+        return false;
+    }
+    for (uint32_t il = 0; il < DS4_N_LAYER; il++) {
+        const ds4_layer_weights *layer = &e->weights.layer[il];
+        if (!weights_layer_has_required(layer, il) ||
+            !layer->attn_q_a || !layer->attn_kv ||
+            layer->attn_q_a->type != DS4_TENSOR_Q4_K ||
+            layer->attn_kv->type != DS4_TENSOR_Q4_K ||
+            !layer->ffn_gate_shexp || !layer->ffn_up_shexp ||
+            !layer->ffn_down_shexp ||
+            layer->ffn_gate_shexp->type != DS4_TENSOR_Q8_0 ||
+            layer->ffn_up_shexp->type != DS4_TENSOR_Q8_0 ||
+            layer->ffn_down_shexp->type != DS4_TENSOR_Q8_0 ||
+            !layer->ffn_gate_exps || !layer->ffn_up_exps ||
+            !layer->ffn_down_exps ||
+            layer->ffn_gate_exps->type != DS4_TENSOR_IQ2_XXS ||
+            layer->ffn_up_exps->type != DS4_TENSOR_IQ2_XXS ||
+            layer->ffn_down_exps->type != DS4_TENSOR_Q2_K ||
+            !weights_streaming_layer_experts_uniform(&e->weights, il) ||
+            !metal_graph_decode_iq2_selected_slots_expected(g, layer) ||
+            metal_graph_decode_cpu_router_applicable(g, layer)) {
+            return false;
+        }
+    }
+    return true;
+}
+
+static bool ds4_session_q4_ssd_session_union_eligible(
+        ds4_session         *s,
+        const ds4_gpu_graph *workspace,
+        int                  count) {
+    if (!s || !s->engine || !workspace || count < 2 ||
+        count > DS4_METAL_EXACTN_UNION_MAX_ROWS) {
+        return false;
+    }
+    const ds4_engine *e = s->engine;
+    const ds4_gpu_graph *g = &s->graph;
+    const uint64_t working_set =
+        (uint64_t)(uint32_t)count * (uint64_t)DS4_N_EXPERT_USED;
+    return ds4_q4_ssd_session_union_requested() &&
+           e->backend == DS4_BACKEND_METAL &&
+           e->support_kind == DS4_SUPPORT_NONE &&
+           e->ssd_streaming && !e->tp.active && !e->multi_tier &&
+           e->share_session_prefill_workspace &&
+           e->shared_prefill_workspace_ready &&
+           engine_placement_session_count(e) >= (uint32_t)count &&
+           ds4_q4_ssd_session_union_workspace_compatible(
+               &e->shared_prefill_workspace, workspace) &&
+           working_set <= UINT32_MAX &&
+           e->ssd_streaming_cache_experts >= (uint32_t)working_set &&
+           ds4_gpu_stream_expert_cache_configured_count() >=
+               (uint32_t)working_set &&
+           !s->distributed && !ds4_session_is_cpu(s) &&
+           !ds4_session_is_glm(s) && !ds4_session_cancelled(s) &&
+           s->checkpoint_valid &&
+           g->ssd_streaming && !g->placement && !g->quality &&
+           g->tp_world < 2u && g->raw_cap != 0u &&
+           g->active_tier == 0 && workspace->active_tier == 0 &&
+           workspace->prefill_cap >= (uint32_t)count &&
+           !g->materialize_ffn_out && !g->decode_stage_profile &&
+           !g->decode_index_stage_profile && !g->output_stage_profile &&
+           !g_expert_profile.active &&
+           !graph_power_throttle_enabled(g) &&
+           !g->spec_exactn_union_collect_routes &&
+           !metal_graph_hc_norm_fusion_check_enabled() &&
+           !metal_graph_use_reference_shared_down_hc() &&
+           !metal_graph_use_pro_q4_cpu_router() &&
+           !metal_graph_directional_steering_attn_enabled(g) &&
+           !metal_graph_directional_steering_ffn_enabled(g) &&
+           metal_graph_stream_decode_static_map_enabled() &&
+           metal_graph_debug_get_config()->prefix == NULL &&
+           getenv("DS4_METAL_GRAPH_DUMP_PREFIX") == NULL &&
+           getenv("DS4_METAL_DECODE_STAGE_PROFILE") == NULL &&
+           getenv("DS4_METAL_LAYER_STAGE_PROFILE") == NULL &&
+           getenv("DS4_METAL_ATTN_OUT_STAGE_PROFILE") == NULL &&
+           getenv("DS4_METAL_FLASH_ATTN_STAGE_PROFILE") == NULL &&
+           getenv("DS4_METAL_MOE_ONE_STAGE_PROFILE") == NULL &&
+           getenv("DS4_METAL_MOE_STAGE_PROFILE") == NULL &&
+           getenv("DS4_METAL_SELECTED_PROFILE") == NULL &&
+           getenv("DS4_METAL_Q4_SELECTED_PROFILE") == NULL &&
+           getenv("DS4_METAL_DSPARK_EXACT_ROWS_ASYNC_TAILS") == NULL &&
+           getenv("DS4_TP_ABLATE") == NULL &&
+           getenv("DS4_MOE_REPLAY_SELECTED_IDS") == NULL &&
+           getenv("DS4_MOE_RECORD_SELECTED_IDS") == NULL &&
+           ds4_engine_q4_ssd_session_union_layout_supported(e, g);
+}
+#endif
+
 int ds4_session_eval(ds4_session *s, int token, char *err, size_t errlen) {
     bool probe_mtp = true;
 #ifndef DS4_NO_GPU
@@ -69073,7 +72539,6 @@ static bool glm53_graph_encode_native_session_batch(
                                                 rows,
                                                 false,
                                                 false,
-                                                false,
                                                 NULL);
         stage = "FFN steering";
         if (ok) ok = glm_graph_apply_directional_steering_ffn(
@@ -69186,6 +72651,96 @@ static bool ds4_sessions_eval_batch_metal_supported(
     return true;
 }
 
+static bool ds4_sessions_q4_ssd_session_union_supported(
+        ds4_decode_item *items,
+        int              count,
+        ds4_engine      *e) {
+#if !defined(__APPLE__)
+    (void)items;
+    (void)count;
+    (void)e;
+    return false;
+#else
+    if (!items || count < 2 ||
+        count > DS4_METAL_EXACTN_UNION_MAX_ROWS || !e ||
+        !items[0].session) {
+        return false;
+    }
+    ds4_gpu_graph *workspace = &items[0].session->graph;
+    const uint64_t rows = (uint64_t)(uint32_t)count;
+    const uint64_t hc_dim = (uint64_t)DS4_N_HC * DS4_N_EMBD;
+    const uint64_t mix_hc = 2ull * DS4_N_HC +
+                            (uint64_t)DS4_N_HC * DS4_N_HC;
+    if (!metal_graph_batch_after_attn_hc(workspace) ||
+        !metal_graph_batch_ffn_cur(workspace) ||
+        !metal_graph_batch_ffn_norm(workspace) ||
+        !metal_graph_batch_hc_split(workspace) ||
+        !metal_graph_batch_router_selected(workspace) ||
+        !metal_graph_batch_router_weights(workspace) ||
+        ds4_gpu_tensor_bytes(metal_graph_batch_after_attn_hc(workspace)) <
+            rows * hc_dim * sizeof(float) ||
+        ds4_gpu_tensor_bytes(metal_graph_batch_ffn_cur(workspace)) <
+            rows * DS4_N_EMBD * sizeof(float) ||
+        ds4_gpu_tensor_bytes(metal_graph_batch_ffn_norm(workspace)) <
+            rows * DS4_N_EMBD * sizeof(float) ||
+        ds4_gpu_tensor_bytes(metal_graph_batch_hc_split(workspace)) <
+            rows * mix_hc * sizeof(float) ||
+        ds4_gpu_tensor_bytes(metal_graph_batch_router_selected(workspace)) <
+            rows * DS4_N_EXPERT_USED * sizeof(int32_t) ||
+        ds4_gpu_tensor_bytes(metal_graph_batch_router_weights(workspace)) <
+            rows * DS4_N_EXPERT_USED * sizeof(float)) {
+        return false;
+    }
+    for (int i = 0; i < count; i++) {
+        ds4_session *s = items[i].session;
+        if (!s || s->engine != e ||
+            !ds4_q4_ssd_session_union_workspace_compatible(
+                    &e->shared_prefill_workspace, &s->graph) ||
+            !ds4_session_q4_ssd_session_union_eligible(
+                    s, workspace, count)) {
+            return false;
+        }
+    }
+    return true;
+#endif
+}
+
+static bool ds4_sessions_q4_stream_overlap_supported(
+        ds4_decode_item *items,
+        int count,
+        ds4_engine *e) {
+#if !defined(__APPLE__)
+    (void)items;
+    (void)count;
+    (void)e;
+    return false;
+#else
+    if (!items || count <= 0 || !e || !items[0].session) {
+        return false;
+    }
+    const ds4_gpu_graph *first = &items[0].session->graph;
+    const bool resident =
+        e->backend == DS4_BACKEND_METAL &&
+        !e->multi_tier && !first->placement;
+    const bool ssd_streaming = e->ssd_streaming || first->ssd_streaming;
+    if (!ds4_q4_stream_overlap_policy(count,
+                                      resident,
+                                      ssd_streaming,
+                                      first->quality) ||
+        !ds4_engine_has_q4_stream_overlap_weights(e)) {
+        return false;
+    }
+    for (int i = 0; i < count; i++) {
+        ds4_session *s = items[i].session;
+        if (!s || s->engine != e ||
+            !ds4_session_q4_stream_overlap_eligible(s)) {
+            return false;
+        }
+    }
+    return true;
+#endif
+}
+
 static bool metal_graph_native_session_batch_shared_supported(
         ds4_decode_item *items,
         int count,
@@ -69545,12 +73100,246 @@ static bool ds4_sessions_tp_recv_logits(
     return true;
 }
 
+#if defined(__APPLE__)
+/* Queue-0 SSD batch for the AProjQ4 + routed IQ2/Q2 layout.  Each layer first
+ * encodes every independent session through its router into distinct rows of
+ * one borrowed prefill workspace.  The established exact-row backend then
+ * loads one immutable union of selected experts and runs every routed tail.
+ * A synchronous boundary retires that layer before the shared cache can be
+ * reused, avoiding both a second cache and the unordered multi-queue epoch. */
+static bool metal_graph_encode_q4_ssd_session_union(
+        ds4_decode_item   *items,
+        int                count,
+        const ds4_model   *model,
+        const ds4_weights *weights) {
+    if (!items || count < 2 ||
+        count > DS4_METAL_EXACTN_UNION_MAX_ROWS ||
+        !items[0].session || !model || !weights) {
+        return false;
+    }
+
+    ds4_gpu_graph *workspace = &items[0].session->graph;
+    metal_graph_exactn_union_row_alias
+        aliases[DS4_METAL_EXACTN_UNION_MAX_ROWS] = {0};
+    bool saved_capture[DS4_METAL_EXACTN_UNION_MAX_ROWS] = {0};
+    bool collecting = false;
+    bool ok = true;
+
+    for (int i = 0; i < count; i++) {
+        saved_capture[i] =
+            items[i].session->graph.spec_capture_prefixes;
+    }
+
+    ds4_gpu_tensor *selected_rows = ds4_gpu_tensor_view(
+            metal_graph_batch_router_selected(workspace),
+            0,
+            (uint64_t)(uint32_t)count * DS4_N_EXPERT_USED *
+                sizeof(int32_t));
+    ok = selected_rows != NULL;
+
+    /* Allocate and validate every row view before model mapping, KV writes,
+     * or command submission.  Later binds cannot fail due to allocation. */
+    for (int i = 0; ok && i < count; i++) {
+        ds4_gpu_graph *g = &items[i].session->graph;
+        ok = metal_graph_bind_exactn_union_row(
+                g,
+                workspace,
+                (uint32_t)i,
+                metal_graph_cur_hc(g),
+                metal_graph_after_ffn_hc(g),
+                &aliases[i]);
+        metal_graph_unbind_exactn_union_row(g, &aliases[i]);
+    }
+
+    /* The model-view registry is process-global.  Reinstall the complete
+     * static decode set before opening queue 0 so no layer remap can retire a
+     * view referenced by this batch. */
+    if (ok) ok = metal_graph_stream_map_decode_static_all(model, weights);
+    if (ok) {
+        const bool cache_map_state =
+            metal_graph_stream_decode_static_map_state_cache_enabled();
+        for (int i = 0; i < count; i++) {
+            items[i].session->graph.streaming_static_decode_map_current =
+                cache_map_state;
+        }
+    }
+
+    ds4_gpu_set_stream(0);
+    if (ok) ok = ds4_gpu_begin_commands() != 0;
+    for (int i = 0; ok && i < count; i++) {
+        ds4_gpu_graph *g = &items[i].session->graph;
+        g->spec_capture_prefixes = false;
+        metal_graph_dspark_capture_begin(g);
+        ok = ds4_gpu_embed_token_hc_tensor(
+                     metal_graph_cur_hc(g),
+                     model->map,
+                     model->size,
+                     weights->token_embd->abs_offset,
+                     (uint32_t)weights->token_embd->dim[1],
+                     (uint32_t)items[i].token,
+                     DS4_N_EMBD,
+                     DS4_N_HC) != 0;
+    }
+
+    for (uint32_t il = 0; ok && il < DS4_N_LAYER; il++) {
+        const ds4_layer_weights *layer = &weights->layer[il];
+        ok = ds4_gpu_stream_expert_exact_rows_begin_collect() != 0;
+        collecting = ok;
+
+        for (int i = 0; ok && i < count; i++) {
+            ds4_session *s = items[i].session;
+            ds4_gpu_graph *g = &s->graph;
+            const uint32_t pos = (uint32_t)s->checkpoint.len;
+            g->spec_exactn_union_collect_routes = true;
+            ok = metal_graph_bind_exactn_union_row(
+                    g,
+                    workspace,
+                    (uint32_t)i,
+                    metal_graph_cur_hc(g),
+                    metal_graph_after_ffn_hc(g),
+                    &aliases[i]);
+            if (ok) {
+                ok = metal_graph_encode_decode_layer_phase(
+                        g,
+                        model,
+                        layer,
+                        il,
+                        pos,
+                        g->layer_raw_cache[il],
+                        g->raw_cap,
+                        pos % g->raw_cap,
+                        metal_graph_raw_span_for_batch(g, pos, 1),
+                        items[i].token,
+                        METAL_DECODE_LAYER_TO_ROUTER);
+            }
+            metal_graph_unbind_exactn_union_row(g, &aliases[i]);
+            g->spec_exactn_union_collect_routes = false;
+        }
+
+        uint64_t gate_expert_bytes = 0;
+        uint64_t down_expert_bytes = 0;
+        if (ok) {
+            ok = streaming_layer_gate_down_expert_bytes(
+                    layer, &gate_expert_bytes, &down_expert_bytes);
+        }
+        if (ok) {
+            const ds4_gpu_stream_expert_table table =
+                graph_stream_expert_table_make(model,
+                                               layer,
+                                               il,
+                                               gate_expert_bytes,
+                                               down_expert_bytes);
+            ok = ds4_gpu_stream_expert_exact_rows_prepare(
+                         &table,
+                         selected_rows,
+                         (uint32_t)count,
+                         DS4_N_EXPERT_USED) != 0;
+        }
+
+        for (int i = 0; ok && i < count; i++) {
+            ds4_session *s = items[i].session;
+            ds4_gpu_graph *g = &s->graph;
+            const uint32_t pos = (uint32_t)s->checkpoint.len;
+            ok = metal_graph_bind_exactn_union_row(
+                    g,
+                    workspace,
+                    (uint32_t)i,
+                    metal_graph_cur_hc(g),
+                    metal_graph_after_ffn_hc(g),
+                    &aliases[i]);
+            if (ok) {
+                ok = ds4_gpu_stream_expert_exact_rows_set_row(
+                             (uint32_t)i) != 0;
+            }
+            if (ok) {
+                ok = metal_graph_encode_decode_layer_phase(
+                        g,
+                        model,
+                        layer,
+                        il,
+                        pos,
+                        g->layer_raw_cache[il],
+                        g->raw_cap,
+                        pos % g->raw_cap,
+                        metal_graph_raw_span_for_batch(g, pos, 1),
+                        items[i].token,
+                        METAL_DECODE_LAYER_FROM_ROUTER);
+            }
+            metal_graph_unbind_exactn_union_row(g, &aliases[i]);
+        }
+
+        if (ok) {
+            ok = ds4_gpu_end_commands() != 0;
+        } else {
+            (void)ds4_gpu_synchronize();
+        }
+        if (!ok) (void)ds4_gpu_synchronize();
+        if (collecting) {
+            ds4_gpu_stream_expert_exact_rows_release();
+            collecting = false;
+        }
+        if (!ok) break;
+
+        for (int i = 0; ok && i < count; i++) {
+            ds4_gpu_graph *g = &items[i].session->graph;
+            ds4_gpu_tensor *tmp = metal_graph_cur_hc(g);
+            g->cur_hc_by_tier[g->active_tier] =
+                metal_graph_after_ffn_hc(g);
+            g->after_ffn_hc_by_tier[g->active_tier] = tmp;
+            ok = metal_graph_dspark_capture_decode_layer(g, il);
+        }
+        if (ok && il + 1u < DS4_N_LAYER) {
+            ok = ds4_gpu_begin_commands() != 0;
+        }
+    }
+
+    if (ok) ok = ds4_gpu_begin_commands() != 0;
+    for (int i = 0; ok && i < count; i++) {
+        ok = metal_graph_encode_output_head(
+                &items[i].session->graph,
+                model,
+                weights,
+                weights->output->dim[1]);
+    }
+    if (ok) {
+        ok = ds4_gpu_end_commands() != 0;
+    } else {
+        (void)ds4_gpu_synchronize();
+    }
+
+    /* No view or exact-row resource may outlive work that references it. */
+    if (!ok) (void)ds4_gpu_synchronize();
+    if (collecting) ds4_gpu_stream_expert_exact_rows_release();
+    for (int i = 0; i < count; i++) {
+        ds4_gpu_graph *g = &items[i].session->graph;
+        g->spec_exactn_union_collect_routes = false;
+        g->spec_capture_prefixes = saved_capture[i];
+        metal_graph_release_exactn_union_row_alias(g, &aliases[i]);
+    }
+    ds4_gpu_tensor_free(selected_rows);
+    return ok;
+}
+#endif
+
 static int ds4_sessions_eval_batch_metal(
         ds4_decode_item *items,
         int count,
         ds4_engine *e,
         char *err,
         size_t errlen) {
+    const bool ssd_session_union =
+        ds4_sessions_q4_ssd_session_union_supported(items, count, e);
+    /* The generic Metal batch tape assumes resident model ranges.  If the
+     * opt-in changed between admission and execution, fail closed instead of
+     * accidentally running that tape against an SSD-streamed graph. */
+    if (e && e->ssd_streaming && !ssd_session_union) {
+        if (err && errlen) {
+            snprintf(err, errlen,
+                     "Metal SSD session union changed or became ineligible "
+                     "before execution");
+        }
+        return 1;
+    }
     const bool mirror = e->tp.active && e->tp.rank == 0;
     if (mirror) {
         ds4_tp_batch_item *wire = ds4_sessions_tp_batch_items(items, count);
@@ -69570,12 +73359,49 @@ static int ds4_sessions_eval_batch_metal(
 #if defined(__APPLE__)
     if (e->tp.active) ds4_gpu_tp_set_session_batch_mode(1);
 #endif
-    bool ok = ds4_gpu_begin_commands() != 0;
-    const bool native_glm53 = ok &&
+    bool native_glm53 = false;
+    bool native_shared = false;
+    bool native_qkv = false;
+    const bool stream_overlap =
+        !ssd_session_union &&
+        ds4_sessions_q4_stream_overlap_supported(items, count, e);
+    bool ok = true;
+#if defined(__APPLE__)
+    if (ssd_session_union) {
+        ok = metal_graph_encode_q4_ssd_session_union(
+                items, count, &e->model, &e->weights);
+    } else if (stream_overlap) {
+        int started = 0;
+        for (int i = 0; ok && i < count; i++) {
+            ds4_session *s = items[i].session;
+            ds4_gpu_set_stream(i);
+            ok = ds4_gpu_begin_commands() != 0;
+            if (ok) ok = metal_graph_encode_token_raw_swa(&s->graph,
+                                                          &e->model,
+                                                          &e->weights,
+                                                          items[i].token,
+                                                          (uint32_t)s->checkpoint.len,
+                                                          true,
+                                                          false);
+            if (ok) ok = ds4_gpu_end_commands_async() != 0;
+            if (ok) started = i + 1;
+        }
+        if (!ok && ds4_gpu_commands_active()) {
+            (void)ds4_gpu_synchronize();
+        }
+        for (int i = 0; i < started; i++) {
+            if (ds4_gpu_wait_stream(i) == 0) ok = false;
+        }
+        ds4_gpu_set_stream(0);
+        if (!ok) (void)ds4_gpu_synchronize();
+    } else {
+#endif
+    ok = ds4_gpu_begin_commands() != 0;
+    native_glm53 = ok &&
         glm53_graph_native_session_batch_supported(items, count);
-    const bool native_shared = ok && !native_glm53 &&
+    native_shared = ok && !native_glm53 &&
         metal_graph_native_session_batch_shared_supported(items, count, e);
-    const bool native_qkv = native_shared &&
+    native_qkv = native_shared &&
         metal_graph_native_session_batch_qkv_supported(items, count, e);
     if (native_glm53) {
         ok = glm53_graph_encode_native_session_batch(
@@ -69610,6 +73436,9 @@ static int ds4_sessions_eval_batch_metal(
     }
     if (ok) ok = ds4_gpu_end_commands() != 0;
     else (void)ds4_gpu_synchronize();
+#if defined(__APPLE__)
+    }
+#endif
 #if defined(__APPLE__)
     if (e->tp.active) ds4_gpu_tp_set_session_batch_mode(0);
     if (ok && e->tp.active && ds4_gpu_tp_failed()) {
@@ -69661,12 +73490,17 @@ static int ds4_sessions_eval_batch_metal(
     if (getenv("DS4_METAL_SESSION_BATCH_LOG") != NULL) {
         fprintf(stderr,
                 "ds4: Metal session batch rows=%d family=%s "
-                "native_glm53=%d native_shared=%d native_qkv=%d\n",
+                "native_glm53=%d native_shared=%d native_qkv=%d "
+                "stream_overlap=%d "
+                "streams=%d ssd_session_union=%d\n",
                 count,
                 ds4_session_is_glm(items[0].session) ? "glm" : "deepseek",
                 native_glm53 ? 1 : 0,
                 native_shared ? 1 : 0,
-                native_qkv ? 1 : 0);
+                native_qkv ? 1 : 0,
+                stream_overlap ? 1 : 0,
+                stream_overlap ? count : 1,
+                ssd_session_union ? 1 : 0);
     }
     return 0;
 }
@@ -69735,6 +73569,13 @@ static int ds4_sessions_eval_batch_with_prefill_metal(
     const uint64_t hc_dim = (uint64_t)DS4_N_HC * DS4_N_EMBD;
     const bool mirror = e->tp.active && e->tp.rank == 0;
 
+    /* This mixed path bypasses ds4_session_sync(), so run the same one-shot
+     * preparation before it can publish a TP command or encode layer work. */
+    if (ds4_session_prepare_sync(
+            prefill_session, prefill_prompt, err, errlen) != 0) {
+        return 1;
+    }
+
     if (mirror) {
         ds4_tp_batch_item *wire = ds4_sessions_tp_batch_items(items, count);
         const bool sent = wire &&
@@ -69973,9 +73814,25 @@ int ds4_sessions_eval_batch(ds4_decode_item *items, int count,
     if (e->backend == DS4_BACKEND_CUDA) {
         return ds4_sessions_eval_batch_cuda(items, count, err, errlen);
     }
-    if (ds4_sessions_eval_batch_metal_supported(items, count, e)) {
+    const bool q4_ssd_session_union =
+        ds4_sessions_q4_ssd_session_union_supported(items, count, e);
+    if (q4_ssd_session_union ||
+        ds4_sessions_eval_batch_metal_supported(items, count, e)) {
         return ds4_sessions_eval_batch_metal(items, count, e, err, errlen);
     }
+#if defined(__APPLE__)
+    if (e->backend == DS4_BACKEND_METAL && e->ssd_streaming &&
+        ds4_q4_ssd_session_union_required()) {
+        if (err && errlen) {
+            snprintf(err, errlen,
+                     "required Metal Q4 SSD session union is ineligible "
+                     "(need 2..5 valid sessions, AProjQ4 + routed "
+                     "IQ2_XXS/Q2_K, static decode map, and >= rows*6 "
+                     "cached experts)");
+        }
+        return 1;
+    }
+#endif
 #endif
 
     /* Preserve logical all-or-nothing behavior even on the serialized path.
@@ -70071,19 +73928,12 @@ static int ds4_session_eval_dspark_speculative_argmax(
         size_t       errlen) {
     const bool spec_log = getenv("DS4_DSPARK_SPEC_LOG") != NULL;
     const bool stats_enabled = s && ds4_dspark_stats_enabled();
-    const bool scheduler_enabled = s && ds4_dspark_scheduler_enabled(s);
-    const double stats_t0 =
-        (stats_enabled ||
-         (scheduler_enabled && ds4_dspark_scheduler_timing_enabled()))
-            ? now_sec() : 0.0;
+    const double stats_t0 = stats_enabled ? now_sec() : 0.0;
 #define DS4_DSPARK_STATS_FINISH() do {                                      \
         if (stats_enabled) {                                                \
             s->dspark_stats.total_ms += (now_sec() - stats_t0) * 1000.0;    \
         }                                                                   \
     } while (0)
-#define DS4_DSPARK_SCHED_EXTRA_MS()                                         \
-    ((scheduler_enabled && stats_t0 != 0.0) ?                                \
-     s->dspark_last_propose_ms + (now_sec() - stats_t0) * 1000.0 : 0.0)
     if (stats_enabled) {
         s->dspark_stats.cycles++;
         if (n_accept > 0) s->dspark_stats.first_tokens++;
@@ -70102,10 +73952,6 @@ static int ds4_session_eval_dspark_speculative_argmax(
             s->dspark_stats.no_draft++;
             ds4_dspark_stats_note_len(s->dspark_stats.accepted_len_hist, 0);
         }
-        if (s) {
-            ds4_session_dspark_scheduler_note(
-                    s, 0, true, DS4_DSPARK_SCHED_EXTRA_MS());
-        }
         if (spec_log) {
             fprintf(stderr, "ds4: DSpark spec skip no-draft\n");
         }
@@ -70118,6 +73964,21 @@ static int ds4_session_eval_dspark_speculative_argmax(
     if (draft_n > accepted_cap - n_accept) draft_n = accepted_cap - n_accept;
     int room = s->ctx_size - s->checkpoint.len;
     if (draft_n > room - 1) draft_n = room - 1;
+    const uint32_t verify_cap =
+        ds4_session_dspark_verify_block_cap(s);
+    if (draft_n > (int)verify_cap) draft_n = (int)verify_cap;
+    if (draft_n > 1 &&
+        !ds4_session_dspark_batch_raw_safe(s, (uint32_t)draft_n)) {
+        if (spec_log) {
+            fprintf(stderr,
+                    "ds4: DSpark verifier capped to one token for raw ring "
+                    "safety cap=%u window=%u pos=%d\n",
+                    s->graph.raw_cap,
+                    s->graph.raw_window,
+                    s->checkpoint.len);
+        }
+        draft_n = 1;
+    }
     if (draft_n <= 0) {
         s->dspark_draft_valid = false;
         s->dspark_draft_len = 0;
@@ -70173,8 +74034,6 @@ static int ds4_session_eval_dspark_speculative_argmax(
                 ds4_dspark_stats_note_len(
                     s->dspark_stats.accepted_len_hist, 0);
             }
-            ds4_session_dspark_scheduler_note(
-                s, 0, false, DS4_DSPARK_SCHED_EXTRA_MS());
             DS4_DSPARK_STATS_FINISH();
             return n_accept;
         }
@@ -70186,8 +74045,6 @@ static int ds4_session_eval_dspark_speculative_argmax(
             s->dspark_stats.first_misses++;
             ds4_dspark_stats_note_len(s->dspark_stats.accepted_len_hist, 0);
         }
-        ds4_session_dspark_scheduler_note(
-                s, 0, false, DS4_DSPARK_SCHED_EXTRA_MS());
         if (spec_log) {
             fprintf(stderr,
                     "ds4: DSpark spec miss first draft=%d base=%d\n",
@@ -70197,7 +74054,16 @@ static int ds4_session_eval_dspark_speculative_argmax(
         DS4_DSPARK_STATS_FINISH();
         return n_accept;
     }
-    if (drafts[0] == eos_token) draft_n = 1;
+    /* Never verify or commit state beyond EOS.  The verifier must still
+     * evaluate the EOS row itself so its directly committed state matches
+     * ordinary decode, but every later proposal is outside the generation. */
+    for (int i = 0; i < draft_n; i++) {
+        if (drafts[i] == eos_token) {
+            draft_n = i + 1;
+            break;
+        }
+    }
+
     ds4_engine *e = s->engine;
     ds4_spec_frontier frontier;
     memset(&frontier, 0, sizeof(frontier));
@@ -70205,14 +74071,695 @@ static int ds4_session_eval_dspark_speculative_argmax(
     int *row_tops = draft_n > 1 ? row_tops_buf : NULL;
     float *row_logits = s->spec_row_logits;
     const int start = s->checkpoint.len;
+#ifndef DS4_ROCM_BUILD
+    const bool cuda_rollback_replay =
+        e->backend == DS4_BACKEND_CUDA && !e->multi_tier && !e->tp.active;
+#else
+    const bool cuda_rollback_replay = false;
+#endif
+    const bool metal_headless_replay =
+        ds4_session_metal_dspark_headless_replay_enabled(s);
+    /* A four-row chunk at a ratio-4 boundary selects the aligned compressor
+     * kernel, while the legacy capture path intentionally uses sequential
+     * updates. Preserve acceptance arithmetic by retaining the legacy N-row
+     * verifier on those boundaries; all other N>=3 suffixes naturally take
+     * the same sequential compressor path after contracting to N-1 rows. */
+    const uint32_t metal_acceptance_rows =
+        draft_n > 1 ? (uint32_t)draft_n - 1u : 0u;
+    const bool metal_acceptance_aligned_ratio4 =
+        metal_acceptance_rows != 0u &&
+        (metal_acceptance_rows % 4u) == 0u &&
+        ((uint32_t)start % 4u) == 0u;
+    const bool metal_acceptance_only_verify =
+        draft_n > 2 && !metal_acceptance_aligned_ratio4 &&
+        ds4_session_metal_dspark_acceptance_only_verify_enabled(s);
+    /* The current target logits already verify a one-token draft.  With no
+     * mirrored TP worker there is no speculative verifier state to roll back:
+     * replay that token directly and avoid copying every compressor frontier. */
+    const bool skip_single_verify =
+        draft_n == 1 && (s->graph.ssd_streaming || cuda_rollback_replay) &&
+        metal_graph_dspark_cache_current_window_valid(&s->graph);
     const double snapshot_t0 = stats_enabled ? now_sec() : 0.0;
-    bool have_frontier = spec_frontier_snapshot(&frontier, s);
+    bool have_frontier = skip_single_verify ||
+                         spec_frontier_snapshot(&frontier, s);
     if (stats_enabled) {
         s->dspark_stats.snapshot_ms += (now_sec() - snapshot_t0) * 1000.0;
     }
     bool ok = have_frontier && row_logits && (draft_n <= 1 || row_tops);
     bool verifier_may_have_mutated = false;
     bool tp_verify_sent = false;
+    bool cuda_exactn_verified_partial = false;
+    bool metal_exactn_union_verified_partial = false;
+    int preverified_commit_drafts = 0;
+
+    /* Resident CUDA exact-N keeps the canonical one-token arithmetic while
+     * removing the per-token command/readback boundary.  A full match already
+     * owns the exact target state and commits immediately.  Partial/error
+     * attempts restore the pre-cycle frontier before entering the established
+     * batch verifier plus exact replay path below. */
+    const bool cuda_exactn =
+        ok && ds4_session_cuda_dspark_exactn_requested(
+                         s, (uint32_t)draft_n);
+    if (cuda_exactn) {
+        if (stats_enabled) {
+            s->dspark_stats.cuda_exactn_attempts++;
+            s->dspark_stats.cuda_exactn_rows += (uint64_t)draft_n;
+        }
+        bool exactn_ok = true;
+        bool exactn_mismatch = false;
+        int exactn_accepted_prefix = 1;
+        int exactn_last_top = -1;
+        const double exactn_t0 = stats_enabled ? now_sec() : 0.0;
+        ds4_cuda_exactn_timing exactn_timing;
+
+        exactn_ok = metal_graph_verify_decode_exactn_cuda_resident_impl(
+                &s->graph,
+                &e->model,
+                &e->weights,
+                drafts,
+                (uint32_t)draft_n,
+                (uint32_t)start,
+                row_tops,
+                row_logits,
+                stats_enabled ? &exactn_timing : NULL);
+        if (exactn_ok) {
+            for (int row = 0; row + 1 < draft_n; row++) {
+                exactn_last_top = row_tops[row];
+                if (exactn_last_top != drafts[row + 1]) {
+                    exactn_mismatch = true;
+                    break;
+                }
+                exactn_accepted_prefix = row + 2;
+            }
+        }
+        if (stats_enabled) {
+            const double exactn_ms = (now_sec() - exactn_t0) * 1000.0;
+            s->dspark_stats.verify_ms += exactn_ms;
+            s->dspark_stats.cuda_exactn_ms += exactn_ms;
+            s->dspark_stats.cuda_exactn_setup_ms += exactn_timing.setup_ms;
+            s->dspark_stats.cuda_exactn_layer_ms += exactn_timing.layer_ms;
+            s->dspark_stats.cuda_exactn_head_ms += exactn_timing.head_ms;
+            s->dspark_stats.cuda_exactn_read_ms += exactn_timing.read_ms;
+            if (exactn_timing.batch_head_attempted) {
+                s->dspark_stats.cuda_exactn_batch_head_attempts++;
+            }
+            if (exactn_timing.batch_head_used) {
+                s->dspark_stats.cuda_exactn_batch_head_uses++;
+            }
+            if (exactn_timing.batch_head_fallback) {
+                s->dspark_stats.cuda_exactn_batch_head_fallbacks++;
+            }
+            if (exactn_timing.graphs_attempted) {
+                s->dspark_stats.cuda_exactn_graph_attempts++;
+            }
+            if (exactn_timing.graphs_used) {
+                s->dspark_stats.cuda_exactn_graph_uses++;
+            }
+            s->dspark_stats.cuda_exactn_graph_captures +=
+                exactn_timing.graph_captures;
+            s->dspark_stats.cuda_exactn_graph_replays +=
+                exactn_timing.graph_replays;
+            s->dspark_stats.cuda_exactn_graph_warms +=
+                exactn_timing.graph_warms;
+            s->dspark_stats.cuda_exactn_graph_no_slots +=
+                exactn_timing.graph_no_slots;
+            s->dspark_stats.cuda_exactn_graph_failures +=
+                exactn_timing.graph_failures;
+        }
+
+        if (exactn_ok && !exactn_mismatch) {
+            memcpy(s->logits,
+                   row_logits,
+                   (size_t)DS4_N_VOCAB * sizeof(s->logits[0]));
+            /* Exact-N uses private hidden rows and cannot refresh the target
+             * hidden capture consumed by the next DSpark proposal. */
+            ds4_session_dspark_capture_invalidate(s);
+            for (int i = 0; i < draft_n; i++) {
+                token_vec_push(&s->checkpoint, drafts[i]);
+                accepted[n_accept++] = drafts[i];
+            }
+            s->checkpoint_valid = true;
+            ds4_session_dspark_capture_note_checkpoint(s);
+            if (stats_enabled) {
+                s->dspark_stats.full_accepts++;
+                s->dspark_stats.cuda_exactn_full_accepts++;
+                s->dspark_stats.accepted_draft_tokens +=
+                    (uint64_t)draft_n;
+                ds4_dspark_stats_note_len(
+                        s->dspark_stats.accepted_len_hist,
+                        (uint32_t)draft_n);
+            }
+            ds4_session_dspark_stats_note_saved(s, (uint32_t)draft_n);
+            if (spec_log) {
+                fprintf(stderr,
+                        "ds4: DSpark CUDA exactN drafted=%d "
+                        "accepted_draft=%d accepted_total=%d\n",
+                        draft_n,
+                        draft_n,
+                        n_accept);
+            }
+            spec_frontier_free(&frontier);
+            DS4_DSPARK_STATS_FINISH();
+            return n_accept;
+        }
+
+        if (stats_enabled) {
+            s->dspark_stats.cuda_exactn_fallbacks++;
+            if (exactn_mismatch) {
+                s->dspark_stats.cuda_exactn_partial_fallbacks++;
+            } else {
+                s->dspark_stats.cuda_exactn_error_fallbacks++;
+            }
+        }
+        s->checkpoint.len = start;
+        ds4_session_dspark_capture_invalidate(s);
+        const double exactn_restore_t0 = stats_enabled ? now_sec() : 0.0;
+        const bool exactn_restored =
+            have_frontier && spec_frontier_restore(&frontier, s);
+        if (stats_enabled) {
+            s->dspark_stats.cuda_exactn_restore_ms +=
+                (now_sec() - exactn_restore_t0) * 1000.0;
+        }
+        if (!exactn_restored) {
+            snprintf(err, errlen,
+                     "DSpark CUDA exactN rollback failed");
+            s->checkpoint_valid = false;
+            if (stats_enabled) {
+                s->dspark_stats.verifier_errors++;
+                ds4_dspark_stats_note_len(
+                        s->dspark_stats.accepted_len_hist, 0);
+            }
+            spec_frontier_free(&frontier);
+            DS4_DSPARK_STATS_FINISH();
+            return -1;
+        }
+        if (exactn_mismatch) {
+            /* row_tops already proves the exact accepted prefix.  The
+             * frontier above is the only rollback needed; skip the legacy
+             * N-row verifier and let the common replay tail advance exactly
+             * those accepted rows.  Backend errors still use the legacy
+             * verifier because their row_tops are not trustworthy. */
+            cuda_exactn_verified_partial = true;
+            if (stats_enabled) {
+                s->dspark_stats.cuda_exactn_partial_replays++;
+                s->dspark_stats.cuda_exactn_legacy_verify_skips++;
+            }
+        }
+        if (spec_log) {
+            if (exactn_mismatch) {
+                fprintf(stderr,
+                        "ds4: DSpark CUDA exactN partial "
+                        "accepted_prefix=%d verified_next=%d; restored "
+                        "frontier and replaying without legacy verify\n",
+                        exactn_accepted_prefix,
+                        exactn_last_top);
+            } else {
+                fprintf(stderr,
+                        "ds4: DSpark CUDA exactN unavailable; restored "
+                        "frontier for legacy verify/replay\n");
+            }
+        }
+    }
+
+    /* Fast Metal exact-N experiment.  Unlike the token-major oracle below,
+     * this advances all rows layer-major and shares one immutable selected-
+     * expert union per layer.  It may directly commit only a complete match.
+     * A verified partial restores once and replays only the proven prefix;
+     * backend errors still fall through to the oracle or legacy verifier. */
+    const bool exactn_union =
+        ok && ds4_session_metal_dspark_exactn_union_requested(
+                         s, (uint32_t)draft_n);
+    if (exactn_union) {
+        if (stats_enabled) s->dspark_stats.exactn_union_attempts++;
+        bool union_ok = true;
+        bool union_mismatch = false;
+        int union_accepted_prefix = 1;
+        int union_last_top = -1;
+        const double union_t0 = stats_enabled ? now_sec() : 0.0;
+        ds4_metal_exactn_union_timing union_timing = {0};
+
+        const bool static_map_cache =
+            metal_graph_stream_decode_static_map_enabled() &&
+            metal_graph_stream_decode_static_map_state_cache_enabled();
+        if (!static_map_cache ||
+            !s->graph.streaming_static_decode_map_current) {
+            union_ok = metal_graph_stream_map_decode_static_all(
+                    &e->model, &e->weights);
+            if (union_ok) {
+                s->graph.streaming_static_decode_map_current =
+                    static_map_cache;
+            }
+        }
+        if (union_ok) {
+            union_ok = metal_graph_verify_decode_exactn_union_impl(
+                    &s->graph,
+                    &e->model,
+                    &e->weights,
+                    drafts,
+                    (uint32_t)draft_n,
+                    (uint32_t)start,
+                    row_tops,
+                    row_logits,
+                    stats_enabled ? &union_timing : NULL);
+        }
+        if (union_ok) {
+            for (int row = 0; row + 1 < draft_n; row++) {
+                union_last_top = row_tops[row];
+                if (union_last_top != drafts[row + 1]) {
+                    union_mismatch = true;
+                    break;
+                }
+                union_accepted_prefix = row + 2;
+            }
+        }
+        if (stats_enabled) {
+            s->dspark_stats.verify_ms +=
+                (now_sec() - union_t0) * 1000.0;
+            if (union_timing.batch_head_attempted) {
+                s->dspark_stats.metal_exactn_batch_head_attempts++;
+            }
+            if (union_timing.batch_head_used) {
+                s->dspark_stats.metal_exactn_batch_head_uses++;
+            }
+            if (union_timing.batch_head_fallback) {
+                s->dspark_stats.metal_exactn_batch_head_fallbacks++;
+            }
+        }
+
+        if (union_ok && !union_mismatch) {
+            memcpy(s->logits,
+                   row_logits,
+                   (size_t)DS4_N_VOCAB * sizeof(s->logits[0]));
+            /* The union tape does not refresh the DSpark target-hidden rows.
+             * Never relabel the old capture as the newly committed frontier. */
+            ds4_session_dspark_capture_invalidate(s);
+            for (int i = 0; i < draft_n; i++) {
+                token_vec_push(&s->checkpoint, drafts[i]);
+                accepted[n_accept++] = drafts[i];
+            }
+            s->checkpoint_valid = true;
+            ds4_session_dspark_capture_note_checkpoint(s);
+            if (stats_enabled) {
+                s->dspark_stats.full_accepts++;
+                s->dspark_stats.exactn_union_full_accepts++;
+                s->dspark_stats.accepted_draft_tokens +=
+                    (uint64_t)draft_n;
+                ds4_dspark_stats_note_len(
+                        s->dspark_stats.accepted_len_hist,
+                        (uint32_t)draft_n);
+            }
+            ds4_session_dspark_stats_note_saved(s, (uint32_t)draft_n);
+            if (spec_log) {
+                fprintf(stderr,
+                        "ds4: DSpark Metal exactN union drafted=%d "
+                        "accepted_draft=%d accepted_total=%d\n",
+                        draft_n,
+                        draft_n,
+                        n_accept);
+            }
+            spec_frontier_free(&frontier);
+            DS4_DSPARK_STATS_FINISH();
+            return n_accept;
+        }
+
+        if (stats_enabled) {
+            s->dspark_stats.exactn_union_fallbacks++;
+            if (union_mismatch) {
+                s->dspark_stats.exactn_union_partial_fallbacks++;
+            } else {
+                s->dspark_stats.exactn_union_error_fallbacks++;
+            }
+        }
+        s->checkpoint.len = start;
+        ds4_session_dspark_capture_invalidate(s);
+        if (!have_frontier || !spec_frontier_restore(&frontier, s)) {
+            snprintf(err, errlen,
+                     "DSpark Metal exactN union rollback failed");
+            s->checkpoint_valid = false;
+            if (stats_enabled) {
+                s->dspark_stats.verifier_errors++;
+                ds4_dspark_stats_note_len(
+                        s->dspark_stats.accepted_len_hist, 0);
+            }
+            spec_frontier_free(&frontier);
+            DS4_DSPARK_STATS_FINISH();
+            return -1;
+        }
+        if (union_mismatch) {
+            /* The union verifier completed every row and its top-1 results
+             * already prove the accepted prefix.  The restored pre-cycle
+             * frontier is the only rollback required; replay that prefix
+             * canonically and avoid a second N-row SSD verifier pass. */
+            metal_exactn_union_verified_partial = true;
+            preverified_commit_drafts = union_accepted_prefix;
+            if (stats_enabled) {
+                s->dspark_stats.exactn_union_partial_replays++;
+                s->dspark_stats.exactn_union_legacy_verify_skips++;
+            }
+        }
+        if (spec_log) {
+            if (union_mismatch) {
+                fprintf(stderr,
+                        "ds4: DSpark Metal exactN union partial "
+                        "accepted_prefix=%d verified_next=%d; restored "
+                        "frontier and replaying without legacy verify\n",
+                        union_accepted_prefix,
+                        union_last_top);
+            } else {
+                fprintf(stderr,
+                        "ds4: DSpark Metal exactN union unavailable; "
+                        "restored frontier for fallback\n");
+            }
+        }
+    }
+
+    /* Correctness-first exact-N oracle for Metal SSD streaming.  Every row
+     * uses the ordinary decode entry point and completes before the next row
+     * starts.  This intentionally has no microbatch speed claim: it establishes
+     * the byte-identical state/output reference that a union-route kernel must
+     * match before its state can be committed. */
+    const bool exactn =
+        !metal_exactn_union_verified_partial && ok &&
+        ds4_session_metal_dspark_exactn_requested(
+                         s, (uint32_t)draft_n);
+    if (exactn) {
+        if (stats_enabled) s->dspark_stats.exactn_attempts++;
+        bool exactn_ok = true;
+        bool exactn_mismatch = false;
+        int exactn_accepted_prefix = 1;
+        int exactn_last_top = -1;
+        const double exactn_t0 = stats_enabled ? now_sec() : 0.0;
+
+        /* Evaluating draft[i] produces the target logits that verify
+         * draft[i+1].  sample_probs is private scratch, so s->logits remains
+         * the pre-cycle row until a full block is known to be committable. */
+        for (int i = 0; exactn_ok && i + 1 < draft_n; i++) {
+            exactn_ok = metal_graph_eval_token_raw_swa(
+                    &s->graph,
+                    &e->model,
+                    &e->weights,
+                    drafts[i],
+                    (uint32_t)(start + i),
+                    s->sample_probs);
+            if (!exactn_ok) break;
+            if (stats_enabled) s->dspark_stats.exactn_boundary_rows++;
+            exactn_last_top = sample_argmax(s->sample_probs, DS4_N_VOCAB);
+            if (exactn_last_top != drafts[i + 1]) {
+                exactn_mismatch = true;
+                break;
+            }
+            exactn_accepted_prefix = i + 2;
+        }
+
+        /* All inter-row tops matched.  Advance the last accepted token too,
+         * yielding both its exact persistent state and continuation logits. */
+        if (exactn_ok && exactn_mismatch) {
+            /* The boundary oracle stops immediately after the last accepted
+             * token, so unlike the union tape its live frontier and
+             * sample_probs already are the canonical partial commit. */
+            memcpy(s->logits,
+                   s->sample_probs,
+                   (size_t)DS4_N_VOCAB * sizeof(s->logits[0]));
+            for (int i = 0; i < exactn_accepted_prefix; i++) {
+                token_vec_push(&s->checkpoint, drafts[i]);
+                accepted[n_accept++] = drafts[i];
+            }
+            s->checkpoint_valid = true;
+            ds4_session_dspark_capture_note_checkpoint(s);
+            if (stats_enabled) {
+                s->dspark_stats.partial_accepts++;
+                s->dspark_stats.exactn_partial_accepts++;
+                s->dspark_stats.accepted_draft_tokens +=
+                    (uint64_t)exactn_accepted_prefix;
+                ds4_dspark_stats_note_len(
+                        s->dspark_stats.accepted_len_hist,
+                        (uint32_t)exactn_accepted_prefix);
+            }
+            ds4_session_dspark_stats_note_saved(
+                    s, (uint32_t)exactn_accepted_prefix);
+            if (stats_enabled) {
+                s->dspark_stats.verify_ms +=
+                    (now_sec() - exactn_t0) * 1000.0;
+            }
+            if (spec_log) {
+                fprintf(stderr,
+                        "ds4: DSpark Metal exactN boundary direct-partial "
+                        "drafted=%d committed=%d accepted_total=%d\n",
+                        draft_n,
+                        exactn_accepted_prefix,
+                        n_accept);
+            }
+            spec_frontier_free(&frontier);
+            DS4_DSPARK_STATS_FINISH();
+            return n_accept;
+        }
+
+        if (exactn_ok && !exactn_mismatch) {
+            exactn_ok = metal_graph_eval_token_raw_swa(
+                    &s->graph,
+                    &e->model,
+                    &e->weights,
+                    drafts[draft_n - 1],
+                    (uint32_t)(start + draft_n - 1),
+                    row_logits);
+            if (exactn_ok && stats_enabled) {
+                s->dspark_stats.exactn_boundary_rows++;
+            }
+        }
+        if (stats_enabled) {
+            s->dspark_stats.verify_ms +=
+                (now_sec() - exactn_t0) * 1000.0;
+        }
+
+        if (exactn_ok && !exactn_mismatch) {
+            memcpy(s->logits,
+                   row_logits,
+                   (size_t)DS4_N_VOCAB * sizeof(s->logits[0]));
+            for (int i = 0; i < draft_n; i++) {
+                token_vec_push(&s->checkpoint, drafts[i]);
+                accepted[n_accept++] = drafts[i];
+            }
+            s->checkpoint_valid = true;
+            ds4_session_dspark_capture_note_checkpoint(s);
+            if (stats_enabled) {
+                s->dspark_stats.full_accepts++;
+                s->dspark_stats.exactn_full_accepts++;
+                s->dspark_stats.accepted_draft_tokens +=
+                    (uint64_t)draft_n;
+                ds4_dspark_stats_note_len(
+                        s->dspark_stats.accepted_len_hist,
+                        (uint32_t)draft_n);
+            }
+            ds4_session_dspark_stats_note_saved(s, (uint32_t)draft_n);
+            if (spec_log) {
+                fprintf(stderr,
+                        "ds4: DSpark Metal exactN boundary oracle drafted=%d "
+                        "accepted_draft=%d accepted_total=%d\n",
+                        draft_n,
+                        draft_n,
+                        n_accept);
+            }
+            spec_frontier_free(&frontier);
+            DS4_DSPARK_STATS_FINISH();
+            return n_accept;
+        }
+
+        /* Only a backend error reaches this point: partial accepts commit
+         * directly above.  Restore every pre-cycle frontier and let the
+         * established legacy verifier/replay recover conservatively. */
+        if (stats_enabled) {
+            s->dspark_stats.exactn_fallbacks++;
+            if (exactn_mismatch) {
+                s->dspark_stats.exactn_partial_fallbacks++;
+            } else {
+                s->dspark_stats.exactn_error_fallbacks++;
+            }
+        }
+        s->checkpoint.len = start;
+        ds4_session_dspark_capture_invalidate(s);
+        if (!have_frontier || !spec_frontier_restore(&frontier, s)) {
+            snprintf(err, errlen,
+                     "DSpark Metal exactN boundary rollback failed");
+            s->checkpoint_valid = false;
+            if (stats_enabled) {
+                s->dspark_stats.verifier_errors++;
+                ds4_dspark_stats_note_len(
+                        s->dspark_stats.accepted_len_hist, 0);
+            }
+            spec_frontier_free(&frontier);
+            DS4_DSPARK_STATS_FINISH();
+            return -1;
+        }
+        if (spec_log) {
+            if (exactn_mismatch) {
+                fprintf(stderr,
+                        "ds4: DSpark Metal exactN boundary partial "
+                        "accepted_prefix=%d verified_next=%d; falling back "
+                        "to legacy verify/replay\n",
+                        exactn_accepted_prefix,
+                        exactn_last_top);
+            } else {
+                fprintf(stderr,
+                        "ds4: DSpark Metal exactN boundary decode failed; "
+                        "falling back to legacy verify/replay\n");
+            }
+        }
+    }
+
+    const bool exact2 =
+        !cuda_exactn && !metal_exactn_union_verified_partial &&
+        !exactn && ok &&
+        ds4_session_dspark_exact2_requested(
+                         s, (uint32_t)draft_n);
+    if (exact2) {
+        if (stats_enabled) s->dspark_stats.exact2_attempts++;
+        /* sample_probs is a session-owned vocab-sized scratch row.  Keeping
+         * row0 there leaves the pre-cycle s->logits intact if exact-2 fails
+         * and the legacy verifier has to take over. */
+        int exact_top0 = -1;
+        const double exact_t0 = stats_enabled ? now_sec() : 0.0;
+        const bool metal_exact2 =
+            ds4_session_metal_dspark_exact2_enabled(s);
+        bool exact_ok = true;
+        if (metal_exact2) {
+            const bool static_map_cache =
+                metal_graph_stream_decode_static_map_enabled() &&
+                metal_graph_stream_decode_static_map_state_cache_enabled();
+            if (!static_map_cache ||
+                !s->graph.streaming_static_decode_map_current) {
+                exact_ok = metal_graph_stream_map_decode_static_all(
+                        &e->model, &e->weights);
+                if (exact_ok) {
+                    s->graph.streaming_static_decode_map_current =
+                        static_map_cache;
+                }
+            }
+        }
+        if (exact_ok) {
+            exact_ok = metal_graph_verify_decode2_exact_impl(
+                    &s->graph,
+                    &e->model,
+                    &e->weights,
+                    drafts[0],
+                    drafts[1],
+                    (uint32_t)start,
+                    false,
+                    drafts[1],
+                    &exact_top0,
+                    NULL,
+                    s->sample_probs,
+                    row_logits);
+        }
+        if (stats_enabled) {
+            s->dspark_stats.verify_ms +=
+                (now_sec() - exact_t0) * 1000.0;
+        }
+
+        const int exact_commit =
+            exact_ok && exact_top0 == drafts[1] ? 2 : 1;
+        bool exact_partial_replayed = false;
+        if (exact_ok && exact_commit == 1) {
+            /* Prefix-state capture adds dozens of D2D operations to every
+             * full accept. Exact-2 therefore skips it and pays a single
+             * headless token0 replay only on a partial accept. The already-
+             * computed exact row0 logits remain valid. */
+            s->checkpoint.len = start;
+            ds4_session_dspark_capture_invalidate(s);
+            exact_ok = spec_frontier_restore(&frontier, s);
+            const double replay_t0 = stats_enabled ? now_sec() : 0.0;
+            if (exact_ok) {
+                exact_ok = metal_graph_eval_token_raw_swa(&s->graph,
+                                                           &e->model,
+                                                           &e->weights,
+                                                           drafts[0],
+                                                           (uint32_t)start,
+                                                           NULL);
+                exact_partial_replayed = exact_ok;
+            }
+            if (stats_enabled) {
+                s->dspark_stats.replay_ms +=
+                    (now_sec() - replay_t0) * 1000.0;
+            }
+        }
+        if (exact_ok) {
+            const float *continuation_logits =
+                exact_commit == 2 ? row_logits : s->sample_probs;
+            memcpy(s->logits,
+                   continuation_logits,
+                   (size_t)DS4_N_VOCAB * sizeof(s->logits[0]));
+
+            /* exact-2 uses private row views and therefore does not refresh
+             * the DSpark target-hidden capture.  Never relabel the old hidden
+             * row as current: the ordinary first-token decode of the next
+             * cycle will capture the new state before running the proposer. */
+            if (!exact_partial_replayed) {
+                ds4_session_dspark_capture_invalidate(s);
+            }
+            for (int i = 0; i < exact_commit; i++) {
+                token_vec_push(&s->checkpoint, drafts[i]);
+                accepted[n_accept++] = drafts[i];
+            }
+            s->checkpoint_valid = true;
+            ds4_session_dspark_capture_note_checkpoint(s);
+            if (stats_enabled) {
+                if (exact_commit == draft_n) {
+                    s->dspark_stats.full_accepts++;
+                    s->dspark_stats.exact2_full_accepts++;
+                } else {
+                    s->dspark_stats.partial_accepts++;
+                    s->dspark_stats.exact2_partial_accepts++;
+                }
+                s->dspark_stats.accepted_draft_tokens +=
+                    (uint64_t)exact_commit;
+                ds4_dspark_stats_note_len(
+                        s->dspark_stats.accepted_len_hist,
+                        (uint32_t)exact_commit);
+            }
+            ds4_session_dspark_stats_note_saved(
+                    s, (uint32_t)exact_commit);
+            if (spec_log) {
+                fprintf(stderr,
+                        "ds4: DSpark %s exact2 drafted=2 verified_next=%d "
+                        "accepted_draft=%d accepted_total=%d\n",
+                        ds4_backend_name(e->backend),
+                        exact_top0,
+                        exact_commit,
+                        n_accept);
+            }
+            spec_frontier_free(&frontier);
+            DS4_DSPARK_STATS_FINISH();
+            return n_accept;
+        }
+
+        /* A backend error may happen after exact-2 has touched persistent KV
+         * state.  Restore the pre-verify snapshot and retain the established
+         * batch-verify/replay path as a correctness fallback. */
+        if (stats_enabled) s->dspark_stats.exact2_fallbacks++;
+        s->checkpoint.len = start;
+        ds4_session_dspark_capture_invalidate(s);
+        if (!have_frontier || !spec_frontier_restore(&frontier, s)) {
+            snprintf(err, errlen, "DSpark %s exact2 rollback failed",
+                     ds4_backend_name(e->backend));
+            s->checkpoint_valid = false;
+            if (stats_enabled) {
+                s->dspark_stats.verifier_errors++;
+                ds4_dspark_stats_note_len(
+                        s->dspark_stats.accepted_len_hist, 0);
+            }
+            spec_frontier_free(&frontier);
+            DS4_DSPARK_STATS_FINISH();
+            return -1;
+        }
+        if (spec_log) {
+            fprintf(stderr,
+                    "ds4: DSpark %s exact2 unavailable; falling back to "
+                    "batch verify plus exact replay\n",
+                    ds4_backend_name(e->backend));
+        }
+    }
+
+    /* Replay below still evaluates a skipped one-token draft exactly and
+     * advances all KV and compressor state. */
     if (ok && ds4_session_tp_leader(s)) {
         /* Announce the block before mutating anything: the worker runs its
          * half of the verify and then waits for our commit decision. */
@@ -70225,26 +74772,57 @@ static int ds4_session_eval_dspark_speculative_argmax(
         }
         tp_verify_sent = true;
     }
-    if (ok) {
+    if (ok && !skip_single_verify && !cuda_exactn_verified_partial &&
+        !metal_exactn_union_verified_partial) {
         for (int i = 0; i < draft_n; i++) token_vec_push(&s->checkpoint, drafts[i]);
         verifier_may_have_mutated = true;
         ds4_verify_suffix_timing verify_timing;
         const double verify_t0 = stats_enabled ? now_sec() : 0.0;
-        ok = metal_graph_verify_suffix_tops(&s->graph,
-                                            &e->model,
-                                            &e->weights,
-                                            &s->checkpoint,
-                                            (uint32_t)start,
-                                            (uint32_t)draft_n,
-                                            draft_n > 1 &&
-                                                draft_n <=
-                                                    (int)DS4_SPEC_PREFIX_SLOTS + 1,
-                                            true,
-                                            row_tops,
-                                            NULL,
-                                            stats_enabled ? &verify_timing : NULL);
+        const bool saved_force_sequential =
+            s->graph.spec_force_sequential_compressor;
+        if (cuda_rollback_replay) {
+            s->graph.spec_force_sequential_compressor = true;
+        }
+        if (metal_acceptance_only_verify) {
+            if (stats_enabled) {
+                s->dspark_stats.metal_acceptance_only_attempts++;
+                s->dspark_stats.metal_acceptance_only_rows_saved++;
+            }
+            ok = metal_graph_verify_suffix_acceptance_tops(
+                    &s->graph,
+                    &e->model,
+                    &e->weights,
+                    &s->checkpoint,
+                    (uint32_t)start,
+                    (uint32_t)draft_n,
+                    row_tops,
+                    stats_enabled ? &verify_timing : NULL);
+        } else {
+            ok = metal_graph_verify_suffix_tops(
+                    &s->graph,
+                    &e->model,
+                    &e->weights,
+                    &s->checkpoint,
+                    (uint32_t)start,
+                    (uint32_t)draft_n,
+                    !cuda_rollback_replay &&
+                        draft_n > 1 &&
+                        draft_n <= (int)DS4_SPEC_PREFIX_SLOTS + 1,
+                    !cuda_rollback_replay,
+                    row_tops,
+                    NULL,
+                    stats_enabled ? &verify_timing : NULL);
+        }
+        s->graph.spec_force_sequential_compressor =
+            saved_force_sequential;
         if (stats_enabled) {
-            s->dspark_stats.verify_ms += (now_sec() - verify_t0) * 1000.0;
+            const double legacy_verify_ms =
+                (now_sec() - verify_t0) * 1000.0;
+            s->dspark_stats.verify_ms += legacy_verify_ms;
+            if (cuda_exactn) {
+                s->dspark_stats.cuda_exactn_legacy_verify_ms +=
+                    legacy_verify_ms;
+            }
             s->dspark_stats.verify_upload_ms += verify_timing.upload_ms;
             s->dspark_stats.verify_layer_ms += verify_timing.layer_ms;
             s->dspark_stats.verify_head_ms += verify_timing.head_ms;
@@ -70264,8 +74842,8 @@ static int ds4_session_eval_dspark_speculative_argmax(
         }
     }
 
-    int commit_drafts = 0;
-    if (ok) {
+    int commit_drafts = preverified_commit_drafts;
+    if (ok && commit_drafts == 0) {
         commit_drafts = 1;
         for (int i = 1; i < draft_n; i++) {
             if (row_tops[i - 1] != drafts[i]) break;
@@ -70318,9 +74896,8 @@ static int ds4_session_eval_dspark_speculative_argmax(
             ds4_dspark_stats_note_len(s->dspark_stats.accepted_len_hist,
                                       (uint32_t)emitted_drafts);
         }
-        ds4_session_dspark_scheduler_note(
-                s, (uint32_t)emitted_drafts, false,
-                DS4_DSPARK_SCHED_EXTRA_MS());
+        ds4_session_dspark_stats_note_saved(
+                s, (uint32_t)emitted_drafts);
         if (spec_log) {
             fprintf(stderr,
                     "ds4: DSpark spec direct-full drafted=%d accepted=%d\n",
@@ -70391,9 +74968,8 @@ static int ds4_session_eval_dspark_speculative_argmax(
                         s->dspark_stats.accepted_len_hist,
                         (uint32_t)emitted_drafts);
             }
-            ds4_session_dspark_scheduler_note(
-                    s, (uint32_t)emitted_drafts, false,
-                    DS4_DSPARK_SCHED_EXTRA_MS());
+            ds4_session_dspark_stats_note_saved(
+                    s, (uint32_t)emitted_drafts);
             if (spec_log) {
                 fprintf(stderr,
                         "ds4: DSpark spec direct-partial drafted=%d committed=%d accepted=%d\n",
@@ -70455,9 +75031,8 @@ static int ds4_session_eval_dspark_speculative_argmax(
                         s->dspark_stats.accepted_len_hist,
                         (uint32_t)emitted_drafts);
             }
-            ds4_session_dspark_scheduler_note(
-                    s, (uint32_t)emitted_drafts, false,
-                    DS4_DSPARK_SCHED_EXTRA_MS());
+            ds4_session_dspark_stats_note_saved(
+                    s, (uint32_t)emitted_drafts);
             if (spec_log) {
                 fprintf(stderr,
                         "ds4: DSpark spec prefix-extended drafted=%d committed=%d accepted=%d\n",
@@ -70516,8 +75091,8 @@ static int ds4_session_eval_dspark_speculative_argmax(
         return n_accept;
     }
 
-    /* Precompute the exact replay count (cap + eos cuts) so the worker can
-     * run the same gated replay evals in lockstep. */
+    /* Precompute the exact replay count (capacity and EOS cuts) so a TP worker
+     * can mirror the same fallback decode sequence in lockstep. */
     int replay_budget = commit_drafts;
     if (replay_budget > accepted_cap - n_accept)
         replay_budget = accepted_cap - n_accept;
@@ -70525,6 +75100,8 @@ static int ds4_session_eval_dspark_speculative_argmax(
     for (int i = 0; i < replay_budget; i++) {
         if (drafts[i] == eos_token) { replay_budget = i + 1; break; }
     }
+
+    /* Tell the worker how many exact replay evals to mirror under TP. */
     if (tp_verify_sent &&
         !ds4_tp_send_verify_commit(e->tp.ctx,
                                    DS4_TP_VERIFY_ROLLBACK_REPLAY,
@@ -70540,18 +75117,38 @@ static int ds4_session_eval_dspark_speculative_argmax(
         s->dspark_stats.replay_fallbacks++;
     }
     for (int i = 0; i < replay_budget; i++) {
+        /* Only the final replayed token supplies the continuation logits.
+         * Eligible single-device backends can skip the output head and
+         * readback for accepted prefix tokens without changing decode/KV. */
+        const bool headless_prefix =
+            i + 1 < replay_budget &&
+            (cuda_rollback_replay || metal_headless_replay);
+        float *replay_logits = headless_prefix ? NULL : row_logits;
+        if (stats_enabled && metal_headless_replay && headless_prefix) {
+            s->dspark_stats.metal_replay_headless_tokens++;
+        }
         ok = metal_graph_eval_token_raw_swa(&s->graph,
                                             &e->model,
                                             &e->weights,
                                             drafts[i],
                                             (uint32_t)s->checkpoint.len,
-                                            row_logits);
+                                            replay_logits);
         if (!ok) {
             snprintf(err, errlen, "%s decode failed", ds4_backend_name(e->backend));
             s->checkpoint_valid = false;
             if (stats_enabled) {
                 s->dspark_stats.verifier_errors++;
-                s->dspark_stats.replay_ms += (now_sec() - replay_t0) * 1000.0;
+                const double failed_replay_ms =
+                    (now_sec() - replay_t0) * 1000.0;
+                s->dspark_stats.replay_ms += failed_replay_ms;
+                if (cuda_exactn_verified_partial) {
+                    s->dspark_stats.cuda_exactn_partial_replay_ms +=
+                        failed_replay_ms;
+                }
+                if (metal_exactn_union_verified_partial) {
+                    s->dspark_stats.exactn_union_partial_replay_ms +=
+                        failed_replay_ms;
+                }
                 ds4_dspark_stats_note_len(s->dspark_stats.accepted_len_hist, 0);
             }
             spec_frontier_free(&frontier);
@@ -70564,7 +75161,14 @@ static int ds4_session_eval_dspark_speculative_argmax(
         if (drafts[i] == eos_token) break;
     }
     if (stats_enabled) {
-        s->dspark_stats.replay_ms += (now_sec() - replay_t0) * 1000.0;
+        const double replay_ms = (now_sec() - replay_t0) * 1000.0;
+        s->dspark_stats.replay_ms += replay_ms;
+        if (cuda_exactn_verified_partial) {
+            s->dspark_stats.cuda_exactn_partial_replay_ms += replay_ms;
+        }
+        if (metal_exactn_union_verified_partial) {
+            s->dspark_stats.exactn_union_partial_replay_ms += replay_ms;
+        }
     }
     /* Vocab-split head: the last replay eval produced only our logits half;
      * merge the worker's before installing them as the session logits. */
@@ -70592,11 +75196,8 @@ static int ds4_session_eval_dspark_speculative_argmax(
     } else if (stats_enabled) {
         ds4_dspark_stats_note_len(s->dspark_stats.accepted_len_hist, 0);
     }
-    ds4_session_dspark_scheduler_note(
-            s,
-            (uint32_t)replayed_drafts,
-            false,
-            DS4_DSPARK_SCHED_EXTRA_MS());
+    ds4_session_dspark_stats_note_saved(
+            s, (uint32_t)replayed_drafts);
     if (spec_log) {
         fprintf(stderr,
                 "ds4: DSpark spec partial drafted=%d verified=%d accepted=%d\n",
@@ -70606,7 +75207,6 @@ static int ds4_session_eval_dspark_speculative_argmax(
     }
     spec_frontier_free(&frontier);
     DS4_DSPARK_STATS_FINISH();
-#undef DS4_DSPARK_SCHED_EXTRA_MS
 #undef DS4_DSPARK_STATS_FINISH
     return n_accept;
 }
@@ -70643,23 +75243,15 @@ static int ds4_session_eval_dspark_speculative_stochastic(
         uint64_t *rng,
         int *accepted,
         int accepted_cap,
-        char *err,
-        size_t errlen) {
+    char *err,
+    size_t errlen) {
     const bool stats_enabled = s && ds4_dspark_stats_enabled();
-    const bool scheduler_enabled = s && ds4_dspark_scheduler_enabled(s);
-    const double stats_t0 =
-        (stats_enabled ||
-         (scheduler_enabled && ds4_dspark_scheduler_timing_enabled()))
-            ? now_sec() : 0.0;
+    const double stats_t0 = stats_enabled ? now_sec() : 0.0;
 #define DS4_DSPARK_STOCH_FINISH() do {                                      \
         if (stats_enabled) {                                                \
             s->dspark_stats.total_ms += (now_sec() - stats_t0) * 1000.0;    \
         }                                                                   \
     } while (0)
-#define DS4_DSPARK_STOCH_EXTRA_MS()                                         \
-    ((scheduler_enabled && stats_t0 != 0.0) ?                               \
-     s->dspark_last_propose_ms + (now_sec() - stats_t0) * 1000.0 : 0.0)
-
     if (stats_enabled) {
         s->dspark_stats.cycles++;
         if (n_accept > 0) s->dspark_stats.first_tokens++;
@@ -70670,10 +75262,6 @@ static int ds4_session_eval_dspark_speculative_stochastic(
             s->dspark_stats.no_draft++;
             ds4_dspark_stats_note_len(s->dspark_stats.accepted_len_hist, 0);
         }
-        if (s) {
-            ds4_session_dspark_scheduler_note(
-                s, 0, true, DS4_DSPARK_STOCH_EXTRA_MS());
-        }
         DS4_DSPARK_STOCH_FINISH();
         return n_accept;
     }
@@ -70746,8 +75334,6 @@ static int ds4_session_eval_dspark_speculative_stochastic(
             s->dspark_stats.first_misses++;
             ds4_dspark_stats_note_len(s->dspark_stats.accepted_len_hist, 0);
         }
-        ds4_session_dspark_scheduler_note(
-            s, 0, false, DS4_DSPARK_STOCH_EXTRA_MS());
         DS4_DSPARK_STOCH_FINISH();
         return n_accept;
     }
@@ -70843,9 +75429,7 @@ static int ds4_session_eval_dspark_speculative_stochastic(
                 ds4_dspark_stats_note_len(s->dspark_stats.accepted_len_hist,
                                           (uint32_t)emitted);
             }
-            ds4_session_dspark_scheduler_note(
-                s, (uint32_t)emitted, false,
-                DS4_DSPARK_STOCH_EXTRA_MS());
+            ds4_session_dspark_stats_note_saved(s, (uint32_t)emitted);
             spec_frontier_free(&frontier);
             DS4_DSPARK_STOCH_FINISH();
             return n_accept;
@@ -70973,16 +75557,72 @@ static int ds4_session_eval_dspark_speculative_stochastic(
         ds4_dspark_stats_note_len(s->dspark_stats.accepted_len_hist,
                                   (uint32_t)emitted);
     }
-    ds4_session_dspark_scheduler_note(
-        s, (uint32_t)emitted, false, DS4_DSPARK_STOCH_EXTRA_MS());
+    ds4_session_dspark_stats_note_saved(s, (uint32_t)emitted);
     spec_frontier_free(&frontier);
     DS4_DSPARK_STOCH_FINISH();
-#undef DS4_DSPARK_STOCH_EXTRA_MS
 #undef DS4_DSPARK_STOCH_FINISH
     return n_accept;
 }
 #endif
 
+#if defined(DS4_TEST_HOOKS) && !defined(DS4_NO_GPU)
+/* Test-only injection point for deterministic exact-N full/partial/EOS
+ * fixtures.  It calls the production speculative cycle after installing the
+ * requested draft block; no verifier state transition is duplicated here. */
+int ds4_test_session_eval_exact_drafts(
+        ds4_session *s,
+        const int   *drafts,
+        int          draft_n,
+        int          eos_token,
+        int         *accepted,
+        int          accepted_cap,
+        char        *err,
+        size_t       errlen) {
+    if (!s || !drafts || !accepted || draft_n < 1 || draft_n > 5 ||
+        draft_n > (int)DS4_DSPARK_MAX_BLOCK_SIZE ||
+        accepted_cap < draft_n) {
+        if (err && errlen) {
+            snprintf(err, errlen, "invalid exact-N test draft block");
+        }
+        return -1;
+    }
+    for (int i = 0; i < draft_n; i++) {
+        s->dspark_draft_tokens[i] = drafts[i];
+    }
+    s->dspark_draft_len = (uint32_t)draft_n;
+    s->dspark_draft_valid = true;
+    return ds4_session_eval_dspark_speculative_argmax(
+            s,
+            0,
+            draft_n,
+            eos_token,
+            false,
+            DS4_THINK_HIGH,
+            accepted,
+            accepted_cap,
+            err,
+            errlen);
+}
+
+/* Keep the model-backed oracle independent of stderr formatting. */
+int ds4_test_session_exactn_union_stats(
+        const ds4_session *s,
+        uint64_t           out[10]) {
+    if (!s || !out) return -1;
+    out[0] = s->dspark_stats.exactn_union_attempts;
+    out[1] = s->dspark_stats.exactn_union_full_accepts;
+    out[2] = s->dspark_stats.exactn_union_fallbacks;
+    out[3] = s->dspark_stats.exactn_union_partial_fallbacks;
+    out[4] = s->dspark_stats.exactn_union_error_fallbacks;
+    out[5] = s->dspark_stats.exactn_union_partial_replays;
+    out[6] = s->dspark_stats.exactn_union_legacy_verify_skips;
+    out[7] = s->dspark_stats.metal_exactn_batch_head_attempts;
+    out[8] = s->dspark_stats.metal_exactn_batch_head_uses;
+    out[9] = s->dspark_stats.metal_exactn_batch_head_fallbacks;
+    return 0;
+}
+#endif
+
 /* TP worker side of a mirrored speculative-verify block. Runs its half of the
  * same batch verify as the leader, including per-layer combine gates, purely
  * for KV, compressor, and indexer side effects; then it obeys the commit
@@ -73984,7 +78624,7 @@ static int ds4_session_eval_speculative_argmax_impl(
             accepted[0] = first_token;
             return 1;
         }
-        if (s->engine->glm_mtp && DS4_N_NEXTN_PREDICT != 0 &&
+        if (!ignore_eos && s->engine->glm_mtp && DS4_N_NEXTN_PREDICT != 0 &&
             s->glm_graph_ready) {
             if (ds4_session_tp_leader(s)) {
                 ds4_engine *ge = s->engine;
@@ -74036,34 +78676,14 @@ static int ds4_session_eval_speculative_argmax_impl(
     const bool strict_dspark =
         e->support_kind == DS4_SUPPORT_DSPARK &&
         (e->quality || e->dspark_strict);
-    const bool dspark_scheduler_bypass =
-        e->support_kind == DS4_SUPPORT_DSPARK &&
-        s->dspark_sched_bypass;
     bool can_prepare_support_draft =
         !strict_dspark &&
-        !dspark_scheduler_bypass &&
         first_token != eos_token && max_tokens > 1 && accepted_cap > 1;
     if (can_prepare_support_draft &&
         e->tp.active &&
         e->support_kind == DS4_SUPPORT_MTP_LEGACY) {
         can_prepare_support_draft = false;
     }
-    bool dspark_tail_skip = false;
-    if (can_prepare_support_draft && e->support_kind == DS4_SUPPORT_DSPARK &&
-        ds4_dspark_scheduler_enabled(s)) {
-        const uint32_t tail_min = ds4_dspark_scheduler_tail_min_tokens();
-        if (tail_min != 0 && (uint32_t)max_tokens < tail_min) {
-            can_prepare_support_draft = false;
-            dspark_tail_skip = true;
-            if (ds4_dspark_stats_enabled()) s->dspark_stats.tail_skips++;
-            if (getenv("DS4_DSPARK_SPEC_LOG") != NULL) {
-                fprintf(stderr,
-                        "ds4: DSpark scheduler tail skip max=%d min=%u\n",
-                        max_tokens,
-                        tail_min);
-            }
-        }
-    }
     const bool seed_batch_dspark =
         can_prepare_support_draft &&
         e->support_kind == DS4_SUPPORT_DSPARK &&
@@ -74100,8 +78720,6 @@ static int ds4_session_eval_speculative_argmax_impl(
     accepted[n_accept++] = first_token;
     if (first_token == eos_token || max_tokens == 1 || n_accept >= accepted_cap) return n_accept;
     if (strict_dspark) return n_accept;
-    if (dspark_scheduler_bypass) return n_accept;
-    if (dspark_tail_skip) return n_accept;
 
     if (e->support_kind == DS4_SUPPORT_DSPARK) {
         return ds4_session_eval_dspark_speculative_argmax(s,
@@ -74116,7 +78734,7 @@ static int ds4_session_eval_speculative_argmax_impl(
                                                           errlen);
     }
 
-    if (metal_graph_cuda_splitkv_spec_requested() &&
+    if (!ignore_eos && metal_graph_cuda_splitkv_spec_requested() &&
         (!e->mtp_ready || e->mtp_draft_tokens <= 1)) {
         int extra = ds4_session_eval_splitkv_spec_after_first(
                 s,
@@ -74831,18 +79449,9 @@ int ds4_session_eval_speculative(ds4_session *s, int first_token,
     const bool stochastic_dspark =
         e && e->support_kind == DS4_SUPPORT_DSPARK && e->dspark &&
         !e->quality && !e->dspark_strict && e->dspark_exact_sampling;
-    bool can_prepare = stochastic_dspark && !s->dspark_sched_bypass &&
+    bool can_prepare = stochastic_dspark &&
         first_token != eos_token &&
         max_tokens > 1 && accepted_cap > 1;
-    bool tail_skip = false;
-    if (can_prepare && ds4_dspark_scheduler_enabled(s)) {
-        const uint32_t tail_min = ds4_dspark_scheduler_tail_min_tokens();
-        if (tail_min != 0 && (uint32_t)max_tokens < tail_min) {
-            can_prepare = false;
-            tail_skip = true;
-            if (ds4_dspark_stats_enabled()) s->dspark_stats.tail_skips++;
-        }
-    }
 
     s->dspark_sample_temperature = temperature;
     s->dspark_sample_rng = can_prepare ? rng : NULL;
@@ -74856,7 +79465,7 @@ int ds4_session_eval_speculative(ds4_session *s, int first_token,
 
     int n_accept = 0;
     accepted[n_accept++] = first_token;
-    if (!can_prepare || tail_skip || first_token == eos_token ||
+    if (!can_prepare || first_token == eos_token ||
         max_tokens == 1 || n_accept >= accepted_cap) {
         s->dspark_sample_temperature = 0.0f;
         return n_accept;
diff --git a/ds4.h b/ds4.h
index e6dae1b9f0..1bfc99d796 100644
--- a/ds4.h
+++ b/ds4.h
@@ -389,6 +389,12 @@ int ds4_engine_tp_bind(ds4_engine *e, struct ds4_tp *tp, char *err, size_t errle
 
 int ds4_session_create(ds4_session **out, ds4_engine *e, int ctx_size);
 void ds4_session_free(ds4_session *s);
+/* Prepares one-shot resources for the next prompt without changing session
+ * state. Timed callers can invoke this before ds4_session_sync(). */
+int ds4_session_prepare_sync(ds4_session *s,
+                             const ds4_tokens *prompt,
+                             char *err,
+                             size_t errlen);
 int ds4_session_power(ds4_session *s);
 int ds4_session_set_power(ds4_session *s, int power_percent);
 float ds4_session_directional_steering_ffn(ds4_session *s);
@@ -477,7 +483,12 @@ int ds4_test_speculative_delta_sample(const float *target_logits,
                                       float *target_probs);
 int ds4_test_argmax_excluding_logits(const float *logits, uint32_t n_vocab,
                                      int excluded_id);
+int ds4_test_indexer_q_type_supported(uint32_t type);
 uint64_t ds4_test_mixed_native_count(void);
+#if defined(__APPLE__)
+int ds4_test_q4_stream_overlap_policy(
+        int count, bool resident, bool ssd_streaming, bool quality);
+#endif
 #endif
 int ds4_session_top_logprobs(ds4_session *s, ds4_token_score *out, int k);
 int ds4_session_token_logprob(ds4_session *s, int token, ds4_token_score *out);
diff --git a/ds4_bench.c b/ds4_bench.c
index 44eb49ddc6..8be25b7ef7 100644
--- a/ds4_bench.c
+++ b/ds4_bench.c
@@ -814,6 +814,17 @@ int main(int argc, char **argv) {
             .cap = frontier,
         };
 
+        /* Keep one-shot resident-weight preparation outside the prefill TPS
+         * window.  The preflight uses this frontier's real suffix/chunk width,
+         * so decode-only and sub-threshold runs allocate no sidecars. */
+        if (ds4_session_prepare_sync(
+                session, &prefix, err, sizeof(err)) != 0) {
+            fprintf(stderr,
+                    "ds4-bench: prefill preparation to %d failed: %s\n",
+                    frontier, err);
+            rc = 1;
+            break;
+        }
         const double prefill_t0 = bench_now_sec();
 #if defined(DS4_BENCH_HAVE_CUDA_PROFILER)
         const bool cuda_profile_prefill =
diff --git a/ds4_cli.c b/ds4_cli.c
index 9d2c6e3ac8..ff9b78d8ed 100644
--- a/ds4_cli.c
+++ b/ds4_cli.c
@@ -551,11 +551,17 @@ static int run_sampled_generation(ds4_engine *engine, const cli_config *cfg, con
         ds4_session_free(session);
         return 1;
     }
+    char err[160];
+    if (ds4_session_prepare_sync(
+            session, prompt, err, sizeof(err)) != 0) {
+        fprintf(stderr, "ds4: prompt preparation failed: %s\n", err);
+        ds4_session_free(session);
+        return 1;
+    }
     /* Pay the one-time first-submission GPU cost before the prefill timer
      * starts (matches the TP worker's startup warmup). */
     ds4_session_gpu_warmup(session);
 
-    char err[160];
     ds4_think_mode think_mode = cli_effective_think_mode(&cfg->gen);
     token_printer printer = {
         .engine = engine,
@@ -1567,6 +1573,13 @@ static int run_chat_turn(ds4_engine *engine, cli_config *cfg, repl_chat *chat,
         .input_tokens = suffix,
         .use_color = ds4_log_is_tty(stderr),
     };
+    if (ds4_session_prepare_sync(
+            chat->session, &chat->transcript, err, sizeof(err)) != 0) {
+        chat->transcript.len = rollback_len;
+        repl_chat_trim_images(chat, rollback_images);
+        fprintf(stderr, "ds4: prompt preparation failed: %s\n", err);
+        return 1;
+    }
     const double t_prefill0 = cli_now_sec();
     ds4_session_set_progress(chat->session, cli_prefill_progress_cb, &progress);
     ds4_session_set_display_progress(chat->session,
diff --git a/ds4_cuda.cu b/ds4_cuda.cu
index fd36d7014e..28a0161c9e 100644
--- a/ds4_cuda.cu
+++ b/ds4_cuda.cu
@@ -6,6 +6,7 @@
 #include <cub/block/block_radix_sort.cuh>
 
 #include <stdint.h>
+#include <stddef.h>
 #include <errno.h>
 #include <limits.h>
 #include <float.h>
@@ -14,12 +15,19 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <strings.h>
+#include <sys/mman.h>
 #include <sys/stat.h>
 #include <time.h>
 #include <unistd.h>
 #include <unordered_map>
 #include <vector>
 #include <algorithm>
+#include <atomic>
+#include <mutex>
+#include <condition_variable>
+#include <thread>
+#include <utility>
 
 #include "cuda/mmq/ds4_mmq.h"
 #include "cuda/mmq/ds4_repack.h"
@@ -30,7 +38,19 @@
 #endif
 
 #define CUDA_QK_K 256
-#define DS4_CUDA_UNUSED __attribute__((unused))
+
+/* Environment switches used by opt-in CUDA experiments are value-aware:
+ * spelling a switch as 0/off/no/false must behave like leaving it unset.
+ * Keep this helper above the MMQ policy code so early dispatch gates and
+ * later streaming policies share the exact same interpretation. */
+static int cuda_env_value_enabled(const char *value) {
+    if (!value || !value[0]) return 0;
+    if (strcmp(value, "0") == 0 || strcasecmp(value, "false") == 0 ||
+        strcasecmp(value, "no") == 0 || strcasecmp(value, "off") == 0) {
+        return 0;
+    }
+    return 1;
+}
 
 enum {
     /* attention_decode_mixed_kernel stores raw-window scores plus visible
@@ -72,10 +92,26 @@ typedef struct {
     int16_t bsums[CUDA_QK_K / 16];
 } cuda_block_q8_K;
 
+/* Canonical activation layout consumed by the MMVQ entries in
+ * cuda/mmq/.  Keep this private copy byte-for-byte aligned with
+ * block_q8_1 in ggml-common.h: half2(d, sum), then 32 signed codes. */
+typedef struct {
+    __half2 ds;
+    int8_t qs[32];
+} cuda_block_q8_1;
+static_assert(sizeof(cuda_block_q8_1) == 36u,
+              "canonical Q8_1 block layout drift");
+static_assert(offsetof(cuda_block_q8_1, qs) == 4u,
+              "canonical Q8_1 code offset drift");
+
 typedef struct {
     uint16_t d;
     uint16_t qs[CUDA_QK_K / 8];
 } cuda_block_iq2_xxs;
+static_assert(sizeof(cuda_block_iq2_xxs) == 66u,
+              "canonical IQ2_XXS block layout drift");
+static_assert(sizeof(cuda_block_q2_K) == 84u,
+              "canonical Q2_K block layout drift");
 
 #include "ds4_gpu_mgpu.h"
 #include "ds4_iq2_tables_cuda.inc"
@@ -126,17 +162,26 @@ static int g_cuda_exact_score_split_vec4_plain;
 static int g_cuda_exact_score_split_dim2;
 static int g_cuda_exact_score_split_fuse_inv_rope;
 static int g_cuda_moe_decode_graph;
+static int g_cuda_direct_q2_prefill;
+static int g_cuda_f16_pair_compressor_store;
+static int g_cuda_is_gb10[DS4_MAX_GPUS];
 static int g_current_logical_tier = -1;
 static int g_ssd_streaming_mode;
 
 typedef struct {
     int valid;
+    int top6_unique;
+    int storage_kind;
     int logical_tier;
     const void *model_map;
     uint32_t layer;
     uint32_t n_total_expert;
     uint32_t slot_count;
     uint32_t compact_count;
+    uint32_t slot_base;
+    uint32_t weight_domain;
+    uint64_t generation;
+    uint64_t upload_event_value;
     uint64_t gate_offset;
     uint64_t up_offset;
     uint64_t down_offset;
@@ -155,29 +200,76 @@ typedef struct {
 
 static cuda_stream_selected_cache g_stream_selected_cache;
 
+enum {
+    CUDA_STREAM_SELECTED_STORAGE_NONE = 0,
+    CUDA_STREAM_SELECTED_STORAGE_TRANSIENT = 1,
+    CUDA_STREAM_SELECTED_STORAGE_PERSISTENT = 2,
+};
+
+/* Sole owner of allocations used by the transient selected-expert binding.
+ * g_stream_selected_cache only publishes non-owning views into this storage. */
+typedef struct {
+    char *gate;
+    char *up;
+    char *down;
+    int32_t *remap;
+    uint64_t gate_capacity;
+    uint64_t up_capacity;
+    uint64_t down_capacity;
+    uint64_t remap_capacity;
+    int owner_device;
+    int poisoned;
+} cuda_stream_selected_transient_storage;
+
+static cuda_stream_selected_transient_storage
+    g_stream_selected_transient_storage = {
+        NULL, NULL, NULL, NULL, 0, 0, 0, 0, -1, 0,
+    };
+static int cuda_stream_selected_upload_drain_checked(void);
+static int cuda_stream_selected_consume_drain(void);
+static void cuda_stream_selected_consume_release(void);
+static void cuda_stream_selected_cache_release(void);
+static int cuda_stream_expert_storage_release(int reset_class);
+
+typedef struct {
+    void *base;
+    char *gate;
+    char *up;
+    char *down;
+    int32_t *remap;
+    uint64_t bytes;
+    uint64_t up_offset;
+    uint64_t down_offset;
+    uint64_t gate_expert_bytes;
+    uint64_t down_expert_bytes;
+    uint64_t configured_expert_bytes;
+    uint64_t remap_capacity;
+    uint32_t capacity;
+    uint32_t valid_count;
+    uint32_t configured_budget;
+    int owner_device;
+    int poisoned;
+} cuda_stream_expert_persistent_arena;
+
+static cuda_stream_expert_persistent_arena g_stream_expert_persistent_arena;
+static std::mutex g_stream_expert_persistent_arena_mutex;
+static int cuda_stream_expert_persistent_arena_release(int reset_class);
+
 static void cuda_stream_selected_cache_invalidate(void) {
     g_stream_selected_cache.valid = 0;
-}
-
-static void cuda_stream_selected_cache_release(void) {
-    const int tier = g_stream_selected_cache.logical_tier;
-    if (tier >= 0 && tier < g_n_gpus) {
-        (void)ds4_gpu_set_current_device(tier);
-    }
-    if (g_stream_selected_cache.gate_ptr) {
-        (void)cudaFree(g_stream_selected_cache.gate_ptr);
-    }
-    if (g_stream_selected_cache.up_ptr) {
-        (void)cudaFree(g_stream_selected_cache.up_ptr);
-    }
-    if (g_stream_selected_cache.down_ptr) {
-        (void)cudaFree(g_stream_selected_cache.down_ptr);
-    }
-    if (g_stream_selected_cache.slot_selected_ptr) {
-        (void)cudaFree(g_stream_selected_cache.slot_selected_ptr);
-    }
-    memset(&g_stream_selected_cache, 0, sizeof(g_stream_selected_cache));
-    g_stream_selected_cache.logical_tier = -1;
+    g_stream_selected_cache.storage_kind =
+        CUDA_STREAM_SELECTED_STORAGE_NONE;
+    g_stream_selected_cache.upload_event_value = 0;
+    g_stream_selected_cache.gate_ptr = NULL;
+    g_stream_selected_cache.up_ptr = NULL;
+    g_stream_selected_cache.down_ptr = NULL;
+    g_stream_selected_cache.gate_capacity = 0;
+    g_stream_selected_cache.up_capacity = 0;
+    g_stream_selected_cache.down_capacity = 0;
+    g_stream_selected_cache.slot_selected_ptr = NULL;
+    g_stream_selected_cache.slot_selected_capacity = 0;
+    memset(&g_stream_selected_cache.slot_selected_tensor, 0,
+           sizeof(g_stream_selected_cache.slot_selected_tensor));
 }
 
 typedef struct {
@@ -283,6 +375,14 @@ static void cuda_decode_dispatch_env_refresh(void) {
     g_cuda_exact_score_split_fuse_inv_rope =
         getenv("DS4_CUDA_EXACT_SCORE_SPLIT_FUSE_INV_ROPE") != NULL;
     g_cuda_moe_decode_graph = getenv("DS4_CUDA_MOE_DECODE_GRAPH") != NULL;
+    /* Graph dumps consume the gate/up/mid intermediates that direct Q2
+     * prefill intentionally reuses as scratch. */
+    g_cuda_direct_q2_prefill =
+        getenv("DS4_CUDA_NO_DIRECT_Q2_PREFILL") == NULL &&
+        getenv("DS4_METAL_GRAPH_DUMP_PREFIX") == NULL &&
+        getenv("DS4_ROCM_GRAPH_DUMP_PREFIX") == NULL;
+    g_cuda_f16_pair_compressor_store =
+        getenv("DS4_CUDA_NO_F16_PAIR_COMPRESSOR_STORE") == NULL;
 }
 
 /* WITH_DEVICE(d) { ... } scope macro.
@@ -376,6 +476,46 @@ struct cuda_q8_f32_range {
     int device_id;          /* physical CUDA device id; 0 in single-tier */
 };
 
+/* Resident DeepSeek-V4 Flash attn_q_b acceleration.  Keep this cache
+ * separate from the older, opportunistic Q8 cache: Q4 preparation is an
+ * explicit all-or-nothing session preflight, while Q8 entries may be built
+ * lazily by unrelated projections. */
+struct cuda_q4_attn_q_b_f16_range {
+    const void *host_base;
+    uint64_t model_size;
+    uint64_t offset;
+    uint64_t weight_bytes;
+    uint64_t in_dim;
+    uint64_t out_dim;
+    uint64_t f16_bytes;
+    __half *device_ptr;
+    int device_id;
+};
+
+/* ds4_cuda.cu historically does not include ds4_gpu.h.  Mirror the public
+ * descriptor exactly so the extern "C" preparation entry retains that ABI. */
+typedef struct ds4_gpu_q4_attn_q_b_f16_sidecar_desc {
+    uint64_t weight_offset;
+    uint64_t weight_bytes;
+    uint64_t in_dim;
+    uint64_t out_dim;
+    uint32_t weight_type;
+    uint32_t layer;
+} ds4_gpu_q4_attn_q_b_f16_sidecar_desc;
+
+/* Decode-only exact compressor layout. Each lane retains its original
+ * contiguous 128-element accumulation chunk, while the 32 lanes' weights at
+ * a given iteration are interleaved for one coalesced transaction. */
+struct cuda_f16_pair_chunk32_range {
+    const void *host_base;
+    uint64_t weight0_offset;
+    uint64_t weight1_offset;
+    uint64_t in_dim;
+    uint32_t width;
+    __half2 *device_ptr;
+    int device_id;
+};
+
 enum cuda_derived_kind {
     CUDA_DERIVED_IQ2_XXS_ALIGNED_MOE = 4,
     CUDA_DERIVED_Q8_0_ALIGNED_DENSE = 5,
@@ -401,12 +541,82 @@ static std::vector<cuda_q8_f16_range> g_q8_f16_ranges;
 static std::unordered_map<uint64_t, size_t> g_q8_f16_by_offset;
 static std::vector<cuda_q8_f32_range> g_q8_f32_ranges;
 static std::unordered_map<uint64_t, size_t> g_q8_f32_by_offset;
+static std::vector<cuda_q4_attn_q_b_f16_range>
+    g_q4_attn_q_b_f16_ranges;
+static std::mutex g_q4_attn_q_b_f16_cache_mutex;
+static std::mutex g_q4_attn_q_b_f16_build_mutex;
+static uint64_t g_q4_attn_q_b_f16_bytes;
+static uint64_t g_q4_attn_q_b_f16_generation = 1u;
+static int g_q4_attn_q_b_f16_hard_disabled;
+static int g_q4_attn_q_b_f16_dispatch_disabled;
+static int g_q4_attn_q_b_f16_pending_evict;
+/* The long-prefill transient path expands exactly one q_b matrix at a time.
+ * Keep its weight and activation staging, plus the optional diagnostic F16
+ * projection, in one dedicated allocation so it cannot alias the legacy
+ * global CUDA scratch used by unrelated projections.  A single mutex covers
+ * both ownership and the complete enqueue sequence: the decode stream orders
+ * GPU work, while the mutex prevents two host threads from interleaving
+ * sequences that reuse this storage. */
+static __half *g_q4_attn_q_b_transient_f16_scratch;
+static uint64_t g_q4_attn_q_b_transient_f16_scratch_bytes;
+static int g_q4_attn_q_b_transient_f16_scratch_device = -1;
+static int g_q4_attn_q_b_transient_f16_runtime_disabled;
+static std::mutex g_q4_attn_q_b_transient_f16_mutex;
+/* Support-model residency means two independent GGUF mappings coexist.
+ * Keep the large q_b expansion disabled in that conservative mode. */
+static int g_q4_attn_q_b_f16_multi_model_active;
+static std::vector<cuda_f16_pair_chunk32_range> g_f16_pair_chunk32_ranges;
+static int g_f16_pair_chunk32_disabled_after_oom;
 static std::vector<cuda_derived_range> g_derived_ranges;
 static const void *g_derived_replace_map;
 static uint64_t g_derived_artifact_bytes;
 static double g_derived_artifact_build_secs;
 static int g_derived_replaces_complete;
 static void *g_aligned_q81_scratch;
+/* Preserve the larger direct-prefill arena while also covering token-aware
+ * Q4 grouped attention-A (8 tokens * 16 groups * K=4096 plus ids/alignment). */
+static const size_t CUDA_ALIGNED_Q81_SCRATCH_BYTES =
+    96u * 1024u * 1024u;
+
+/* Opt-in producer-fold sidecars.  A slot belongs permanently to one
+ * (physical device, stream) for the lifetime of a CUDA session.  That makes
+ * reuse stream ordered without events, while a one-shot publication prevents
+ * a stale activation pointer from being accepted by a later consumer.
+ *
+ * Host contract: ds4's CUDA inference dispatcher serializes a session on one
+ * host thread.  The mutex protects registry state and cross-stream/device
+ * invalidation, but it does not make prepare -> producer enqueue -> publish or
+ * take -> consumer enqueue atomic when arbitrary host threads share one CUDA
+ * stream.  Such external multi-thread stream submission is unsupported for
+ * this experiment; keep the fold disabled in that embedding. */
+#define CUDA_Q8_FOLD_SLOTS 16u
+#define CUDA_Q8_FOLD_SLOT_BYTES 16384u
+struct cuda_q8_fold_slot {
+    void         *q81;
+    size_t        capacity;
+    const void   *src;
+    const void   *model_map;
+    uint64_t      in_dim;
+    uint64_t      epoch;
+    cudaStream_t  stream;
+    int           device;
+    int           owner_valid;
+    int           ready;
+};
+static cuda_q8_fold_slot g_q8_fold_slots[CUDA_Q8_FOLD_SLOTS];
+static std::mutex g_q8_fold_mutex;
+static uint64_t g_q8_fold_epoch = 1u;
+static int g_q8_fold_last_device = -1;
+static uint64_t g_q8_fold_prepares;
+static uint64_t g_q8_fold_publishes;
+static uint64_t g_q8_fold_hits;
+static uint64_t g_q8_fold_misses;
+static uint64_t g_q8_fold_capture_rejects;
+static uint64_t g_q8_fold_invalidations;
+static int g_q8_fold_report_registered;
+static std::once_flag g_q8_fold_mode_once;
+static int g_q8_fold_mode;
+static std::atomic<int> g_q8_fold_ever_enabled{0};
 static uint64_t g_model_range_bytes;
 static uint64_t g_q8_f16_bytes;
 static uint64_t g_q8_f32_bytes;
@@ -431,14 +641,158 @@ static void *g_model_stage_raw[4];
 static void *g_model_stage[4];
 static cudaEvent_t g_model_stage_event[4];
 static uint64_t g_model_stage_bytes;
+static uint64_t g_model_stage_align = 1;
 static void *g_stream_selected_stage_raw[4];
 static void *g_stream_selected_stage[4];
 static cudaEvent_t g_stream_selected_stage_event[4];
 static uint64_t g_stream_selected_stage_bytes;
+static uint64_t g_stream_selected_stage_align = 1;
 static cudaStream_t g_stream_selected_upload_stream;
+static int g_stream_selected_upload_owner_device = -1;
+static int g_stream_selected_stage_poisoned;
+static int g_stream_selected_stage_test_fail_drain;
+static int32_t *g_stream_selected_remap_stage;
+static uint64_t g_stream_selected_remap_stage_capacity;
+static cudaStream_t g_stream_selected_readback_stream;
+static void *g_stream_selected_readback_stage;
+static uint64_t g_stream_selected_readback_stage_capacity;
+static cudaEvent_t g_stream_selected_compute_ready_event;
+static cudaEvent_t g_stream_selected_readback_done_event;
+static cudaEvent_t g_stream_selected_upload_done_event;
+static cudaEvent_t g_stream_selected_consume_done_event;
+static int g_stream_selected_event_owner_device = -1;
+static int g_stream_selected_consume_owner_device = -1;
+static uint64_t g_stream_selected_compute_event_value;
+static uint64_t g_stream_selected_upload_event_value;
+static uint64_t g_stream_selected_cache_generation;
+static uint64_t g_stream_selected_consume_generation;
+static int g_stream_selected_consume_pending;
+static int g_stream_selected_consume_poisoned;
+static std::mutex g_stream_selected_consume_mutex;
+static std::condition_variable g_stream_selected_consume_cv;
+static uint32_t g_stream_selected_consume_host_readers;
+static int g_stream_selected_writer_active;
+static std::once_flag g_stream_selected_batch_io_once;
+static int g_stream_selected_batch_io_enabled;
+static int g_stream_selected_batch_io_required;
+static int g_stream_selected_batch_io_oracle;
+static int g_stream_selected_batch_io_report_registered;
+static uint64_t g_stream_selected_batch_io_candidates;
+static uint64_t g_stream_selected_batch_io_attempts;
+static uint64_t g_stream_selected_batch_io_completed;
+static uint64_t g_stream_selected_batch_io_legacy;
+static uint64_t g_stream_selected_batch_io_safe_fallbacks;
+static uint64_t g_stream_selected_batch_io_failures;
+static uint64_t g_stream_selected_batch_io_required_failures;
+static uint64_t g_stream_selected_batch_io_oracle_runs;
+static uint64_t g_stream_selected_batch_io_oracle_failures;
+static uint64_t g_stream_selected_batch_io_tasks;
+static uint64_t g_stream_selected_batch_io_segments;
+static uint64_t g_stream_selected_batch_io_reads;
+static uint64_t g_stream_selected_batch_io_bytes;
+static std::once_flag g_stream_selected_event_pipeline_once;
+static int g_stream_selected_event_pipeline_enabled;
+static int g_stream_selected_event_pipeline_required;
+static int g_stream_selected_event_pipeline_oracle;
+static int g_stream_selected_event_pipeline_report_registered;
+static std::atomic<uint64_t> g_stream_selected_event_candidates{0};
+static std::atomic<uint64_t> g_stream_selected_event_signals{0};
+static std::atomic<uint64_t> g_stream_selected_event_readbacks{0};
+static std::atomic<uint64_t> g_stream_selected_event_uploads{0};
+static std::atomic<uint64_t> g_stream_selected_event_compute_waits{0};
+static std::atomic<uint64_t> g_stream_selected_event_safe_fallbacks{0};
+static std::atomic<uint64_t> g_stream_selected_event_failures{0};
+static std::atomic<uint64_t> g_stream_selected_event_required_failures{0};
+static std::atomic<uint64_t> g_stream_selected_event_oracle_runs{0};
+static std::atomic<uint64_t> g_stream_selected_event_oracle_failures{0};
+static std::once_flag g_iq2_ssd_grouped_once;
+static int g_iq2_ssd_grouped_enabled;
+static int g_iq2_ssd_grouped_required;
+static int g_iq2_ssd_grouped_stats;
+static int g_iq2_ssd_grouped_report_registered;
+static std::atomic<uint64_t> g_iq2_ssd_grouped_candidates{0};
+static std::atomic<uint64_t> g_iq2_ssd_grouped_eligible{0};
+static std::atomic<uint64_t> g_iq2_ssd_grouped_attempts{0};
+static std::atomic<uint64_t> g_iq2_ssd_grouped_completed{0};
+static std::atomic<uint64_t> g_iq2_ssd_grouped_not_applicable{0};
+static std::atomic<uint64_t> g_iq2_ssd_grouped_safe_fallbacks{0};
+static std::atomic<uint64_t> g_iq2_ssd_grouped_failures{0};
+static std::atomic<uint64_t> g_iq2_ssd_grouped_required_failures{0};
+static std::atomic<uint64_t> g_iq2_ssd_grouped_upload_waits{0};
+static std::atomic<uint64_t> g_iq2_ssd_grouped_lease_waits{0};
+static std::atomic<uint64_t> g_iq2_ssd_grouped_lease_records{0};
+static std::atomic<uint64_t> g_iq2_ssd_grouped_lease_drains{0};
+static std::once_flag g_stream_expert_persistent_once;
+static int g_stream_expert_persistent_enabled;
+static int g_stream_expert_persistent_required;
+static int g_stream_expert_persistent_stats;
+static int g_stream_expert_persistent_oracle;
+static int g_stream_expert_persistent_report_registered;
+static std::atomic<uint64_t> g_stream_expert_persistent_plan_attempts{0};
+static std::atomic<uint64_t> g_stream_expert_persistent_plans_built{0};
+static std::atomic<uint64_t> g_stream_expert_persistent_commits{0};
+static std::atomic<uint64_t> g_stream_expert_persistent_rollbacks{0};
+static std::atomic<uint64_t> g_stream_expert_persistent_hits{0};
+static std::atomic<uint64_t> g_stream_expert_persistent_misses{0};
+static std::atomic<uint64_t> g_stream_expert_persistent_duplicates{0};
+static std::atomic<uint64_t> g_stream_expert_persistent_free_assignments{0};
+static std::atomic<uint64_t> g_stream_expert_persistent_evictions{0};
+static std::atomic<uint64_t> g_stream_expert_persistent_rejects{0};
+static std::atomic<uint64_t> g_stream_expert_persistent_budget_rejects{0};
+static std::atomic<uint64_t> g_stream_expert_persistent_class_rejects{0};
+static std::atomic<uint64_t> g_stream_expert_persistent_protected_rejects{0};
+static std::atomic<uint64_t> g_stream_expert_persistent_key_misses{0};
+static std::atomic<uint64_t> g_stream_expert_persistent_overflow_rejects{0};
+static std::atomic<uint64_t> g_stream_expert_persistent_oracle_runs{0};
+static std::atomic<uint64_t> g_stream_expert_persistent_oracle_failures{0};
+static std::atomic<uint64_t> g_stream_expert_persistent_arena_allocations{0};
+static std::atomic<uint64_t> g_stream_expert_persistent_arena_reuses{0};
+static std::atomic<uint64_t> g_stream_expert_persistent_arena_releases{0};
+static std::atomic<uint64_t> g_stream_expert_persistent_arena_failures{0};
+static std::atomic<uint64_t> g_stream_expert_persistent_arena_oracle_runs{0};
+static std::atomic<uint64_t> g_stream_expert_persistent_arena_oracle_failures{0};
+static std::atomic<uint64_t> g_stream_expert_persistent_epochs_attempted{0};
+static std::atomic<uint64_t> g_stream_expert_persistent_epochs_published{0};
+static std::atomic<uint64_t> g_stream_expert_persistent_all_hit_epochs{0};
+static std::atomic<uint64_t> g_stream_expert_persistent_miss_epochs{0};
+static std::atomic<uint64_t> g_stream_expert_persistent_miss_experts{0};
+static std::atomic<uint64_t> g_stream_expert_persistent_weight_bytes{0};
+static std::atomic<uint64_t> g_stream_expert_persistent_remap_bytes{0};
+static std::atomic<uint64_t> g_stream_expert_persistent_upload_failures{0};
+static std::atomic<uint64_t> g_stream_expert_persistent_fallbacks{0};
+static std::atomic<uint64_t> g_stream_expert_persistent_slot_invalidations{0};
+static std::atomic<uint64_t> g_stream_expert_persistent_poisons{0};
+static std::atomic<uint64_t> g_stream_expert_persistent_dispatches{0};
+static std::atomic<uint64_t> g_stream_expert_transient_dispatches{0};
+static std::atomic<uint64_t> g_stream_expert_persistent_runtime_oracle_runs{0};
+static std::atomic<uint64_t> g_stream_expert_persistent_runtime_oracle_failures{0};
+static int g_stream_expert_persistent_test_fail_after_enqueue;
+static int g_stream_expert_persistent_runtime_ready;
+static std::once_flag g_q8_hc_expand_policy_once;
+static int g_q8_hc_expand_force_fused;
+static int g_q8_hc_expand_split_requested;
+static int g_q8_hc_expand_stats;
+static int g_q8_hc_expand_report_registered;
+static std::atomic<uint64_t> g_q8_hc_expand_candidates{0};
+static std::atomic<uint64_t> g_q8_hc_expand_fused_attempts{0};
+static std::atomic<uint64_t> g_q8_hc_expand_fused_completed{0};
+static std::atomic<uint64_t> g_q8_hc_expand_split_attempts{0};
+static std::atomic<uint64_t> g_q8_hc_expand_split_completed{0};
+static std::atomic<uint64_t> g_q8_hc_expand_failures{0};
+static std::atomic<uint64_t> g_q8_hc_expand_capture_candidates{0};
+static std::atomic<uint64_t> g_q8_hc_expand_owned_forced_fused{0};
+static std::atomic<uint64_t> g_q8_hc_expand_multi_gpu_forced_fused{0};
+static std::atomic<uint64_t> g_q8_hc_expand_oracle_runs{0};
+static std::atomic<uint64_t> g_q8_hc_expand_oracle_failures{0};
 
 static int cuda_ok(cudaError_t err, const char *what);
+static void cuda_stream_selected_event_pipeline_release(void);
+static int cuda_stream_selected_wait_upload_on(
+        uint64_t event_value, cudaStream_t stream, const char *label);
+extern "C" int ds4_gpu_stream_expert_cache_wait_selected_upload(
+        uint64_t event_value, const char *label);
 extern "C" void ds4_gpu_decode_graphs_invalidate(void);
+static void cuda_decode_graphs_shutdown(void);
 static const char *cuda_model_range_ptr_from_fd(
         const void *model_map,
         uint64_t offset,
@@ -451,6 +805,234 @@ extern "C" int ds4_gpu_lookup_cache_strict(uint64_t source_offset,
                                             uint64_t bytes,
                                             int      expected_device,
                                             void   **out_device_ptr);
+
+static void cuda_q8_fold_report(void) {
+    fprintf(stderr,
+            "ds4: CUDA Q8_1 producer fold: prepares=%llu publishes=%llu "
+            "hits=%llu misses=%llu capture_rejects=%llu invalidations=%llu\n",
+            (unsigned long long)g_q8_fold_prepares,
+            (unsigned long long)g_q8_fold_publishes,
+            (unsigned long long)g_q8_fold_hits,
+            (unsigned long long)g_q8_fold_misses,
+            (unsigned long long)g_q8_fold_capture_rejects,
+            (unsigned long long)g_q8_fold_invalidations);
+}
+
+/* Experimental until the CUDA matrix has run on GB10/DGX.  The explicit
+ * disable is dominant even when the enable variable is present. */
+static void cuda_q8_fold_init_mode(void) {
+    const char *enable = getenv("DS4_CUDA_ENABLE_Q8_FOLD");
+    const char *disable = getenv("DS4_CUDA_NO_Q8_FOLD");
+    const int disabled = disable && disable[0] &&
+                         strcmp(disable, "0") != 0;
+    const int enabled = enable && strcmp(enable, "1") == 0;
+    g_q8_fold_mode = enabled && !disabled;
+    g_q8_fold_ever_enabled.store(g_q8_fold_mode,
+                                 std::memory_order_release);
+    if (enabled && !disabled && !g_q8_fold_report_registered) {
+        g_q8_fold_report_registered = 1;
+        (void)atexit(cuda_q8_fold_report);
+        fprintf(stderr,
+                "ds4: DS4_CUDA_ENABLE_Q8_FOLD=1 - experimental canonical "
+                "Q8_1 producer fold enabled\n");
+    }
+}
+
+static int cuda_q8_fold_enabled(void) {
+    std::call_once(g_q8_fold_mode_once, cuda_q8_fold_init_mode);
+    return g_q8_fold_mode;
+}
+
+static void cuda_q8_fold_invalidate_locked(void) {
+    g_q8_fold_epoch++;
+    if (g_q8_fold_epoch == 0u) g_q8_fold_epoch = 1u;
+    for (unsigned i = 0; i < CUDA_Q8_FOLD_SLOTS; i++) {
+        g_q8_fold_slots[i].src = NULL;
+        g_q8_fold_slots[i].model_map = NULL;
+        g_q8_fold_slots[i].in_dim = 0u;
+        g_q8_fold_slots[i].epoch = 0u;
+        g_q8_fold_slots[i].ready = 0;
+    }
+    g_q8_fold_invalidations++;
+}
+
+static void cuda_q8_fold_invalidate_all(void) {
+    if (!g_q8_fold_ever_enabled.load(std::memory_order_acquire)) return;
+    std::lock_guard<std::mutex> lock(g_q8_fold_mutex);
+    cuda_q8_fold_invalidate_locked();
+}
+
+static void cuda_q8_fold_release_all(void) {
+    std::lock_guard<std::mutex> lock(g_q8_fold_mutex);
+    int previous_device = -1;
+    (void)cudaGetDevice(&previous_device);
+    for (unsigned i = 0; i < CUDA_Q8_FOLD_SLOTS; i++) {
+        cuda_q8_fold_slot *slot = &g_q8_fold_slots[i];
+        if (slot->q81) {
+            if (slot->owner_valid) (void)cudaSetDevice(slot->device);
+            (void)cudaFree(slot->q81);
+        }
+        memset(slot, 0, sizeof(*slot));
+        slot->device = -1;
+    }
+    if (previous_device >= 0) (void)cudaSetDevice(previous_device);
+    g_q8_fold_last_device = -1;
+    cuda_q8_fold_invalidate_locked();
+}
+
+static int cuda_q8_fold_stream_is_eager(cudaStream_t stream) {
+    cudaStreamCaptureStatus status = cudaStreamCaptureStatusNone;
+    const cudaError_t err = cudaStreamIsCapturing(stream, &status);
+    if (err == cudaSuccess && status == cudaStreamCaptureStatusNone) return 1;
+    (void)cudaGetLastError();
+    std::lock_guard<std::mutex> lock(g_q8_fold_mutex);
+    g_q8_fold_capture_rejects++;
+    cuda_q8_fold_invalidate_locked();
+    return 0;
+}
+
+/* Reserve the stream-owned sidecar before launching a producer.  The
+ * allocation is deliberately forbidden during capture.  New producers on
+ * the same stream may replace an untaken publication: CUDA stream ordering
+ * guarantees that the old consumer, when there was one, has already read it. */
+static cuda_block_q8_1 *cuda_q8_fold_prepare(
+        const void *src, uint64_t in_dim, const void *model_map,
+        cudaStream_t stream) {
+    if (!cuda_q8_fold_enabled() || !src || in_dim != 4096u ||
+        (in_dim & 31u) != 0u || g_n_gpus != 1 ||
+        !model_map || model_map != g_model_host_base) {
+        return NULL;
+    }
+    if (!cuda_q8_fold_stream_is_eager(stream)) return NULL;
+
+    int device = -1;
+    if (cudaGetDevice(&device) != cudaSuccess) {
+        (void)cudaGetLastError();
+        cuda_q8_fold_invalidate_all();
+        return NULL;
+    }
+    const uint64_t blocks = in_dim / 32u;
+    if (blocks > SIZE_MAX / sizeof(cuda_block_q8_1)) return NULL;
+    const size_t bytes = (size_t)blocks * sizeof(cuda_block_q8_1);
+    if (bytes > CUDA_Q8_FOLD_SLOT_BYTES) return NULL;
+
+    std::lock_guard<std::mutex> lock(g_q8_fold_mutex);
+    if (g_q8_fold_last_device >= 0 &&
+        g_q8_fold_last_device != device) {
+        cuda_q8_fold_invalidate_locked();
+    }
+    g_q8_fold_last_device = device;
+    cuda_q8_fold_slot *slot = NULL;
+    for (unsigned i = 0; i < CUDA_Q8_FOLD_SLOTS; i++) {
+        cuda_q8_fold_slot *candidate = &g_q8_fold_slots[i];
+        if (candidate->owner_valid && candidate->device == device &&
+            candidate->stream == stream) {
+            slot = candidate;
+            break;
+        }
+        if (!slot && !candidate->owner_valid) slot = candidate;
+    }
+    if (!slot) return NULL;
+
+    if (!slot->owner_valid) {
+        void *sidecar = NULL;
+        const cudaError_t alloc_err =
+            cudaMalloc(&sidecar, CUDA_Q8_FOLD_SLOT_BYTES);
+        if (alloc_err != cudaSuccess || !sidecar) {
+            (void)cudaGetLastError();
+            return NULL;
+        }
+        slot->q81 = sidecar;
+        slot->capacity = CUDA_Q8_FOLD_SLOT_BYTES;
+        slot->stream = stream;
+        slot->device = device;
+        slot->owner_valid = 1;
+    }
+    if (!slot->q81 || slot->capacity < bytes) return NULL;
+
+    slot->src = src;
+    slot->model_map = model_map;
+    slot->in_dim = in_dim;
+    slot->epoch = g_q8_fold_epoch;
+    slot->ready = 0;
+    g_q8_fold_prepares++;
+    return (cuda_block_q8_1 *)slot->q81;
+}
+
+static void cuda_q8_fold_publish(
+        const void *src, uint64_t in_dim, const void *model_map,
+        cudaStream_t stream, const void *q81) {
+    if (!q81) return;
+    if (!cuda_q8_fold_stream_is_eager(stream)) return;
+    int device = -1;
+    if (cudaGetDevice(&device) != cudaSuccess) {
+        (void)cudaGetLastError();
+        cuda_q8_fold_invalidate_all();
+        return;
+    }
+    std::lock_guard<std::mutex> lock(g_q8_fold_mutex);
+    if (g_q8_fold_last_device >= 0 &&
+        g_q8_fold_last_device != device) {
+        cuda_q8_fold_invalidate_locked();
+    }
+    g_q8_fold_last_device = device;
+    for (unsigned i = 0; i < CUDA_Q8_FOLD_SLOTS; i++) {
+        cuda_q8_fold_slot *slot = &g_q8_fold_slots[i];
+        if (slot->q81 == q81 && slot->device == device &&
+            slot->src == src &&
+            slot->model_map == model_map && slot->in_dim == in_dim &&
+            slot->stream == stream && slot->epoch == g_q8_fold_epoch) {
+            slot->ready = 1;
+            g_q8_fold_publishes++;
+            return;
+        }
+    }
+}
+
+static int cuda_q8_fold_take(
+        const void *src, uint64_t in_dim, cudaStream_t stream,
+        const void **q81) {
+    if (q81) *q81 = NULL;
+    if (!q81 || !cuda_q8_fold_enabled() || !src || in_dim == 0u ||
+        g_n_gpus != 1 || !g_model_host_base) {
+        return 0;
+    }
+    if (!cuda_q8_fold_stream_is_eager(stream)) return 0;
+    int device = -1;
+    if (cudaGetDevice(&device) != cudaSuccess) {
+        (void)cudaGetLastError();
+        cuda_q8_fold_invalidate_all();
+        return 0;
+    }
+
+    std::lock_guard<std::mutex> lock(g_q8_fold_mutex);
+    if (g_q8_fold_last_device >= 0 &&
+        g_q8_fold_last_device != device) {
+        cuda_q8_fold_invalidate_locked();
+    }
+    g_q8_fold_last_device = device;
+    for (unsigned i = 0; i < CUDA_Q8_FOLD_SLOTS; i++) {
+        cuda_q8_fold_slot *slot = &g_q8_fold_slots[i];
+        if (slot->ready && slot->src == src && slot->in_dim == in_dim &&
+            slot->stream == stream && slot->device == device &&
+            slot->model_map == g_model_host_base &&
+            slot->epoch == g_q8_fold_epoch) {
+            *q81 = slot->q81;
+            slot->ready = 0;
+            slot->src = NULL;
+            g_q8_fold_hits++;
+            return 1;
+        }
+    }
+    g_q8_fold_misses++;
+    /* A pointer miss is a sequencing mismatch, not merely an absent key.
+     * Tensor scratch addresses can recur after their contents were replaced;
+     * retaining any ready publication across that mismatch could therefore
+     * turn a later address match into a stale hit.  Bump the session epoch
+     * and discard every publication while already holding the registry lock. */
+    cuda_q8_fold_invalidate_locked();
+    return 0;
+}
 __global__ static void dequant_q8_0_to_f16_kernel(
         __half *out,
         const unsigned char *w,
@@ -463,6 +1045,14 @@ __global__ static void dequant_q8_0_to_f32_kernel(
         uint64_t in_dim,
         uint64_t out_dim,
         uint64_t blocks);
+__global__ static void dequant_q4_K_to_f16_kernel(
+        __half *out,
+        const cuda_block_q4_K *w,
+        uint64_t in_dim,
+        uint64_t out_dim,
+        uint64_t blocks);
+
+extern "C" int ds4_gpu_release_q4_attn_q_b_f16_sidecars(void);
 
 static int cuda_aligned_iq2_enabled(void) {
     const char *s = getenv("DS4_CUDA_MOE_NO_IQ2_ALIGNED");
@@ -555,6 +1145,7 @@ static int cuda_span_fully_replaced(
 static void *cuda_tmp_alloc(uint64_t bytes, const char *what) {
     if (bytes == 0) return NULL;
     if (g_cuda_tmp_bytes >= bytes) return g_cuda_tmp;
+    cuda_q8_fold_invalidate_all();
     if (g_cuda_tmp) {
         if (!cuda_ok(cudaDeviceSynchronize(),
                      "synchronize CUDA scratch growth")) {
@@ -581,6 +1172,7 @@ static void *cuda_tmp_alloc(uint64_t bytes, const char *what) {
 static void *tt_scratch_ensure(uint64_t bytes, const char *what) {
     if (bytes == 0) return NULL;
     if (g_tt_scratch_bytes >= bytes) return g_tt_scratch;
+    cuda_q8_fold_invalidate_all();
 
     int device = -1;
     if (!cuda_ok(cudaGetDevice(&device), "get device for token-tile scratch")) {
@@ -638,6 +1230,7 @@ static void *cuda_tmp_alloc_on(int logical_tier, uint64_t bytes, const char *wha
     }
     ds4_gpu_ctx *ctx = &g_gpu[logical_tier];
     if (ctx->scratch_bytes >= bytes) return ctx->scratch;
+    cuda_q8_fold_invalidate_all();
     int prev = -1;
     cudaError_t derr = cudaGetDevice(&prev);
     if (derr != cudaSuccess) {
@@ -690,6 +1283,8 @@ static const char *cuda_model_ptr(const void *model_map, uint64_t offset) {
 
 static const char *cuda_model_range_ptr(const void *model_map, uint64_t offset, uint64_t bytes, const char *what) {
     if (bytes == 0) return cuda_model_ptr(model_map, offset);
+    /* Whole-image ownership and host registration describe only the active
+     * target map. A second DSpark mmap must resolve through its own cache. */
     const uint64_t end = offset + bytes;
     if (end < offset) return NULL;
     auto exact = g_model_range_by_offset.find(offset);
@@ -862,7 +1457,11 @@ static inline cublasHandle_t cuda_cublas_for_tier(int logical_tier) {
  * DS4_CUDA_DECODE_GRAPHS=0 (or off/no/false) disables everything. */
 #define CUDA_DECODE_GRAPH_LAYERS   64u
 #define CUDA_DECODE_GRAPH_ISLANDS   2u
-#define CUDA_DECODE_GRAPH_VARIANTS  4u
+#define CUDA_DECODE_GRAPH_BASE_VARIANTS    4u
+#define CUDA_DECODE_GRAPH_EXACTN_VARIANTS  5u
+#define CUDA_DECODE_GRAPH_VARIANTS \
+    (CUDA_DECODE_GRAPH_BASE_VARIANTS + CUDA_DECODE_GRAPH_EXACTN_VARIANTS)
+#define CUDA_DECODE_GRAPH_VARIANT_EXACTN 0x80000000u
 
 /* Mirrors the public `struct ds4_decode_graph_key` decl in ds4_gpu.h
  * byte-for-byte (ds4_cuda.cu does not include that header; it carries
@@ -896,6 +1495,22 @@ static cudaStream_t g_decode_graph_stream = NULL;
 static int g_decode_graph_capturing = 0;
 static uint64_t g_decode_graph_replays = 0;
 static uint64_t g_decode_graph_captures = 0;
+static uint64_t g_decode_graph_warms = 0;
+static uint64_t g_decode_graph_no_slots = 0;
+static uint64_t g_decode_graph_failures = 0;
+
+extern "C" void ds4_gpu_decode_graph_counters(
+        uint64_t *captures,
+        uint64_t *replays,
+        uint64_t *warms,
+        uint64_t *no_slots,
+        uint64_t *failures) {
+    if (captures) *captures = g_decode_graph_captures;
+    if (replays) *replays = g_decode_graph_replays;
+    if (warms) *warms = g_decode_graph_warms;
+    if (no_slots) *no_slots = g_decode_graph_no_slots;
+    if (failures) *failures = g_decode_graph_failures;
+}
 
 extern "C" int ds4_gpu_decode_graphs_supported(void) {
     static int init = 0;
@@ -908,7 +1523,20 @@ extern "C" int ds4_gpu_decode_graphs_supported(void) {
              strcmp(s, "off") == 0 || strcmp(s, "OFF") == 0 ||
              strcmp(s, "no") == 0 || strcmp(s, "NO") == 0 ||
              strcmp(s, "false") == 0 || strcmp(s, "FALSE") == 0);
-        if (off) {
+        const char *oracle =
+            getenv("DS4_CUDA_Q4_ATTN_OUT_HC_ORACLE");
+        const char *grouped_oracle =
+            getenv("DS4_CUDA_Q4_GROUPED_ATTN_A_ORACLE");
+        const int oracle_on =
+            (oracle && *oracle && strcmp(oracle, "0") != 0) ||
+            (grouped_oracle && *grouped_oracle &&
+             strcmp(grouped_oracle, "0") != 0);
+        if (oracle_on) {
+            fprintf(stderr,
+                    "ds4: CUDA decode graph capture disabled for Q4 "
+                    "attention oracle\n");
+            enabled = 0;
+        } else if (off) {
             fprintf(stderr, "ds4: DS4_CUDA_DECODE_GRAPHS=%s - decode graph capture disabled\n", s);
             enabled = 0;
         } else {
@@ -950,12 +1578,35 @@ extern "C" void ds4_gpu_decode_graphs_invalidate(void) {
     }
 }
 
+static void cuda_decode_graphs_shutdown(void) {
+    if (g_decode_graph_stream) {
+        (void)cudaStreamSynchronize(g_decode_graph_stream);
+    }
+    ds4_gpu_decode_graphs_invalidate();
+    if (g_decode_graph_stream) {
+        (void)cudaStreamDestroy(g_decode_graph_stream);
+        g_decode_graph_stream = NULL;
+    }
+    g_decode_graph_capturing = 0;
+    g_decode_graph_replays = 0;
+    g_decode_graph_captures = 0;
+    g_decode_graph_warms = 0;
+    g_decode_graph_no_slots = 0;
+    g_decode_graph_failures = 0;
+}
+
 static cuda_decode_graph_entry *cuda_decode_graph_find(
         const ds4_decode_graph_key *key) {
     if (key->il >= CUDA_DECODE_GRAPH_LAYERS ||
         key->island >= CUDA_DECODE_GRAPH_ISLANDS) return NULL;
+    const bool exactn_domain =
+        (key->variant & CUDA_DECODE_GRAPH_VARIANT_EXACTN) != 0u;
+    const uint32_t first = exactn_domain ?
+        CUDA_DECODE_GRAPH_BASE_VARIANTS : 0u;
+    const uint32_t end = exactn_domain ?
+        CUDA_DECODE_GRAPH_VARIANTS : CUDA_DECODE_GRAPH_BASE_VARIANTS;
     cuda_decode_graph_entry *slot = NULL;
-    for (uint32_t v = 0; v < CUDA_DECODE_GRAPH_VARIANTS; v++) {
+    for (uint32_t v = first; v < end; v++) {
         cuda_decode_graph_entry *e = &g_decode_graphs[key->il][key->island][v];
         if (e->state != 0 &&
             memcmp(&e->key, key, sizeof(*key)) == 0) return e;
@@ -972,12 +1623,27 @@ static cuda_decode_graph_entry *cuda_decode_graph_find(
 extern "C" int ds4_gpu_decode_graph_begin(const ds4_decode_graph_key *key) {
     if (!key || !ds4_gpu_decode_graphs_supported()) return -1;
     if (g_decode_graph_capturing) return -1;   /* no nesting */
+    /* A captured/replayed island owns its stream schedule.  Never carry an
+     * eager host-side sidecar publication across that boundary. */
+    cuda_q8_fold_invalidate_all();
     cuda_decode_graph_entry *e = cuda_decode_graph_find(key);
-    if (!e || e->state == 3) return -1;
+    if (!e) {
+        g_decode_graph_no_slots++;
+        if (getenv("DS4_CUDA_DECODE_GRAPH_LOG") != NULL) {
+            fprintf(stderr,
+                    "ds4: decode graph no slot il=%u island=%u variant=0x%08x domain=%s\n",
+                    key->il, key->island, key->variant,
+                    (key->variant & CUDA_DECODE_GRAPH_VARIANT_EXACTN) ?
+                        "exactn" : "decode");
+        }
+        return -1;
+    }
+    if (e->state == 3) return -1;
     if (e->state == 0) {
         /* Warm pass: run eagerly once so lazy allocators (tmp scratch,
          * cuBLAS workspaces) reach steady-state sizes before capture. */
         e->state = 1;
+        g_decode_graph_warms++;
         return -1;
     }
     if (e->state == 2) {
@@ -987,6 +1653,7 @@ extern "C" int ds4_gpu_decode_graph_begin(const ds4_decode_graph_key *key) {
                     key->il, key->island, cudaGetErrorString(err));
             (void)cudaGetLastError();
             cuda_decode_graph_entry_kill(e);
+            g_decode_graph_failures++;
             return -1;     /* caller encodes eagerly; nothing was consumed */
         }
         e->hits++;
@@ -999,6 +1666,7 @@ extern "C" int ds4_gpu_decode_graph_begin(const ds4_decode_graph_key *key) {
                      "decode graph stream create")) {
             g_decode_graph_stream = NULL;
             cuda_decode_graph_entry_kill(e);
+            g_decode_graph_failures++;
             return -1;
         }
     }
@@ -1010,6 +1678,7 @@ extern "C" int ds4_gpu_decode_graph_begin(const ds4_decode_graph_key *key) {
                  "decode graph begin capture")) {
         (void)cublasSetStream(cuda_cublas_for_tier(0), NULL);
         cuda_decode_graph_entry_kill(e);
+        g_decode_graph_failures++;
         return -1;
     }
     g_decode_graph_capturing = 1;
@@ -1029,10 +1698,12 @@ extern "C" int ds4_gpu_decode_graph_end(const ds4_decode_graph_key *key) {
         (void)cudaGetLastError();
         if (graph) (void)cudaGraphDestroy(graph);
         if (e) cuda_decode_graph_entry_kill(e);
+        g_decode_graph_failures++;
         return -1;         /* caller re-encodes the island eagerly */
     }
     if (!e) {              /* cannot happen: begin() found it */
         (void)cudaGraphDestroy(graph);
+        g_decode_graph_failures++;
         return -1;
     }
     cudaGraphExec_t exec = NULL;
@@ -1043,6 +1714,7 @@ extern "C" int ds4_gpu_decode_graph_end(const ds4_decode_graph_key *key) {
                 key->il, key->island, cudaGetErrorString(err));
         (void)cudaGetLastError();
         cuda_decode_graph_entry_kill(e);
+        g_decode_graph_failures++;
         return -1;
     }
     /* Capture recorded the work without executing it: launch now so this
@@ -1054,14 +1726,17 @@ extern "C" int ds4_gpu_decode_graph_end(const ds4_decode_graph_key *key) {
         (void)cudaGetLastError();
         (void)cudaGraphExecDestroy(exec);
         cuda_decode_graph_entry_kill(e);
+        g_decode_graph_failures++;
         return -1;
     }
     e->exec = exec;
     e->state = 2;
     g_decode_graph_captures++;
     if (getenv("DS4_CUDA_DECODE_GRAPH_LOG") != NULL) {
-        fprintf(stderr, "ds4: decode graph captured il=%u island=%u (total %llu)\n",
-                key->il, key->island,
+        fprintf(stderr, "ds4: decode graph captured il=%u island=%u variant=0x%08x domain=%s (total %llu)\n",
+                key->il, key->island, key->variant,
+                (key->variant & CUDA_DECODE_GRAPH_VARIANT_EXACTN) ?
+                    "exactn" : "decode",
                 (unsigned long long)g_decode_graph_captures);
     }
     return 0;
@@ -1082,14 +1757,22 @@ extern "C" int ds4_gpu_decode_graph_end(const ds4_decode_graph_key *key) {
  * prefill logits drift at ULP scale: validated against the official
  * continuation vectors rather than byte-diffs.  DS4_CUDA_MMQ=0 restores
  * the legacy dispatch. */
-/* Producer-fold registry lookup the vendored ds4_mmq.cu entries probe for
- * pre-quantized q8_1 activations (the fork's flat-pool M2-Inc2a fold).
- * The flat-pool producers are not ported, so nothing is ever registered:
- * always miss, and the entries run their own activation quantize. */
+/* Producer-fold registry lookup used by the vendored MMVQ entries.  Stream
+ * identity is part of the ABI: accepting a sidecar emitted on another stream
+ * without an event would be a data race, so such a lookup must miss. */
 extern "C" int ds4_cuda_q8_fold_take_q81(const void *src, uint64_t in_dim,
+                                         cudaStream_t stream,
                                          const void **q81) {
-    (void)src; (void)in_dim; (void)q81;
-    return 0;
+    return cuda_q8_fold_take(src, in_dim, stream, q81);
+}
+
+/* Test-only fail-closed control used by the resident Q4 benchmark.  Keep this
+ * out of the environment surface: production dispatch retains its established
+ * MMQ-to-Q8_K rollback unless an explicit test caller enables strict mode. */
+static int g_cuda_test_q4_mmq_strict;
+
+extern "C" void ds4_cuda_test_set_q4_mmq_strict(int required) {
+    g_cuda_test_q4_mmq_strict = required != 0;
 }
 
 static int cuda_use_mmq(void) {
@@ -1115,8 +1798,9 @@ static int cuda_use_mmq(void) {
 /* MXFP4 has no dequant+cublas fallback, so it must retain MMQ on multi-GPU
  * placements where the optional Q8/IQ2 prefill tier stays disabled. MMQ
  * resolves the active CUDA device on every call; initialization only warms
- * its device-info singleton. The experimental global persistent scratch is
- * intentionally rejected because one pointer cannot span CUDA devices. */
+ * its device-info singleton. The grouped persistent Q8_1 arena is owned by
+ * MMQ, device-bound, and never consumed by the MXFP4 wrappers, so it does not
+ * constrain this required multi-GPU path. */
 static int cuda_use_mxfp4_mmq(void) {
     static int init = 0;
     static int use = 0;
@@ -1126,10 +1810,6 @@ static int cuda_use_mxfp4_mmq(void) {
         if (s && s[0] == '0') {
             fprintf(stderr,
                     "ds4: DS4_CUDA_MMQ=0 - MXFP4 MMQ disabled\n");
-        } else if (g_n_gpus > 1 &&
-                   getenv("DS4_CUDA_MMQ_Q81_PERSISTENT") != NULL) {
-            fprintf(stderr,
-                    "ds4: persistent MMQ Q8_1 scratch is unavailable with multi-GPU MXFP4\n");
         } else {
             int device = 0;
             if (cudaGetDevice(&device) == cudaSuccess &&
@@ -1206,6 +1886,7 @@ extern "C" void ds4_gpu_decode_graph_abort(const ds4_decode_graph_key *key) {
         cuda_decode_graph_entry *e = cuda_decode_graph_find(key);
         if (e) cuda_decode_graph_entry_kill(e);
     }
+    g_decode_graph_failures++;
 }
 
 /* Multi-tier-aware weight pointer resolver.
@@ -1260,13 +1941,54 @@ extern "C" int ds4_gpu_register_support_map(const void *map, uint64_t size, uint
     return 1;
 }
 
+static int cuda_resolved_model_range_is_device_resident(
+        const void *model_map,
+        uint64_t offset,
+        uint64_t bytes,
+        const char *resolved_ptr) {
+    if (!resolved_ptr) return 0;
+    if (model_map == g_model_host_base &&
+        g_model_device_owned && g_model_device_base &&
+        resolved_ptr == g_model_device_base + offset) {
+        return 1;
+    }
+    const uint64_t end = offset + bytes;
+    if (end < offset) return 0;
+    for (const cuda_model_range &r : g_model_ranges) {
+        if (r.host_base != model_map || !r.device_ptr ||
+            offset < r.offset) {
+            continue;
+        }
+        const uint64_t delta = offset - r.offset;
+        if (delta > r.bytes || bytes > r.bytes - delta) continue;
+        const char *range_ptr = r.device_ptr + delta;
+        if (range_ptr == resolved_ptr) {
+            /* host_registered ranges are CUDA-addressable but page-backed by
+             * the mmap. All other cuda_model_range entries originate from
+             * cudaMalloc (standalone or arena-backed) and are resident even
+             * on GB10's physically unified memory. */
+            return r.host_registered == 0;
+        }
+    }
+    return 0;
+}
+
 static const char *cuda_resolve_weight_ptr(const void *model_map,
                                             uint64_t offset,
                                             uint64_t bytes,
                                             int logical_tier,
-                                            const char *label) {
+                                            const char *label,
+                                            int *device_resident = NULL) {
+    if (device_resident) *device_resident = 0;
     if (g_n_gpus <= 1) {
-        return cuda_model_range_ptr(model_map, offset, bytes, label);
+        const char *ptr = cuda_model_range_ptr(
+            model_map, offset, bytes, label);
+        if (device_resident) {
+            *device_resident =
+                cuda_resolved_model_range_is_device_resident(
+                    model_map, offset, bytes, ptr);
+        }
+        return ptr;
     }
     if (g_support_host_base && model_map == g_support_host_base) {
         offset += g_support_offset_bias;
@@ -1281,6 +2003,7 @@ static const char *cuda_resolve_weight_ptr(const void *model_map,
     void *dev_ptr = NULL;
     if (ds4_gpu_lookup_cache_strict(offset, bytes, physical_device, &dev_ptr)
         && dev_ptr) {
+        if (device_resident) *device_resident = 1;
         return (const char *)dev_ptr;
     }
     /* GLM multi-tier: generic launchers resolve by the OUT tensor's tier,
@@ -1292,6 +2015,7 @@ static const char *cuda_resolve_weight_ptr(const void *model_map,
         cur_dev != physical_device &&
         ds4_gpu_lookup_cache_strict(offset, bytes, cur_dev, &dev_ptr) &&
         dev_ptr) {
+        if (device_resident) *device_resident = 1;
         return (const char *)dev_ptr;
     }
     fprintf(stderr,
@@ -1373,6 +2097,208 @@ static int cuda_env_flag_enabled(const char *name, int fallback) {
     return strcmp(env, "0") != 0;
 }
 
+enum cuda_q8_hc_expand_path {
+    CUDA_Q8_HC_EXPAND_FUSED = 1,
+    CUDA_Q8_HC_EXPAND_SPLIT = 2,
+};
+
+struct ds4_cuda_q8_hc_expand_report;
+typedef struct ds4_cuda_q8_hc_expand_report
+    ds4_cuda_q8_hc_expand_report;
+typedef struct {
+    uint64_t candidates;
+    uint64_t fused_attempts;
+    uint64_t fused_completed;
+    uint64_t split_attempts;
+    uint64_t split_completed;
+    uint64_t failures;
+    uint64_t capture_candidates;
+    uint64_t owned_forced_fused;
+    uint64_t multi_gpu_forced_fused;
+    uint64_t oracle_runs;
+    uint64_t oracle_failures;
+    int force_fused;
+    int split_requested;
+    int stats;
+} cuda_q8_hc_expand_report_layout;
+
+/* Resolve the Q8 shared-down/HC A/B policy without consulting global state.
+ * The opt-in split is deliberately narrow: single GPU and non-owned only.
+ * Force-fused wins conflicts, while owned/multi-GPU dispatches retain their
+ * existing fused implementation instead of turning an A/B switch into a
+ * hard inference failure.  Capture is observable but does not change the
+ * selected graph, so eager warm/capture record the same kernel sequence. */
+static int cuda_q8_hc_expand_resolve_policy(
+        int force_fused, int disable_fused, int n_gpus, int owned,
+        int capture, int *owned_forced_out, int *multi_gpu_forced_out,
+        int *capture_out) {
+    const int split_requested = disable_fused && !force_fused;
+    const int owned_forced = split_requested && owned;
+    const int multi_gpu_forced =
+        split_requested && !owned && n_gpus != 1;
+    if (owned_forced_out) *owned_forced_out = owned_forced;
+    if (multi_gpu_forced_out) *multi_gpu_forced_out = multi_gpu_forced;
+    if (capture_out) *capture_out = capture != 0;
+    return split_requested && n_gpus == 1 && !owned
+        ? CUDA_Q8_HC_EXPAND_SPLIT : CUDA_Q8_HC_EXPAND_FUSED;
+}
+
+extern "C" int ds4_cuda_test_q8_hc_expand_policy(
+        int force_fused, int disable_fused, int n_gpus, int owned,
+        int capture, int *fused_out, int *owned_forced_out,
+        int *multi_gpu_forced_out, int *capture_out) {
+    if (!fused_out || !owned_forced_out || !multi_gpu_forced_out ||
+        !capture_out) {
+        return 0;
+    }
+    const int path = cuda_q8_hc_expand_resolve_policy(
+        force_fused, disable_fused, n_gpus, owned, capture,
+        owned_forced_out, multi_gpu_forced_out, capture_out);
+    *fused_out = path == CUDA_Q8_HC_EXPAND_FUSED;
+    return 1;
+}
+
+extern "C" int ds4_cuda_test_q8_hc_expand_env_value(
+        const char *value) {
+    return cuda_env_value_enabled(value);
+}
+
+static void cuda_q8_hc_expand_report_at_exit(void) {
+    fprintf(stderr,
+            "ds4: CUDA Q8 shared-down/HC: candidates=%llu "
+            "fused=%llu/%llu split=%llu/%llu failures=%llu "
+            "capture=%llu owned_forced=%llu multi_gpu_forced=%llu "
+            "oracle=%llu/%llu\n",
+            (unsigned long long)g_q8_hc_expand_candidates.load(),
+            (unsigned long long)g_q8_hc_expand_fused_completed.load(),
+            (unsigned long long)g_q8_hc_expand_fused_attempts.load(),
+            (unsigned long long)g_q8_hc_expand_split_completed.load(),
+            (unsigned long long)g_q8_hc_expand_split_attempts.load(),
+            (unsigned long long)g_q8_hc_expand_failures.load(),
+            (unsigned long long)g_q8_hc_expand_capture_candidates.load(),
+            (unsigned long long)g_q8_hc_expand_owned_forced_fused.load(),
+            (unsigned long long)g_q8_hc_expand_multi_gpu_forced_fused.load(),
+            (unsigned long long)g_q8_hc_expand_oracle_runs.load(),
+            (unsigned long long)g_q8_hc_expand_oracle_failures.load());
+}
+
+static void cuda_q8_hc_expand_policy_init(void) {
+    std::call_once(g_q8_hc_expand_policy_once, []() {
+        g_q8_hc_expand_force_fused = cuda_env_value_enabled(
+            getenv("DS4_CUDA_Q8_HC_EXPAND_FUSED"));
+        const int disable_fused = cuda_env_value_enabled(
+            getenv("DS4_CUDA_DISABLE_Q8_HC_EXPAND_FUSED"));
+        g_q8_hc_expand_split_requested =
+            disable_fused && !g_q8_hc_expand_force_fused;
+        g_q8_hc_expand_stats = cuda_env_value_enabled(
+            getenv("DS4_CUDA_Q8_HC_EXPAND_STATS"));
+        if ((g_q8_hc_expand_force_fused || disable_fused ||
+             g_q8_hc_expand_stats) && !g_q8_hc_expand_report_registered) {
+            g_q8_hc_expand_report_registered = 1;
+            (void)atexit(cuda_q8_hc_expand_report_at_exit);
+        }
+        if (g_q8_hc_expand_force_fused) {
+            fprintf(stderr,
+                    "ds4: CUDA Q8 shared-down/HC fused path forced%s\n",
+                    disable_fused ? " (overrides split request)" : "");
+        } else if (g_q8_hc_expand_split_requested) {
+            fprintf(stderr,
+                    "ds4: CUDA Q8 shared-down/HC split A/B enabled "
+                    "for single-GPU non-owned dispatches\n");
+        }
+    });
+}
+
+static int cuda_q8_hc_expand_policy_path(int owned) {
+    cuda_q8_hc_expand_policy_init();
+    int owned_forced = 0;
+    int multi_gpu_forced = 0;
+    int capture = 0;
+    const int path = cuda_q8_hc_expand_resolve_policy(
+        g_q8_hc_expand_force_fused,
+        g_q8_hc_expand_split_requested,
+        g_n_gpus, owned, g_decode_graph_capturing,
+        &owned_forced, &multi_gpu_forced, &capture);
+    if (!g_q8_hc_expand_report_registered) return path;
+    g_q8_hc_expand_candidates.fetch_add(1u, std::memory_order_relaxed);
+    if (capture) {
+        g_q8_hc_expand_capture_candidates.fetch_add(
+            1u, std::memory_order_relaxed);
+    }
+    if (owned_forced) {
+        g_q8_hc_expand_owned_forced_fused.fetch_add(
+            1u, std::memory_order_relaxed);
+    }
+    if (multi_gpu_forced) {
+        g_q8_hc_expand_multi_gpu_forced_fused.fetch_add(
+            1u, std::memory_order_relaxed);
+    }
+    if (path == CUDA_Q8_HC_EXPAND_SPLIT) {
+        g_q8_hc_expand_split_attempts.fetch_add(
+            1u, std::memory_order_relaxed);
+    } else {
+        g_q8_hc_expand_fused_attempts.fetch_add(
+            1u, std::memory_order_relaxed);
+    }
+    return path;
+}
+
+static int cuda_q8_hc_expand_policy_complete(int path, int result) {
+    if (!g_q8_hc_expand_report_registered) return result;
+    if (result) {
+        if (path == CUDA_Q8_HC_EXPAND_SPLIT) {
+            g_q8_hc_expand_split_completed.fetch_add(
+                1u, std::memory_order_relaxed);
+        } else {
+            g_q8_hc_expand_fused_completed.fetch_add(
+                1u, std::memory_order_relaxed);
+        }
+    } else {
+        g_q8_hc_expand_failures.fetch_add(1u, std::memory_order_relaxed);
+    }
+    return result;
+}
+
+extern "C" void ds4_cuda_q8_hc_expand_get_report(
+        ds4_cuda_q8_hc_expand_report *report) {
+    if (!report) return;
+    cuda_q8_hc_expand_policy_init();
+    cuda_q8_hc_expand_report_layout out = {};
+    out.candidates = g_q8_hc_expand_candidates.load();
+    out.fused_attempts = g_q8_hc_expand_fused_attempts.load();
+    out.fused_completed = g_q8_hc_expand_fused_completed.load();
+    out.split_attempts = g_q8_hc_expand_split_attempts.load();
+    out.split_completed = g_q8_hc_expand_split_completed.load();
+    out.failures = g_q8_hc_expand_failures.load();
+    out.capture_candidates = g_q8_hc_expand_capture_candidates.load();
+    out.owned_forced_fused = g_q8_hc_expand_owned_forced_fused.load();
+    out.multi_gpu_forced_fused = g_q8_hc_expand_multi_gpu_forced_fused.load();
+    out.oracle_runs = g_q8_hc_expand_oracle_runs.load();
+    out.oracle_failures = g_q8_hc_expand_oracle_failures.load();
+    out.force_fused = g_q8_hc_expand_force_fused;
+    out.split_requested = g_q8_hc_expand_split_requested;
+    out.stats = g_q8_hc_expand_stats;
+    memcpy(report, &out, sizeof(out));
+}
+
+/* Conservative AProjQ4 port of the GB10 Q8 decode dispatch ideas.  Q4_K
+ * must keep the canonical MMVQ Q8_1 activation quantizer and reduction DAG;
+ * switching these projections to the local Q8_K fallback changes output
+ * bits.  The helpers below therefore only select launch/packing consumers
+ * around the existing MMVQ entries.  Non-GB10, multi-GPU, quality mode and
+ * DS4_CUDA_MMQ=0 all retain the ordinary caller fallback.  The global kill
+ * switch is intentionally checked in addition to each path's local rollback.
+ */
+static int cuda_q4_gb10_fast_path_enabled(
+        int logical_tier, const char *local_rollback_env) {
+    if (g_n_gpus != 1) return 0;
+    if (logical_tier < 0) logical_tier = 0;
+    if (logical_tier != 0 || !g_cuda_is_gb10[logical_tier]) return 0;
+    if (getenv("DS4_CUDA_NO_Q4_GB10_FAST") != NULL) return 0;
+    if (local_rollback_env && getenv(local_rollback_env) != NULL) return 0;
+    return cuda_use_mmq();
+}
+
 extern "C" int ds4_gpu_set_decode_fast_attention(int enabled) {
     const int old = g_decode_fast_attention;
     g_decode_fast_attention = enabled != 0;
@@ -1584,6 +2510,889 @@ static int cuda_q8_f16_preload_allowed(const char *label, uint64_t in_dim, uint6
     return cuda_q8_f16_cache_allowed(label, in_dim, out_dim);
 }
 
+enum {
+    CUDA_Q4_ATTN_Q_B_TYPE = 12u,
+    CUDA_Q4_ATTN_Q_B_IN_DIM = 1024u,
+    CUDA_Q4_ATTN_Q_B_OUT_DIM = 32768u,
+    CUDA_Q4_ATTN_Q_B_MAX_ENTRIES = 80u,
+};
+
+static uint64_t cuda_q4_attn_q_b_f16_cache_limit_bytes(void) {
+    int present = 0;
+    const uint64_t parsed = cuda_parse_mib_env(
+        "DS4_CUDA_Q4_ATTN_Q_B_F16_CACHE_MB", &present);
+    return present ? parsed : 3072ull * 1048576ull;
+}
+
+static uint32_t cuda_q4_attn_q_b_f16_min_tokens(void) {
+    int present = 0;
+    return cuda_parse_u32_env_clamped(
+        "DS4_CUDA_Q4_ATTN_Q_B_F16_CACHE_MIN_TOKENS",
+        512u, 32u, UINT32_MAX, &present);
+}
+
+static int cuda_q4_attn_q_b_f16_requested(void) {
+    return cuda_env_value_enabled(
+               getenv("DS4_CUDA_ENABLE_Q4_ATTN_Q_B_F16_CACHE")) ||
+           cuda_env_value_enabled(
+               getenv("DS4_CUDA_REQUIRE_Q4_ATTN_Q_B_F16_CACHE"));
+}
+
+static int cuda_q4_attn_q_b_f16_required(void) {
+    return cuda_env_value_enabled(
+        getenv("DS4_CUDA_REQUIRE_Q4_ATTN_Q_B_F16_CACHE"));
+}
+
+static int cuda_q4_attn_q_b_f16_disabled(void) {
+    return cuda_env_value_enabled(
+        getenv("DS4_CUDA_DISABLE_Q4_ATTN_Q_B_F16_CACHE"));
+}
+
+static uint32_t cuda_q4_attn_q_b_transient_f16_min_tokens(void) {
+    int present = 0;
+    return cuda_parse_u32_env_clamped(
+        "DS4_CUDA_Q4_ATTN_Q_B_TRANSIENT_F16_MIN_TOKENS",
+        4096u, 32u, UINT32_MAX, &present);
+}
+
+static int cuda_q4_attn_q_b_transient_f16_disabled(void) {
+    return cuda_env_value_enabled(
+        getenv("DS4_CUDA_DISABLE_Q4_ATTN_Q_B_TRANSIENT_F16"));
+}
+
+/* Diagnostic-only until CUDA logits/acceptance testing establishes the extra
+ * projection rounding as an acceptable release boundary.  The established
+ * path keeps the cuBLAS output and RMS/RoPE input in F32. */
+static int cuda_q4_attn_q_b_f16_output_requested(void) {
+    return cuda_env_value_enabled(
+        getenv("DS4_CUDA_ENABLE_Q4_ATTN_Q_B_F16_OUTPUT"));
+}
+
+static int cuda_q4_attn_q_b_transient_f16_scratch_size(
+        uint32_t max_rows,
+        int include_output,
+        uint64_t *weight_bytes,
+        uint64_t *activation_offset,
+        uint64_t *output_offset,
+        uint64_t *scratch_bytes) {
+    const uint64_t wh_bytes =
+        (uint64_t)CUDA_Q4_ATTN_Q_B_IN_DIM *
+        CUDA_Q4_ATTN_Q_B_OUT_DIM * sizeof(__half);
+    const uint64_t xh_off = (wh_bytes + 255u) & ~UINT64_C(255);
+    const uint64_t xh_row_bytes =
+        (uint64_t)CUDA_Q4_ATTN_Q_B_IN_DIM * sizeof(__half);
+    if (max_rows == 0u ||
+        (uint64_t)max_rows > (UINT64_MAX - xh_off) / xh_row_bytes) {
+        return 0;
+    }
+    const uint64_t xh_end =
+        xh_off + (uint64_t)max_rows * xh_row_bytes;
+    if (!include_output) {
+        if (xh_end > SIZE_MAX) return 0;
+        if (weight_bytes) *weight_bytes = wh_bytes;
+        if (activation_offset) *activation_offset = xh_off;
+        if (output_offset) *output_offset = 0u;
+        if (scratch_bytes) *scratch_bytes = xh_end;
+        return 1;
+    }
+    if (xh_end > UINT64_MAX - 255u) return 0;
+    const uint64_t qh_off = (xh_end + 255u) & ~UINT64_C(255);
+    const uint64_t qh_row_bytes =
+        (uint64_t)CUDA_Q4_ATTN_Q_B_OUT_DIM * sizeof(__half);
+    if ((uint64_t)max_rows >
+        (UINT64_MAX - qh_off) / qh_row_bytes) {
+        return 0;
+    }
+    const uint64_t total =
+        qh_off + (uint64_t)max_rows * qh_row_bytes;
+    if (total > SIZE_MAX) return 0;
+    if (weight_bytes) *weight_bytes = wh_bytes;
+    if (activation_offset) *activation_offset = xh_off;
+    if (output_offset) *output_offset = qh_off;
+    if (scratch_bytes) *scratch_bytes = total;
+    return 1;
+}
+
+/* The automatic path is intentionally limited to physical device storage.
+ * cuda_model_range_ptr may also expose managed/HMM or mapped host memory; a
+ * per-layer dequantization from either source would fold migration/PCIe cost
+ * into prefill and could be much slower than the native Q4 kernel. */
+static int cuda_q4_attn_q_b_source_is_device_resident(
+        const void *ptr,
+        int expected_device) {
+    if (!ptr) return 0;
+    cudaPointerAttributes attr = {};
+    const cudaError_t err = cudaPointerGetAttributes(&attr, ptr);
+    if (err != cudaSuccess) {
+        (void)cudaGetLastError();
+        return 0;
+    }
+#if CUDART_VERSION >= 10000
+    return attr.type == cudaMemoryTypeDevice &&
+           attr.device == expected_device;
+#else
+    return attr.memoryType == cudaMemoryTypeDevice &&
+           attr.device == expected_device;
+#endif
+}
+
+extern "C" int ds4_cuda_test_model_range_is_device_resident(
+        const void *model_map,
+        uint64_t model_size,
+        uint64_t offset,
+        uint64_t bytes,
+        int logical_tier) {
+    if (!model_map || model_size == 0u || bytes == 0u ||
+        offset > model_size || bytes > model_size - offset ||
+        g_n_gpus != 1 || logical_tier != 0 ||
+        model_map != g_model_host_base ||
+        model_size != g_model_registered_size ||
+        !g_model_device_owned || !g_model_device_base) {
+        return 0;
+    }
+
+    const char *resolved = g_model_device_base + offset;
+    if (!cuda_resolved_model_range_is_device_resident(
+            model_map, offset, bytes, resolved)) {
+        return 0;
+    }
+    return cuda_q4_attn_q_b_source_is_device_resident(
+        resolved, g_gpu[logical_tier].device_id);
+}
+
+extern "C" int ds4_cuda_test_model_range_device_ptr(
+        const void *model_map,
+        uint64_t model_size,
+        uint64_t offset,
+        uint64_t bytes,
+        int logical_tier,
+        const void **device_ptr) {
+    if (device_ptr) *device_ptr = NULL;
+    if (!device_ptr ||
+        !ds4_cuda_test_model_range_is_device_resident(
+            model_map, model_size, offset, bytes, logical_tier)) {
+        return 0;
+    }
+    *device_ptr = g_model_device_base + offset;
+    return 1;
+}
+
+/* Match the existing CUDA weight-cache safety floor without coupling this
+ * cache to the Q8-specific reserve environment variable.  The explicit
+ * future-session reserve supplied by ds4.c is added independently. */
+static uint64_t cuda_q4_attn_q_b_f16_reserve_bytes(uint64_t total_bytes) {
+    if (total_bytes >= 112ull * 1024ull * 1024ull * 1024ull) {
+        return 512ull * 1048576ull;
+    }
+    if (total_bytes >= 40ull * 1024ull * 1024ull * 1024ull) {
+        const uint64_t min_reserve = 768ull * 1048576ull;
+        const uint64_t pct_reserve = total_bytes / 100u;
+        return pct_reserve > min_reserve ? pct_reserve : min_reserve;
+    }
+    const uint64_t min_reserve = 4096ull * 1048576ull;
+    const uint64_t pct_reserve = total_bytes / 20u;
+    return pct_reserve > min_reserve ? pct_reserve : min_reserve;
+}
+
+static int cuda_q4_attn_q_b_transient_f16_scratch_release(void) {
+    std::lock_guard<std::mutex> lock(
+        g_q4_attn_q_b_transient_f16_mutex);
+    if (!g_q4_attn_q_b_transient_f16_scratch) {
+        g_q4_attn_q_b_transient_f16_scratch_bytes = 0;
+        g_q4_attn_q_b_transient_f16_scratch_device = -1;
+        g_q4_attn_q_b_transient_f16_runtime_disabled = 0;
+        return 1;
+    }
+
+    int previous_device = -1;
+    (void)cudaGetDevice(&previous_device);
+    const int owner = g_q4_attn_q_b_transient_f16_scratch_device;
+    if (owner < 0 || cudaSetDevice(owner) != cudaSuccess ||
+        cudaDeviceSynchronize() != cudaSuccess) {
+        (void)cudaGetLastError();
+        if (previous_device >= 0) (void)cudaSetDevice(previous_device);
+        return 0;
+    }
+    if (cudaFree(g_q4_attn_q_b_transient_f16_scratch) != cudaSuccess) {
+        (void)cudaGetLastError();
+        if (previous_device >= 0) (void)cudaSetDevice(previous_device);
+        return 0;
+    }
+    g_q4_attn_q_b_transient_f16_scratch = NULL;
+    g_q4_attn_q_b_transient_f16_scratch_bytes = 0;
+    g_q4_attn_q_b_transient_f16_scratch_device = -1;
+    g_q4_attn_q_b_transient_f16_runtime_disabled = 0;
+    {
+        std::lock_guard<std::mutex> cache_lock(
+            g_q4_attn_q_b_f16_cache_mutex);
+        if (++g_q4_attn_q_b_f16_generation == 0u) {
+            g_q4_attn_q_b_f16_generation = 1u;
+        }
+    }
+    if (previous_device >= 0) (void)cudaSetDevice(previous_device);
+    return 1;
+}
+
+static int cuda_q4_attn_q_b_transient_f16_scratch_ensure(
+        uint32_t max_rows,
+        uint64_t working_set_reserve_bytes,
+        uint64_t *prepared_bytes) {
+    const int include_output =
+        cuda_q4_attn_q_b_f16_output_requested();
+    uint64_t scratch_bytes = 0;
+    if (!cuda_q4_attn_q_b_transient_f16_scratch_size(
+            max_rows, include_output,
+            NULL, NULL, NULL, &scratch_bytes)) {
+        return 0;
+    }
+
+    std::lock_guard<std::mutex> lock(
+        g_q4_attn_q_b_transient_f16_mutex);
+    const int target_device = g_gpu[0].device_id;
+    if (g_q4_attn_q_b_transient_f16_scratch &&
+        g_q4_attn_q_b_transient_f16_scratch_device == target_device &&
+        g_q4_attn_q_b_transient_f16_scratch_bytes >= scratch_bytes) {
+        return 1;
+    }
+    /* A CUDA backend instance does not migrate its single-GPU arena between
+     * physical devices. Cleanup/reinit owns that transition; preserving the
+     * old allocation is safer than partially replacing it here. */
+    if (g_q4_attn_q_b_transient_f16_scratch &&
+        g_q4_attn_q_b_transient_f16_scratch_device != target_device) {
+        return 0;
+    }
+
+    int previous_device = -1;
+    if (cudaGetDevice(&previous_device) != cudaSuccess ||
+        cudaSetDevice(target_device) != cudaSuccess) {
+        (void)cudaGetLastError();
+        if (previous_device >= 0) (void)cudaSetDevice(previous_device);
+        return 0;
+    }
+
+    size_t free_bytes = 0;
+    size_t total_bytes = 0;
+    if (cudaMemGetInfo(&free_bytes, &total_bytes) != cudaSuccess) {
+        (void)cudaGetLastError();
+        if (previous_device >= 0) (void)cudaSetDevice(previous_device);
+        return 0;
+    }
+    const uint64_t effective_free = (uint64_t)free_bytes;
+    const uint64_t reserve =
+        cuda_q4_attn_q_b_f16_reserve_bytes((uint64_t)total_bytes);
+    uint64_t required_free = scratch_bytes;
+    if (required_free > UINT64_MAX - reserve) {
+        if (previous_device >= 0) (void)cudaSetDevice(previous_device);
+        return 0;
+    }
+    required_free += reserve;
+    if (required_free > UINT64_MAX - working_set_reserve_bytes) {
+        if (previous_device >= 0) (void)cudaSetDevice(previous_device);
+        return 0;
+    }
+    required_free += working_set_reserve_bytes;
+    if (required_free > effective_free) {
+        if (getenv("DS4_CUDA_WEIGHT_CACHE_VERBOSE") != NULL) {
+            fprintf(stderr,
+                    "ds4: CUDA Q4 attn_q_b transient F16 preflight skipped: "
+                    "need %.2f MiB scratch + %.2f GiB reserve + %.2f GiB "
+                    "future sessions, only %.2f GiB currently free\n",
+                    (double)scratch_bytes / 1048576.0,
+                    (double)reserve / 1073741824.0,
+                    (double)working_set_reserve_bytes / 1073741824.0,
+                    (double)effective_free / 1073741824.0);
+        }
+        if (previous_device >= 0) (void)cudaSetDevice(previous_device);
+        return 0;
+    }
+
+    /* Allocate first so a failed grow cannot invalidate the capacity published
+     * by the current generation. The conservative admission above intentionally
+     * does not count the old arena as reclaimable. */
+    __half *scratch = NULL;
+    if (cudaMalloc((void **)&scratch, (size_t)scratch_bytes) != cudaSuccess) {
+        (void)cudaGetLastError();
+        if (previous_device >= 0) (void)cudaSetDevice(previous_device);
+        return 0;
+    }
+
+    /* Any old arena can still be referenced by stream work queued by a previous
+     * command batch. Drain before replacing it; this synchronization happens in
+     * prompt-aware preflight, never in the timed layer loop. */
+    if (cudaDeviceSynchronize() != cudaSuccess) {
+        (void)cudaGetLastError();
+        (void)cudaFree(scratch);
+        if (previous_device >= 0) (void)cudaSetDevice(previous_device);
+        return 0;
+    }
+    if (g_q4_attn_q_b_transient_f16_scratch) {
+        if (cudaFree(g_q4_attn_q_b_transient_f16_scratch) != cudaSuccess) {
+            (void)cudaGetLastError();
+            (void)cudaFree(scratch);
+            if (previous_device >= 0) (void)cudaSetDevice(previous_device);
+            return 0;
+        }
+    }
+    g_q4_attn_q_b_transient_f16_scratch = scratch;
+    g_q4_attn_q_b_transient_f16_scratch_bytes = scratch_bytes;
+    g_q4_attn_q_b_transient_f16_scratch_device = target_device;
+    g_q4_attn_q_b_transient_f16_runtime_disabled = 0;
+    /* Session preflight caches the backend generation.  Capacity is part of
+     * readiness: after a grow, sessions prepared for a smaller prompt must
+     * revisit preflight before their next larger sync. */
+    {
+        std::lock_guard<std::mutex> cache_lock(
+            g_q4_attn_q_b_f16_cache_mutex);
+        if (++g_q4_attn_q_b_f16_generation == 0u) {
+            g_q4_attn_q_b_f16_generation = 1u;
+        }
+    }
+    if (prepared_bytes) *prepared_bytes = scratch_bytes;
+    if (previous_device >= 0) (void)cudaSetDevice(previous_device);
+    fprintf(stderr,
+            "ds4: CUDA prepared reusable Q4 attn_q_b transient F16 scratch "
+            "(%.2f MiB, max batch %u tokens, output %s)\n",
+            (double)scratch_bytes / 1048576.0, max_rows,
+            include_output ? "F16 diagnostic" : "F32");
+    return 1;
+}
+
+static int cuda_q4_attn_q_b_f16_key_equal(
+        const cuda_q4_attn_q_b_f16_range &entry,
+        const void *model_map,
+        uint64_t model_size,
+        uint64_t weight_offset,
+        uint64_t weight_bytes,
+        uint64_t in_dim,
+        uint64_t out_dim,
+        int device_id) {
+    return entry.host_base == model_map &&
+           entry.model_size == model_size &&
+           entry.offset == weight_offset &&
+           entry.weight_bytes == weight_bytes &&
+           entry.in_dim == in_dim &&
+           entry.out_dim == out_dim &&
+           entry.device_id == device_id;
+}
+
+/* Caller holds g_q4_attn_q_b_f16_cache_mutex. */
+static const __half *cuda_q4_attn_q_b_f16_cache_lookup_locked(
+        const void *model_map,
+        uint64_t model_size,
+        uint64_t weight_offset,
+        uint64_t weight_bytes,
+        uint64_t in_dim,
+        uint64_t out_dim,
+        int device_id) {
+    for (const cuda_q4_attn_q_b_f16_range &entry :
+         g_q4_attn_q_b_f16_ranges) {
+        if (cuda_q4_attn_q_b_f16_key_equal(
+                entry, model_map, model_size, weight_offset, weight_bytes,
+                in_dim, out_dim, device_id)) {
+            return entry.device_ptr;
+        }
+    }
+    return NULL;
+}
+
+static int cuda_q4_attn_q_b_f16_multi_model_policy_active(void) {
+    std::lock_guard<std::mutex> lock(g_q4_attn_q_b_f16_cache_mutex);
+    return g_q4_attn_q_b_f16_multi_model_active;
+}
+
+static void cuda_q4_attn_q_b_f16_set_multi_model_policy(int active) {
+    std::lock_guard<std::mutex> lock(g_q4_attn_q_b_f16_cache_mutex);
+    g_q4_attn_q_b_f16_multi_model_active = active ? 1 : 0;
+}
+
+extern "C" uint64_t ds4_gpu_q4_attn_q_b_f16_cache_generation(void) {
+    std::lock_guard<std::mutex> lock(g_q4_attn_q_b_f16_cache_mutex);
+    return g_q4_attn_q_b_f16_generation;
+}
+
+/* Release implementation for callers that already own the build mutex.
+ * Maintaining the global build->cache lock order lets make_room cover its
+ * check and eviction atomically without recursively taking build_mutex. */
+static int cuda_q4_attn_q_b_f16_release_under_build_lock(void) {
+    std::lock_guard<std::mutex> cache_lock(g_q4_attn_q_b_f16_cache_mutex);
+
+    int previous_device = -1;
+    (void)cudaGetDevice(&previous_device);
+    int synchronized_device = INT_MIN;
+    for (const cuda_q4_attn_q_b_f16_range &entry :
+         g_q4_attn_q_b_f16_ranges) {
+        if (!entry.device_ptr || entry.device_id == synchronized_device) {
+            continue;
+        }
+        if (cudaSetDevice(entry.device_id) != cudaSuccess ||
+            cudaDeviceSynchronize() != cudaSuccess) {
+            (void)cudaGetLastError();
+            g_q4_attn_q_b_f16_hard_disabled = 1;
+            g_q4_attn_q_b_f16_dispatch_disabled = 1;
+            g_q4_attn_q_b_f16_pending_evict = 1;
+            if (previous_device >= 0) (void)cudaSetDevice(previous_device);
+            return 0;
+        }
+        synchronized_device = entry.device_id;
+    }
+
+    int ok = 1;
+    size_t keep = 0;
+    uint64_t remaining_bytes = 0;
+    const size_t original_count = g_q4_attn_q_b_f16_ranges.size();
+    for (size_t i = 0; i < original_count; i++) {
+        const cuda_q4_attn_q_b_f16_range entry =
+            g_q4_attn_q_b_f16_ranges[i];
+        int freed = entry.device_ptr == NULL;
+        if (!freed && cudaSetDevice(entry.device_id) == cudaSuccess &&
+            cudaFree(entry.device_ptr) == cudaSuccess) {
+            freed = 1;
+        } else if (!freed) {
+            (void)cudaGetLastError();
+            ok = 0;
+        }
+        if (!freed) {
+            if (keep != i) g_q4_attn_q_b_f16_ranges[keep] = entry;
+            keep++;
+            remaining_bytes += entry.f16_bytes;
+        }
+    }
+    if (previous_device >= 0) (void)cudaSetDevice(previous_device);
+    g_q4_attn_q_b_f16_ranges.resize(keep);
+    g_q4_attn_q_b_f16_bytes = remaining_bytes;
+    if (!ok) {
+        /* Successful frees are removed immediately, so a later retry cannot
+         * double-free them.  Failed allocations retain their full key and
+         * byte metadata until the retry succeeds. */
+        g_q4_attn_q_b_f16_hard_disabled = 1;
+        g_q4_attn_q_b_f16_dispatch_disabled = 1;
+        g_q4_attn_q_b_f16_pending_evict = 1;
+        if (keep != original_count) {
+            if (++g_q4_attn_q_b_f16_generation == 0u) {
+                g_q4_attn_q_b_f16_generation = 1u;
+            }
+        }
+        return 0;
+    }
+    g_q4_attn_q_b_f16_hard_disabled = 0;
+    g_q4_attn_q_b_f16_dispatch_disabled = 0;
+    g_q4_attn_q_b_f16_pending_evict = 0;
+    if (++g_q4_attn_q_b_f16_generation == 0u) {
+        g_q4_attn_q_b_f16_generation = 1u;
+    }
+    return ok;
+}
+
+extern "C" int ds4_gpu_release_q4_attn_q_b_f16_sidecars(void) {
+    std::lock_guard<std::mutex> build_lock(g_q4_attn_q_b_f16_build_mutex);
+    return cuda_q4_attn_q_b_f16_release_under_build_lock();
+}
+
+/* Called after a persistent fast-path failure while both use locks are held.
+ * Block future dispatch/build attempts, drop cache then scratch ownership, and
+ * only then take build_mutex for synchronous eviction. This preserves the
+ * global build -> scratch -> cache lifecycle order. Re-arm the circuit breaker
+ * after eviction so this session cannot rebuild and repeat the same failure;
+ * an explicit lifecycle release/make_room resets it later. */
+static void cuda_q4_attn_q_b_f16_runtime_failure_evict(
+        std::unique_lock<std::mutex> *cache_use_lock,
+        std::unique_lock<std::mutex> *scratch_use_lock) {
+    g_q4_attn_q_b_f16_hard_disabled = 1;
+    g_q4_attn_q_b_f16_dispatch_disabled = 1;
+    g_q4_attn_q_b_f16_pending_evict = 1;
+    cache_use_lock->unlock();
+    scratch_use_lock->unlock();
+
+    std::lock_guard<std::mutex> build_lock(
+        g_q4_attn_q_b_f16_build_mutex);
+    const int released =
+        cuda_q4_attn_q_b_f16_release_under_build_lock();
+    {
+        std::lock_guard<std::mutex> cache_lock(
+            g_q4_attn_q_b_f16_cache_mutex);
+        g_q4_attn_q_b_f16_hard_disabled = 1;
+        g_q4_attn_q_b_f16_dispatch_disabled = 1;
+        g_q4_attn_q_b_f16_pending_evict = released ? 0 : 1;
+    }
+    if (!released) {
+        fprintf(stderr,
+                "ds4: CUDA Q4 attn_q_b sidecar eviction deferred until "
+                "the next successful synchronization point\n");
+    }
+}
+
+static int cuda_q4_attn_q_b_f16_consume_pending_evict(void) {
+    int pending = 0;
+    {
+        std::lock_guard<std::mutex> cache_lock(
+            g_q4_attn_q_b_f16_cache_mutex);
+        pending = g_q4_attn_q_b_f16_pending_evict;
+    }
+    if (!pending) return 1;
+    std::lock_guard<std::mutex> build_lock(
+        g_q4_attn_q_b_f16_build_mutex);
+    const int released =
+        cuda_q4_attn_q_b_f16_release_under_build_lock();
+    {
+        std::lock_guard<std::mutex> cache_lock(
+            g_q4_attn_q_b_f16_cache_mutex);
+        g_q4_attn_q_b_f16_hard_disabled = 1;
+        g_q4_attn_q_b_f16_dispatch_disabled = 1;
+        g_q4_attn_q_b_f16_pending_evict = released ? 0 : 1;
+    }
+    return released;
+}
+
+extern "C" int ds4_gpu_make_room_for_q4_attn_q_b_f16_session(void) {
+    std::lock_guard<std::mutex> build_lock(g_q4_attn_q_b_f16_build_mutex);
+    if (!cuda_q4_attn_q_b_transient_f16_scratch_release()) {
+        return 0;
+    }
+    uint64_t bytes = 0;
+    int needs_reset = 0;
+    {
+        std::lock_guard<std::mutex> lock(g_q4_attn_q_b_f16_cache_mutex);
+        bytes = g_q4_attn_q_b_f16_bytes;
+        needs_reset = bytes != 0u || g_q4_attn_q_b_f16_hard_disabled ||
+                      g_q4_attn_q_b_f16_dispatch_disabled ||
+                      g_q4_attn_q_b_f16_pending_evict;
+    }
+    if (!needs_reset) return 1;
+    if (bytes != 0u) {
+        fprintf(stderr,
+                "ds4: evicting %.2f GiB of CUDA Q4 attn_q_b F16 sidecars "
+                "before allocating another live session\n",
+                (double)bytes / 1073741824.0);
+    }
+    return cuda_q4_attn_q_b_f16_release_under_build_lock();
+}
+
+extern "C" int ds4_gpu_prepare_q4_attn_q_b_f16_sidecars(
+        const void *model_map,
+        uint64_t model_size,
+        const ds4_gpu_q4_attn_q_b_f16_sidecar_desc *descs,
+        uint32_t count,
+        uint32_t max_prefill_rows,
+        uint64_t working_set_reserve_bytes,
+        uint64_t *prepared_bytes) {
+    if (prepared_bytes) *prepared_bytes = 0;
+    const int required = cuda_q4_attn_q_b_f16_required();
+    const int persistent_requested =
+        cuda_q4_attn_q_b_f16_requested();
+    const int persistent_disabled =
+        cuda_q4_attn_q_b_f16_disabled();
+    const int persistent_selected =
+        required || (persistent_requested && !persistent_disabled);
+    const uint32_t persistent_min_tokens =
+        cuda_q4_attn_q_b_f16_min_tokens();
+    /* An explicit persistent selection owns its configured candidate domain.
+     * Below that domain, preserve native Q4 exactly like Metal and ROCm.
+     * A non-strict ENABLE+DISABLE pair cancels the selection and may still
+     * use the independent automatic transient policy. */
+    if (persistent_selected &&
+        max_prefill_rows < persistent_min_tokens) {
+        return 0;
+    }
+    const int persistent_candidate =
+        persistent_selected &&
+        max_prefill_rows >= persistent_min_tokens;
+    /* REQUIRE is strict only for batches that are actually eligible for the
+     * persistent cache.  A long transient-only batch below CACHE_MIN_TOKENS
+     * must retain the same native-Q4 fallback as Metal and ROCm. */
+    const int strict = required && persistent_candidate;
+    const int transient_candidate =
+        !cuda_q4_attn_q_b_transient_f16_disabled() &&
+        max_prefill_rows >=
+            cuda_q4_attn_q_b_transient_f16_min_tokens();
+    if (!persistent_candidate && !transient_candidate) {
+        return 0;
+    }
+    if ((strict && persistent_disabled) ||
+        g_ssd_streaming_mode || g_quality_mode || g_n_gpus != 1 ||
+        !g_cublas_ready ||
+        !g_gpu[0].cublas_ready || !g_gpu[0].cublas ||
+        cuda_q4_attn_q_b_f16_multi_model_policy_active() ||
+        g_decode_graph_capturing ||
+        !model_map || !descs || count == 0u ||
+        count > CUDA_Q4_ATTN_Q_B_MAX_ENTRIES) {
+        return strict ? -1 : 0;
+    }
+
+    const uint64_t expected_weight_bytes =
+        (uint64_t)CUDA_Q4_ATTN_Q_B_OUT_DIM *
+        (CUDA_Q4_ATTN_Q_B_IN_DIM / CUDA_QK_K) *
+        sizeof(cuda_block_q4_K);
+    const uint64_t one_f16_bytes =
+        (uint64_t)CUDA_Q4_ATTN_Q_B_IN_DIM *
+        CUDA_Q4_ATTN_Q_B_OUT_DIM * sizeof(__half);
+    for (uint32_t i = 0; i < count; i++) {
+        const ds4_gpu_q4_attn_q_b_f16_sidecar_desc &desc = descs[i];
+        if (desc.weight_type != CUDA_Q4_ATTN_Q_B_TYPE ||
+            desc.in_dim != CUDA_Q4_ATTN_Q_B_IN_DIM ||
+            desc.out_dim != CUDA_Q4_ATTN_Q_B_OUT_DIM ||
+            desc.weight_bytes != expected_weight_bytes ||
+            desc.weight_offset > model_size ||
+            desc.weight_bytes > model_size - desc.weight_offset) {
+            return strict ? -1 : 0;
+        }
+    }
+
+    /* Serialize scratch capacity publication with sidecar construction and
+     * lifecycle eviction. The global order is build -> scratch -> cache. */
+    std::lock_guard<std::mutex> build_lock(
+        g_q4_attn_q_b_f16_build_mutex);
+
+    /* Without an explicit resident-cache request, the default long-prefill
+     * specialization retains only one expanded matrix. Verify every source in
+     * the already-owned device image now; the automatic path never populates a
+     * range cache or registers host pages. DISABLE_CACHE rolls an optional
+     * persistent request over to this path; REQUIRE remains strict only for
+     * persistent-cache candidates. */
+    const int use_persistent =
+        persistent_candidate && !persistent_disabled;
+    if (!use_persistent) {
+        if (!transient_candidate) return strict ? -1 : 0;
+        int previous_device = -1;
+        if (cudaGetDevice(&previous_device) != cudaSuccess ||
+            cudaSetDevice(g_gpu[0].device_id) != cudaSuccess) {
+            (void)cudaGetLastError();
+            if (previous_device >= 0) (void)cudaSetDevice(previous_device);
+            return strict ? -1 : 0;
+        }
+        int resident =
+            model_map == g_model_host_base &&
+            g_model_device_owned && g_model_device_base;
+        for (uint32_t i = 0; i < count; i++) {
+            if (!resident) break;
+            const char *source =
+                g_model_device_base + descs[i].weight_offset;
+            if (!cuda_q4_attn_q_b_source_is_device_resident(
+                    source, g_gpu[0].device_id)) {
+                resident = 0;
+                break;
+            }
+        }
+        if (previous_device >= 0) (void)cudaSetDevice(previous_device);
+        if (!resident ||
+            !cuda_q4_attn_q_b_transient_f16_scratch_ensure(
+                max_prefill_rows, working_set_reserve_bytes,
+                prepared_bytes)) {
+            return strict ? -1 : 0;
+        }
+        return 1;
+    }
+
+    /* The persistent path also stages its F16 RHS in the dedicated arena. */
+    if (!cuda_q4_attn_q_b_transient_f16_scratch_ensure(
+            max_prefill_rows, working_set_reserve_bytes, prepared_bytes)) {
+        return strict ? -1 : 0;
+    }
+
+    uint32_t miss_indices[CUDA_Q4_ATTN_Q_B_MAX_ENTRIES];
+    uint32_t miss_count = 0;
+    uint64_t missing_bytes = 0;
+    {
+        std::lock_guard<std::mutex> cache_lock(
+            g_q4_attn_q_b_f16_cache_mutex);
+        if (g_q4_attn_q_b_f16_multi_model_active) {
+            return strict ? -1 : 0;
+        }
+        /* One cache generation belongs to one mmap identity.  Do not let a
+         * second live model consume the remaining budget or poison READY
+         * entries if its cold build fails; the session lifecycle calls
+         * make_room/release before changing ownership. */
+        if (!g_q4_attn_q_b_f16_ranges.empty() &&
+            (g_q4_attn_q_b_f16_ranges[0].host_base != model_map ||
+             g_q4_attn_q_b_f16_ranges[0].model_size != model_size)) {
+            return strict ? -1 : 0;
+        }
+        for (uint32_t i = 0; i < count; i++) {
+            int found = 0;
+            for (const cuda_q4_attn_q_b_f16_range &entry :
+                 g_q4_attn_q_b_f16_ranges) {
+                if (cuda_q4_attn_q_b_f16_key_equal(
+                        entry, model_map, model_size,
+                        descs[i].weight_offset, descs[i].weight_bytes,
+                        descs[i].in_dim, descs[i].out_dim,
+                        g_gpu[0].device_id)) {
+                    found = 1;
+                    break;
+                }
+            }
+            /* Shared/aliased tensors need only one sidecar.  Earlier
+             * descriptors in this same transaction are future READY keys,
+             * even though publication intentionally happens after every
+             * allocation and dequantization succeeds. */
+            for (uint32_t j = 0; !found && j < i; j++) {
+                if (descs[j].weight_offset == descs[i].weight_offset &&
+                    descs[j].weight_bytes == descs[i].weight_bytes &&
+                    descs[j].in_dim == descs[i].in_dim &&
+                    descs[j].out_dim == descs[i].out_dim) {
+                    found = 1;
+                }
+            }
+            if (!found) {
+                miss_indices[miss_count++] = i;
+                if (UINT64_MAX - missing_bytes < one_f16_bytes) {
+                    return strict ? -1 : 0;
+                }
+                missing_bytes += one_f16_bytes;
+            }
+        }
+        if (miss_count != 0u) {
+            if (g_q4_attn_q_b_f16_hard_disabled) {
+                return strict ? -1 : 0;
+            }
+            const uint64_t limit =
+                cuda_q4_attn_q_b_f16_cache_limit_bytes();
+            if (g_q4_attn_q_b_f16_ranges.size() >
+                    CUDA_Q4_ATTN_Q_B_MAX_ENTRIES - miss_count ||
+                g_q4_attn_q_b_f16_bytes > limit ||
+                missing_bytes > limit - g_q4_attn_q_b_f16_bytes) {
+                return strict ? -1 : 0;
+            }
+        }
+    }
+    if (miss_count == 0u) return 1;
+
+    int previous_device = -1;
+    if (cudaGetDevice(&previous_device) != cudaSuccess) {
+        (void)cudaGetLastError();
+        return strict ? -1 : 0;
+    }
+    if (cudaSetDevice(g_gpu[0].device_id) != cudaSuccess) {
+        (void)cudaGetLastError();
+        if (previous_device >= 0) (void)cudaSetDevice(previous_device);
+        return strict ? -1 : 0;
+    }
+
+    const cuda_block_q4_K *sources[CUDA_Q4_ATTN_Q_B_MAX_ENTRIES] = {};
+    __half *sidecars[CUDA_Q4_ATTN_Q_B_MAX_ENTRIES] = {};
+    int build_failed = 0;
+    for (uint32_t mi = 0; mi < miss_count; mi++) {
+        const ds4_gpu_q4_attn_q_b_f16_sidecar_desc &desc =
+            descs[miss_indices[mi]];
+        const char *source = cuda_model_range_ptr(
+            model_map, desc.weight_offset, desc.weight_bytes,
+            "Q4 attn_q_b sidecar source");
+        /* cuda_model_range_ptr is the backend's GPU-addressability contract.
+         * Its result may be device memory, managed/HMM memory, or a mapped
+         * host pointer on coherent systems; all are valid for this one-shot
+         * prewarm kernel. */
+        if (!source) {
+            build_failed = 1;
+            break;
+        }
+        sources[mi] = reinterpret_cast<const cuda_block_q4_K *>(source);
+    }
+
+    size_t free_bytes = 0;
+    size_t total_bytes = 0;
+    if (!build_failed && cudaMemGetInfo(&free_bytes, &total_bytes) != cudaSuccess) {
+        (void)cudaGetLastError();
+        build_failed = 1;
+    }
+    if (!build_failed) {
+        const uint64_t reserve =
+            cuda_q4_attn_q_b_f16_reserve_bytes((uint64_t)total_bytes);
+        uint64_t required_free = missing_bytes;
+        if (UINT64_MAX - required_free < reserve) {
+            build_failed = 1;
+        } else {
+            required_free += reserve;
+        }
+        if (!build_failed &&
+            UINT64_MAX - required_free < working_set_reserve_bytes) {
+            build_failed = 1;
+        } else if (!build_failed) {
+            required_free += working_set_reserve_bytes;
+            if (required_free > (uint64_t)free_bytes) {
+                if (getenv("DS4_CUDA_WEIGHT_CACHE_VERBOSE") != NULL || strict) {
+                    fprintf(stderr,
+                            "ds4: CUDA Q4 attn_q_b F16 prewarm skipped: "
+                            "need %.2f GiB sidecars + %.2f GiB reserve + "
+                            "%.2f GiB future sessions, only %.2f GiB free\n",
+                            (double)missing_bytes / 1073741824.0,
+                            (double)reserve / 1073741824.0,
+                            (double)working_set_reserve_bytes / 1073741824.0,
+                            (double)free_bytes / 1073741824.0);
+                }
+                if (previous_device >= 0) (void)cudaSetDevice(previous_device);
+                return strict ? -1 : 0;
+            }
+        }
+    }
+
+    if (!build_failed) {
+        for (uint32_t mi = 0; mi < miss_count; mi++) {
+            if (cudaMalloc((void **)&sidecars[mi], (size_t)one_f16_bytes) !=
+                cudaSuccess) {
+                (void)cudaGetLastError();
+                build_failed = 1;
+                break;
+            }
+        }
+    }
+    if (!build_failed) {
+        const uint64_t chunks =
+            (uint64_t)CUDA_Q4_ATTN_Q_B_IN_DIM *
+            CUDA_Q4_ATTN_Q_B_OUT_DIM / 16u;
+        for (uint32_t mi = 0; mi < miss_count; mi++) {
+            dequant_q4_K_to_f16_kernel<<<
+                (unsigned)((chunks + 255u) / 256u), 256>>>(
+                    sidecars[mi], sources[mi],
+                    CUDA_Q4_ATTN_Q_B_IN_DIM,
+                    CUDA_Q4_ATTN_Q_B_OUT_DIM,
+                    CUDA_Q4_ATTN_Q_B_IN_DIM / CUDA_QK_K);
+            if (cudaGetLastError() != cudaSuccess) {
+                build_failed = 1;
+                break;
+            }
+        }
+        if (cudaDeviceSynchronize() != cudaSuccess) {
+            (void)cudaGetLastError();
+            build_failed = 1;
+        }
+    }
+
+    if (build_failed) {
+        (void)cudaDeviceSynchronize();
+        for (uint32_t mi = 0; mi < miss_count; mi++) {
+            if (sidecars[mi]) (void)cudaFree(sidecars[mi]);
+        }
+        {
+            std::lock_guard<std::mutex> cache_lock(
+                g_q4_attn_q_b_f16_cache_mutex);
+            g_q4_attn_q_b_f16_hard_disabled = 1;
+        }
+        if (previous_device >= 0) (void)cudaSetDevice(previous_device);
+        return strict ? -1 : 0;
+    }
+
+    {
+        std::lock_guard<std::mutex> cache_lock(
+            g_q4_attn_q_b_f16_cache_mutex);
+        for (uint32_t mi = 0; mi < miss_count; mi++) {
+            const ds4_gpu_q4_attn_q_b_f16_sidecar_desc &desc =
+                descs[miss_indices[mi]];
+            g_q4_attn_q_b_f16_ranges.push_back({
+                model_map, model_size, desc.weight_offset,
+                desc.weight_bytes, desc.in_dim, desc.out_dim,
+                one_f16_bytes, sidecars[mi], g_gpu[0].device_id,
+            });
+        }
+        g_q4_attn_q_b_f16_bytes += missing_bytes;
+    }
+    if (previous_device >= 0) (void)cudaSetDevice(previous_device);
+    if (prepared_bytes) *prepared_bytes = missing_bytes;
+    fprintf(stderr,
+            "ds4: CUDA prewarmed %u resident Q4 attn_q_b F16 sidecars "
+            "(%.2f GiB, min batch %u tokens)\n",
+            miss_count, (double)missing_bytes / 1073741824.0,
+            cuda_q4_attn_q_b_f16_min_tokens());
+    return 1;
+}
+
 static int cuda_q8_f32_cache_allowed(const char *label, uint64_t in_dim, uint64_t out_dim) {
     if (g_q8_cache_suppressed) return 0;
     if (getenv("DS4_CUDA_NO_Q8_F32_CACHE") != NULL) return 0;
@@ -1625,6 +3434,18 @@ static const __half *cuda_q8_f16_ptr(
                 return r.device_ptr;
             }
         }
+        /* Two GGUFs can legitimately reuse the same file offset. The fast
+         * offset index names only one of them, so fall back to the full key. */
+        for (const cuda_q8_f16_range &r : g_q8_f16_ranges) {
+            if (r.host_base == model_map &&
+                r.offset == offset &&
+                r.weight_bytes == weight_bytes &&
+                r.in_dim == in_dim &&
+                r.out_dim == out_dim &&
+                r.device_id == expected_device) {
+                return r.device_ptr;
+            }
+        }
     } else {
         for (const cuda_q8_f16_range &r : g_q8_f16_ranges) {
             if (r.host_base == model_map &&
@@ -1740,6 +3561,16 @@ static float *cuda_q8_f32_ptr(
                 return r.device_ptr;
             }
         }
+        for (const cuda_q8_f32_range &r : g_q8_f32_ranges) {
+            if (r.host_base == model_map &&
+                r.offset == offset &&
+                r.weight_bytes == weight_bytes &&
+                r.in_dim == in_dim &&
+                r.out_dim == out_dim &&
+                r.device_id == expected_device) {
+                return r.device_ptr;
+            }
+        }
     } else {
         for (const cuda_q8_f32_range &r : g_q8_f32_ranges) {
             if (r.host_base == model_map &&
@@ -2031,15 +3862,60 @@ static uint64_t cuda_round_up(uint64_t v, uint64_t align) {
     return rem == 0 ? v : v + (align - rem);
 }
 
-static void *cuda_align_ptr(void *ptr, uint64_t align) {
-    if (align <= 1) return ptr;
-    uintptr_t p = (uintptr_t)ptr;
-    uintptr_t a = (uintptr_t)align;
-    return (void *)(((p + a - 1u) / a) * a);
+/* The aligned pointer can advance by align-1 bytes from the allocation base.
+ * Allocate that prefix explicitly and validate every representation boundary
+ * before exposing the requested usable span to O_DIRECT or CUDA DMA. */
+static int cuda_host_stage_allocation_bytes(uint64_t usable_bytes,
+                                            uint64_t align,
+                                            size_t *allocation_bytes) {
+    if (!allocation_bytes || usable_bytes == 0) return 0;
+    const uint64_t a = align > 1 ? align : 1;
+    const uint64_t extra = a - 1u;
+    if (a > (uint64_t)UINTPTR_MAX ||
+        usable_bytes > (uint64_t)SIZE_MAX ||
+        extra > (uint64_t)SIZE_MAX - usable_bytes) {
+        return 0;
+    }
+    *allocation_bytes = (size_t)(usable_bytes + extra);
+    return 1;
 }
 
-static int cuda_model_stage_pool_alloc(uint64_t bytes) {
-    if (g_model_stage_bytes >= bytes) return 1;
+static int cuda_host_stage_bytes_for_chunk(uint64_t chunk, uint64_t align,
+                                           uint64_t *usable_bytes) {
+    if (!usable_bytes || chunk == 0) return 0;
+    const uint64_t a = align > 1 ? align : 1;
+    if (chunk > UINT64_MAX - a || chunk + a > (uint64_t)SIZE_MAX) {
+        return 0;
+    }
+    *usable_bytes = chunk + a;
+    return 1;
+}
+
+static int cuda_host_stage_aligned_view(void *raw,
+                                        size_t allocation_bytes,
+                                        uint64_t usable_bytes,
+                                        uint64_t align,
+                                        void **aligned_out) {
+    if (!raw || !aligned_out || usable_bytes == 0 ||
+        usable_bytes > (uint64_t)allocation_bytes) {
+        return 0;
+    }
+    const uint64_t align64 = align > 1 ? align : 1;
+    if (align64 > (uint64_t)UINTPTR_MAX) return 0;
+    const uintptr_t a = (uintptr_t)align64;
+    if (a == 0) return 0;
+    const uintptr_t p = (uintptr_t)raw;
+    const uintptr_t rem = p % a;
+    const size_t delta = rem == 0 ? 0u : (size_t)(a - rem);
+    if (delta > allocation_bytes ||
+        usable_bytes > (uint64_t)(allocation_bytes - delta)) {
+        return 0;
+    }
+    *aligned_out = (char *)raw + delta;
+    return 1;
+}
+
+static void cuda_model_stage_pool_release(void) {
     for (size_t i = 0; i < 4; i++) {
         if (g_model_stage_event[i]) {
             (void)cudaEventDestroy(g_model_stage_event[i]);
@@ -2052,6 +3928,20 @@ static int cuda_model_stage_pool_alloc(uint64_t bytes) {
         }
     }
     g_model_stage_bytes = 0;
+    g_model_stage_align = 1;
+}
+
+static int cuda_model_stage_pool_alloc(uint64_t bytes) {
+    const uint64_t align = g_model_direct_align > 1 ?
+                           g_model_direct_align : 1;
+    size_t allocation_bytes = 0;
+    if (!cuda_host_stage_allocation_bytes(bytes, align,
+                                          &allocation_bytes)) {
+        return 0;
+    }
+    if (g_model_stage_bytes >= bytes &&
+        g_model_stage_align == align) return 1;
+    cuda_model_stage_pool_release();
     if (!g_model_upload_stream) {
         cudaError_t err = cudaStreamCreateWithFlags(&g_model_upload_stream, cudaStreamNonBlocking);
         if (err != cudaSuccess) {
@@ -2061,24 +3951,52 @@ static int cuda_model_stage_pool_alloc(uint64_t bytes) {
         }
     }
     for (size_t i = 0; i < 4; i++) {
-        cudaError_t err = cudaMallocHost(&g_model_stage_raw[i], (size_t)bytes);
+        cudaError_t err = cudaMallocHost(&g_model_stage_raw[i],
+                                         allocation_bytes);
         if (err != cudaSuccess) {
             fprintf(stderr, "ds4: CUDA pinned model staging allocation failed: %s\n", cudaGetErrorString(err));
             (void)cudaGetLastError();
+            cuda_model_stage_pool_release();
+            return 0;
+        }
+        if (!cuda_host_stage_aligned_view(g_model_stage_raw[i],
+                                          allocation_bytes, bytes, align,
+                                          &g_model_stage[i])) {
+            fprintf(stderr,
+                    "ds4: CUDA pinned model staging alignment rejected\n");
+            cuda_model_stage_pool_release();
             return 0;
         }
-        g_model_stage[i] = cuda_align_ptr(g_model_stage_raw[i], g_model_direct_align);
         err = cudaEventCreateWithFlags(&g_model_stage_event[i], cudaEventDisableTiming);
         if (err != cudaSuccess) {
             fprintf(stderr, "ds4: CUDA model staging event creation failed: %s\n", cudaGetErrorString(err));
             (void)cudaGetLastError();
+            cuda_model_stage_pool_release();
             return 0;
         }
     }
     g_model_stage_bytes = bytes;
+    g_model_stage_align = align;
     return 1;
 }
 
+/* A range load can fail after prior chunks were queued from the pinned ring.
+ * Drain before any caller can reuse or free that storage. */
+static int cuda_model_upload_fail(const char *what) {
+    if (g_model_upload_stream) {
+        const cudaError_t sync_err =
+            cudaStreamSynchronize(g_model_upload_stream);
+        if (sync_err != cudaSuccess) {
+            fprintf(stderr,
+                    "ds4: CUDA model upload abort sync failed for %s: %s\n",
+                    what ? what : "weights",
+                    cudaGetErrorString(sync_err));
+            (void)cudaGetLastError();
+        }
+    }
+    return 0;
+}
+
 static int cuda_pread_full(int fd, void *buf, uint64_t bytes, uint64_t offset) {
     uint64_t done = 0;
     while (done < bytes) {
@@ -2131,28 +4049,137 @@ static int cuda_model_stage_read(void *stage, uint64_t stage_bytes,
     return cuda_pread_full(g_model_fd, stage, bytes, offset);
 }
 
-static void cuda_stream_selected_stage_release(void) {
+static int cuda_stream_selected_upload_drain_checked(void) {
+    if (!g_stream_selected_upload_stream) return 1;
+    int previous_device = -1;
+    (void)cudaGetDevice(&previous_device);
+    if (g_stream_selected_upload_owner_device < 0 ||
+        cudaSetDevice(g_stream_selected_upload_owner_device) != cudaSuccess) {
+        (void)cudaGetLastError();
+        if (previous_device >= 0) (void)cudaSetDevice(previous_device);
+        return 0;
+    }
+    const cudaError_t err =
+        cudaStreamSynchronize(g_stream_selected_upload_stream);
+    if (previous_device >= 0) (void)cudaSetDevice(previous_device);
+    if (err != cudaSuccess) {
+        fprintf(stderr,
+                "ds4: CUDA compact-cache upload drain failed: %s\n",
+                cudaGetErrorString(err));
+        (void)cudaGetLastError();
+        return 0;
+    }
+    return 1;
+}
+
+static int cuda_stream_selected_stage_release_checked(void) {
+    const int had_upload_stream = g_stream_selected_upload_stream != NULL;
+    int previous_device = -1;
+    (void)cudaGetDevice(&previous_device);
+    if (had_upload_stream &&
+        (g_stream_selected_upload_owner_device < 0 ||
+         cudaSetDevice(g_stream_selected_upload_owner_device) !=
+             cudaSuccess)) {
+        (void)cudaGetLastError();
+        g_stream_selected_stage_poisoned = 1;
+        if (previous_device >= 0) (void)cudaSetDevice(previous_device);
+        return 0;
+    }
+    /* Once event publication is enabled, resize/teardown may arrive while
+     * the last H2D epoch is still in flight.  Drain before destroying the
+     * ring events or freeing their pinned payloads. */
+    if (g_stream_selected_upload_stream) {
+        if (g_stream_selected_stage_test_fail_drain) {
+            g_stream_selected_stage_test_fail_drain = 0;
+            g_stream_selected_stage_poisoned = 1;
+            if (previous_device >= 0) (void)cudaSetDevice(previous_device);
+            return 0;
+        }
+        const cudaError_t sync_err =
+            cudaStreamSynchronize(g_stream_selected_upload_stream);
+        if (sync_err != cudaSuccess) {
+            fprintf(stderr,
+                    "ds4: CUDA selected staging drain failed: %s\n",
+                    cudaGetErrorString(sync_err));
+            (void)cudaGetLastError();
+            g_stream_selected_stage_poisoned = 1;
+            if (previous_device >= 0) (void)cudaSetDevice(previous_device);
+            return 0;
+        }
+    }
+    int released = 1;
     for (size_t i = 0; i < 4; i++) {
         if (g_stream_selected_stage_event[i]) {
-            (void)cudaEventDestroy(g_stream_selected_stage_event[i]);
-            g_stream_selected_stage_event[i] = NULL;
+            if (cudaEventDestroy(g_stream_selected_stage_event[i]) ==
+                    cudaSuccess) {
+                g_stream_selected_stage_event[i] = NULL;
+            } else {
+                (void)cudaGetLastError();
+                released = 0;
+            }
         }
         if (g_stream_selected_stage_raw[i]) {
-            (void)cudaFreeHost(g_stream_selected_stage_raw[i]);
-            g_stream_selected_stage_raw[i] = NULL;
-            g_stream_selected_stage[i] = NULL;
+            if (cudaFreeHost(g_stream_selected_stage_raw[i]) ==
+                    cudaSuccess) {
+                g_stream_selected_stage_raw[i] = NULL;
+                g_stream_selected_stage[i] = NULL;
+            } else {
+                (void)cudaGetLastError();
+                released = 0;
+            }
         }
     }
-    g_stream_selected_stage_bytes = 0;
-    if (g_stream_selected_upload_stream) {
-        (void)cudaStreamDestroy(g_stream_selected_upload_stream);
+    if (g_stream_selected_remap_stage) {
+        if (cudaFreeHost(g_stream_selected_remap_stage) == cudaSuccess) {
+            g_stream_selected_remap_stage = NULL;
+            g_stream_selected_remap_stage_capacity = 0;
+        } else {
+            (void)cudaGetLastError();
+            released = 0;
+        }
+    }
+    if (released && g_stream_selected_upload_stream &&
+        cudaStreamDestroy(g_stream_selected_upload_stream) == cudaSuccess) {
         g_stream_selected_upload_stream = NULL;
+    } else if (released && g_stream_selected_upload_stream) {
+        (void)cudaGetLastError();
+        released = 0;
     }
+    if (!released) {
+        g_stream_selected_stage_poisoned = 1;
+        if (previous_device >= 0) (void)cudaSetDevice(previous_device);
+        return 0;
+    }
+    g_stream_selected_stage_bytes = 0;
+    g_stream_selected_stage_align = 1;
+    g_stream_selected_remap_stage_capacity = 0;
+    g_stream_selected_upload_owner_device = -1;
+    g_stream_selected_stage_poisoned = 0;
+    if (had_upload_stream) {
+        uint64_t value = ++g_stream_selected_upload_event_value;
+        if (value == 0) ++g_stream_selected_upload_event_value;
+    }
+    if (previous_device >= 0) (void)cudaSetDevice(previous_device);
+    return 1;
+}
+
+static void cuda_stream_selected_stage_release(void) {
+    (void)cuda_stream_selected_stage_release_checked();
 }
 
 static int cuda_stream_selected_stage_pool_alloc(uint64_t bytes) {
-    if (g_stream_selected_stage_bytes >= bytes) return 1;
-    cuda_stream_selected_stage_release();
+    const uint64_t align = g_model_direct_align > 1 ?
+                           g_model_direct_align : 1;
+    size_t allocation_bytes = 0;
+    if (!cuda_host_stage_allocation_bytes(bytes, align,
+                                          &allocation_bytes)) {
+        return 0;
+    }
+    if (!g_stream_selected_stage_poisoned &&
+        !g_stream_selected_stage_test_fail_drain &&
+        g_stream_selected_stage_bytes >= bytes &&
+        g_stream_selected_stage_align == align) return 1;
+    if (!cuda_stream_selected_stage_release_checked()) return 0;
     cudaError_t err = cudaStreamCreateWithFlags(
             &g_stream_selected_upload_stream, cudaStreamNonBlocking);
     if (err != cudaSuccess) {
@@ -2162,8 +4189,14 @@ static int cuda_stream_selected_stage_pool_alloc(uint64_t bytes) {
         (void)cudaGetLastError();
         return 0;
     }
+    if (cudaGetDevice(&g_stream_selected_upload_owner_device) != cudaSuccess) {
+        (void)cudaGetLastError();
+        cuda_stream_selected_stage_release();
+        return 0;
+    }
     for (size_t i = 0; i < 4; i++) {
-        err = cudaMallocHost(&g_stream_selected_stage_raw[i], (size_t)bytes);
+        err = cudaMallocHost(&g_stream_selected_stage_raw[i],
+                             allocation_bytes);
         if (err != cudaSuccess) {
             fprintf(stderr,
                     "ds4: CUDA streaming selected staging allocation failed: %s\n",
@@ -2172,8 +4205,14 @@ static int cuda_stream_selected_stage_pool_alloc(uint64_t bytes) {
             cuda_stream_selected_stage_release();
             return 0;
         }
-        g_stream_selected_stage[i] = cuda_align_ptr(
-                g_stream_selected_stage_raw[i], g_model_direct_align);
+        if (!cuda_host_stage_aligned_view(
+                g_stream_selected_stage_raw[i], allocation_bytes,
+                bytes, align, &g_stream_selected_stage[i])) {
+            fprintf(stderr,
+                    "ds4: CUDA streaming selected staging alignment rejected\n");
+            cuda_stream_selected_stage_release();
+            return 0;
+        }
         err = cudaEventCreateWithFlags(&g_stream_selected_stage_event[i],
                                        cudaEventDisableTiming);
         if (err != cudaSuccess) {
@@ -2186,373 +4225,3575 @@ static int cuda_stream_selected_stage_pool_alloc(uint64_t bytes) {
         }
     }
     g_stream_selected_stage_bytes = bytes;
+    g_stream_selected_stage_align = align;
     return 1;
 }
 
-static int cuda_model_copy_to_device_streamed(
-        char *dst,
-        const void *model_map,
-        uint64_t model_size,
-        uint64_t offset,
-        uint64_t bytes,
-        const char *what) {
-    if (!dst || !model_map || offset > model_size ||
-        bytes > model_size - offset) {
-        return 0;
-    }
-    if (bytes == 0) return 1;
-    if (g_model_fd < 0 ||
-        (g_model_fd_host_base != NULL && model_map != g_model_fd_host_base)) {
-        return cuda_ok(cudaMemcpy(dst,
-                                  (const char *)model_map + offset,
-                                  (size_t)bytes,
-                                  cudaMemcpyHostToDevice),
-                       what ? what : "stream selected expert copy");
+static int cuda_stream_selected_remap_stage_ensure(uint64_t count) {
+    if (count == 0 || count > SIZE_MAX / sizeof(int32_t)) return 0;
+    if (g_stream_selected_remap_stage &&
+        g_stream_selected_remap_stage_capacity >= count) {
+        return 1;
     }
-
-    const uint64_t chunk = cuda_model_copy_chunk_bytes();
-    const uint64_t stage_bytes =
-        chunk + (g_model_direct_align > 1 ? g_model_direct_align : 1);
-    if (!cuda_stream_selected_stage_pool_alloc(stage_bytes)) return 0;
-
-    uint64_t copied = 0;
-    uint64_t chunk_idx = 0;
-    while (copied < bytes) {
-        const uint64_t n = bytes - copied < chunk ? bytes - copied : chunk;
-        const uint64_t bi = chunk_idx % 4u;
-        cudaError_t err;
-        if (chunk_idx >= 4u) {
-            err = cudaEventSynchronize(g_stream_selected_stage_event[bi]);
-            if (err != cudaSuccess) {
-                fprintf(stderr,
-                        "ds4: CUDA streaming selected staging wait failed for %s: %s\n",
-                        what ? what : "expert", cudaGetErrorString(err));
-                (void)cudaGetLastError();
-                return 0;
-            }
-        }
-        const char *payload = NULL;
-        if (!cuda_model_stage_read(g_stream_selected_stage[bi],
-                                   g_stream_selected_stage_bytes,
-                                   offset + copied, n, &payload)) {
-            fprintf(stderr,
-                    "ds4: CUDA streaming selected read failed for %s at %.2f MiB: %s\n",
-                    what ? what : "expert", (double)copied / 1048576.0,
-                    strerror(errno));
-            return 0;
-        }
-        err = cudaMemcpyAsync(dst + copied, payload, (size_t)n,
-                              cudaMemcpyHostToDevice,
-                              g_stream_selected_upload_stream);
-        if (err != cudaSuccess) {
-            fprintf(stderr,
-                    "ds4: CUDA streaming selected copy failed for %s at %.2f MiB: %s\n",
-                    what ? what : "expert", (double)copied / 1048576.0,
-                    cudaGetErrorString(err));
+    /* The caller invokes this before submitting the new epoch.  The pool
+     * release drains an older epoch before replacing its pinned storage. */
+    if (g_stream_selected_remap_stage) {
+        if (g_stream_selected_upload_stream &&
+            cudaStreamSynchronize(g_stream_selected_upload_stream) !=
+                cudaSuccess) {
             (void)cudaGetLastError();
+            g_stream_selected_stage_poisoned = 1;
             return 0;
         }
-        err = cudaEventRecord(g_stream_selected_stage_event[bi],
-                              g_stream_selected_upload_stream);
-        if (err != cudaSuccess) {
-            fprintf(stderr,
-                    "ds4: CUDA streaming selected staging record failed for %s: %s\n",
-                    what ? what : "expert", cudaGetErrorString(err));
+        if (cudaFreeHost(g_stream_selected_remap_stage) != cudaSuccess) {
             (void)cudaGetLastError();
+            g_stream_selected_stage_poisoned = 1;
             return 0;
         }
-        cuda_model_drop_file_pages(offset + copied, n);
-        cuda_model_discard_source_pages(model_map, model_size,
-                                        offset + copied, n);
-        copied += n;
-        chunk_idx++;
+        g_stream_selected_remap_stage = NULL;
+        g_stream_selected_remap_stage_capacity = 0;
     }
-
-    const cudaError_t err =
-        cudaStreamSynchronize(g_stream_selected_upload_stream);
+    const size_t bytes = (size_t)count * sizeof(int32_t);
+    cudaError_t err = cudaMallocHost(
+            (void **)&g_stream_selected_remap_stage, bytes);
     if (err != cudaSuccess) {
         fprintf(stderr,
-                "ds4: CUDA streaming selected upload sync failed for %s: %s\n",
-                what ? what : "expert", cudaGetErrorString(err));
+                "ds4: CUDA selected remap staging allocation failed: %s\n",
+                cudaGetErrorString(err));
         (void)cudaGetLastError();
         return 0;
     }
+    g_stream_selected_remap_stage_capacity = count;
     return 1;
 }
 
-static uint64_t cuda_model_cache_limit_bytes(void) {
-    uint64_t gb = 0;
-    const char *env = getenv("DS4_CUDA_WEIGHT_CACHE_LIMIT_GB");
-    if (env && env[0]) {
-        char *end = NULL;
-        unsigned long long v = strtoull(env, &end, 10);
-        if (end != env) gb = (uint64_t)v;
-    }
-    if (gb == 0) return UINT64_MAX;
-    return gb * 1073741824ull;
+typedef struct {
+    char *dst;
+    uint64_t offset;
+    uint64_t bytes;
+    uint32_t ordinal;
+} cuda_stream_selected_copy_task;
+
+typedef struct {
+    uint32_t task_index;
+    uint64_t task_offset;
+    uint64_t offset;
+    uint64_t bytes;
+} cuda_stream_selected_copy_segment;
+
+typedef struct {
+    uint32_t segment_begin;
+    uint32_t segment_end;
+    uint64_t offset;
+    uint64_t bytes;
+} cuda_stream_selected_copy_group;
+
+struct ds4_cuda_stream_selected_batch_io_report;
+typedef struct ds4_cuda_stream_selected_batch_io_report
+    ds4_cuda_stream_selected_batch_io_report;
+typedef struct {
+    uint64_t candidates;
+    uint64_t attempts;
+    uint64_t completed;
+    uint64_t legacy_batches;
+    uint64_t safe_fallbacks;
+    uint64_t failures;
+    uint64_t required_failures;
+    uint64_t oracle_runs;
+    uint64_t oracle_failures;
+    uint64_t tasks;
+    uint64_t segments;
+    uint64_t reads;
+    uint64_t bytes;
+    int enabled;
+    int required;
+    int oracle;
+} cuda_stream_selected_batch_io_report_layout;
+
+static int cuda_stream_selected_env_value_enabled(const char *value) {
+    return cuda_env_value_enabled(value);
 }
 
-static uint64_t cuda_model_arena_chunk_bytes(uint64_t need) {
-    uint64_t mb = 1792;
-    const char *env = getenv("DS4_CUDA_WEIGHT_ARENA_CHUNK_MB");
-    if (env && env[0]) {
-        char *end = NULL;
-        unsigned long long v = strtoull(env, &end, 10);
-        if (end != env && v > 0) mb = (uint64_t)v;
-    }
-    if (mb < 256) mb = 256;
-    if (mb > 8192) mb = 8192;
-    uint64_t bytes = mb * 1048576ull;
-    if (bytes < need) {
-        const uint64_t align = 256ull * 1048576ull;
-        bytes = (need + align - 1u) & ~(align - 1u);
-    }
-    return bytes;
+static int cuda_stream_selected_env_flag(const char *name) {
+    return cuda_stream_selected_env_value_enabled(
+        name ? getenv(name) : NULL);
 }
 
-static char *cuda_model_arena_alloc(uint64_t bytes, const char *what) {
-    if (bytes == 0) return NULL;
-    if (g_model_cache_full) return NULL;
-    const uint64_t align = 256u;
-    const uint64_t aligned = (bytes + align - 1u) & ~(align - 1u);
+extern "C" int ds4_cuda_test_stream_selected_batch_env_value(
+        const char *value) {
+    return cuda_stream_selected_env_value_enabled(value);
+}
 
-    for (cuda_model_arena &a : g_model_arenas) {
-        const uint64_t used = (a.used + align - 1u) & ~(align - 1u);
-        if (used <= a.bytes && aligned <= a.bytes - used) {
-            char *ptr = a.device_ptr + used;
-            a.used = used + aligned;
-            return ptr;
-        }
-    }
+extern "C" int ds4_cuda_test_stream_selected_event_env_value(
+        const char *value) {
+    return cuda_stream_selected_env_value_enabled(value);
+}
 
-    const uint64_t limit = cuda_model_cache_limit_bytes();
-    if (g_model_range_bytes > limit || aligned > limit - g_model_range_bytes) return NULL;
+/* This tagged compatibility type is normally declared beside the CUDA
+ * streaming implementation below.  The phase-one host planner lives here so
+ * its value semantics and policy hooks can be tested before any CUDA arena is
+ * introduced. */
+typedef struct ds4_gpu_stream_expert_table {
+    const void *model_map;
+    uint64_t    model_size;
+    uint32_t    layer;
+    uint32_t    n_total_expert;
+    uint64_t    gate_offset;
+    uint64_t    up_offset;
+    uint64_t    down_offset;
+    uint64_t    gate_expert_bytes;
+    uint64_t    down_expert_bytes;
+} ds4_gpu_stream_expert_table;
 
-    const uint64_t chunk = cuda_model_arena_chunk_bytes(aligned);
-    void *dev = NULL;
-    cudaError_t err = cudaMalloc(&dev, (size_t)chunk);
-    if (err != cudaSuccess) {
-        fprintf(stderr, "ds4: CUDA model arena alloc failed for %s (%.2f MiB chunk): %s\n",
-                what ? what : "weights",
-                (double)chunk / 1048576.0,
-                cudaGetErrorString(err));
-        (void)cudaGetLastError();
-        g_model_cache_full = 1;
-        return NULL;
-    }
-    g_model_arenas.push_back({(char *)dev, chunk, aligned});
-    if (getenv("DS4_CUDA_WEIGHT_CACHE_VERBOSE")) {
-        uint64_t arena_bytes = 0;
-        for (const cuda_model_arena &a : g_model_arenas) arena_bytes += a.bytes;
-        fprintf(stderr, "ds4: CUDA model arena allocated %.2f MiB (arenas %.2f GiB)\n",
-                (double)chunk / 1048576.0,
-                (double)arena_bytes / 1073741824.0);
-    }
-    return (char *)dev;
-}
+struct ds4_cuda_stream_expert_persistent_report;
+typedef struct ds4_cuda_stream_expert_persistent_report
+    ds4_cuda_stream_expert_persistent_report;
+typedef struct {
+    uint64_t plan_attempts;
+    uint64_t plans_built;
+    uint64_t commits;
+    uint64_t rollbacks;
+    uint64_t hits;
+    uint64_t misses;
+    uint64_t duplicates;
+    uint64_t free_assignments;
+    uint64_t evictions;
+    uint64_t rejects;
+    uint64_t budget_rejects;
+    uint64_t class_rejects;
+    uint64_t protected_rejects;
+    uint64_t key_misses;
+    uint64_t overflow_rejects;
+    uint64_t oracle_runs;
+    uint64_t oracle_failures;
+    uint64_t arena_allocations;
+    uint64_t arena_reuses;
+    uint64_t arena_releases;
+    uint64_t arena_failures;
+    uint64_t arena_oracle_runs;
+    uint64_t arena_oracle_failures;
+    uint64_t epochs_attempted;
+    uint64_t epochs_published;
+    uint64_t all_hit_epochs;
+    uint64_t miss_epochs;
+    uint64_t miss_experts;
+    uint64_t weight_bytes_uploaded;
+    uint64_t remap_bytes_uploaded;
+    uint64_t upload_failures;
+    uint64_t fallbacks;
+    uint64_t slot_invalidations;
+    uint64_t poisons;
+    uint64_t persistent_dispatches;
+    uint64_t transient_dispatches;
+    uint64_t runtime_oracle_runs;
+    uint64_t runtime_oracle_failures;
+    int enabled;
+    int required;
+    int stats;
+    int oracle;
+} cuda_stream_expert_persistent_report_layout;
+
+/* Phase one of the CUDA resident-expert cache is deliberately host-only.
+ * It defines and tests the transaction which a later device arena will
+ * execute, without allocating storage or changing the transient cache path.
+ * An arena has one immutable byte-size class.  Full entry keys prevent a
+ * layer, model, or table-layout change from being mistaken for a hit. */
+typedef struct {
+    uint64_t gate_expert_bytes;
+    uint64_t down_expert_bytes;
+    uint64_t bytes_per_slot;
+    uint64_t arena_bytes;
+    uint32_t capacity;
+} cuda_stream_expert_persistent_class;
 
-static const char *cuda_model_range_ptr_from_fd(
-        const void *model_map,
-        uint64_t offset,
-        uint64_t bytes,
-        const char *what) {
-    if (g_model_fd < 0 || bytes == 0) return NULL;
-    if (g_model_fd_host_base != NULL && model_map != g_model_fd_host_base) return NULL;
-    const uint64_t limit = cuda_model_cache_limit_bytes();
-    if (g_model_range_bytes > limit || bytes > limit - g_model_range_bytes) {
-        if (getenv("DS4_CUDA_WEIGHT_CACHE_VERBOSE")) {
-            fprintf(stderr, "ds4: CUDA direct %s %.2f MiB (cache budget %.2f GiB exhausted)\n",
-                    what ? what : "weights",
-                    (double)bytes / 1048576.0,
-                    (double)limit / 1073741824.0);
-        }
-        return cuda_model_ptr(model_map, offset);
-    }
+typedef struct {
+    const void *model_map;
+    uint64_t model_size;
+    uint64_t gate_offset;
+    uint64_t up_offset;
+    uint64_t down_offset;
+    uint64_t gate_expert_bytes;
+    uint64_t down_expert_bytes;
+    uint32_t layer;
+    uint32_t n_total_expert;
+    uint32_t expert_id;
+} cuda_stream_expert_persistent_key;
 
-    char *dev = cuda_model_arena_alloc(bytes, what);
-    if (!dev) {
-        if (getenv("DS4_CUDA_STRICT_WEIGHT_CACHE") != NULL) return NULL;
-        return cuda_model_ptr(model_map, offset);
-    }
-    cudaError_t err = cudaSuccess;
+typedef struct {
+    cuda_stream_expert_persistent_key key;
+    uint64_t last_use;
+    uint32_t pin_count;
+    int valid;
+} cuda_stream_expert_persistent_entry;
 
-    const uint64_t chunk = cuda_model_copy_chunk_bytes();
-    const uint64_t stage_bytes = chunk + (g_model_direct_align > 1 ? g_model_direct_align : 1);
-    if (!cuda_model_stage_pool_alloc(stage_bytes)) return NULL;
+typedef struct {
+    cuda_stream_expert_persistent_class size_class;
+    std::vector<cuda_stream_expert_persistent_entry> slots;
+    /* Stored in reverse order so pop_back() deterministically yields the
+     * lowest free slot. */
+    std::vector<uint32_t> free_slots;
+    uint64_t lru_clock;
+} cuda_stream_expert_persistent_state;
 
-    uint64_t copied = 0;
-    uint64_t chunk_idx = 0;
-    while (copied < bytes) {
-        const uint64_t n = (bytes - copied < chunk) ? (bytes - copied) : chunk;
-        const uint64_t bi = chunk_idx % 4u;
-        if (chunk_idx >= 4u) {
-            err = cudaEventSynchronize(g_model_stage_event[bi]);
-            if (err != cudaSuccess) {
-                fprintf(stderr, "ds4: CUDA model staging wait failed for %s: %s\n",
-                        what ? what : "weights", cudaGetErrorString(err));
-                (void)cudaGetLastError();
-                return NULL;
-            }
-        }
-        const char *payload = NULL;
-        if (!cuda_model_stage_read(g_model_stage[bi], g_model_stage_bytes,
-                                   offset + copied, n, &payload)) {
-            fprintf(stderr, "ds4: CUDA model range read failed for %s at %.2f MiB: %s\n",
-                    what ? what : "weights",
-                    (double)copied / 1048576.0,
-                    strerror(errno));
-            return NULL;
-        }
-        err = cudaMemcpyAsync(dev + copied, payload, (size_t)n,
-                              cudaMemcpyHostToDevice, g_model_upload_stream);
-        if (err != cudaSuccess) {
-            fprintf(stderr, "ds4: CUDA model range copy failed for %s at %.2f MiB: %s\n",
-                    what ? what : "weights",
-                    (double)copied / 1048576.0,
-                    cudaGetErrorString(err));
-            (void)cudaGetLastError();
-            return NULL;
-        }
-        err = cudaEventRecord(g_model_stage_event[bi], g_model_upload_stream);
-        if (err != cudaSuccess) {
-            fprintf(stderr, "ds4: CUDA model staging record failed for %s: %s\n",
-                    what ? what : "weights", cudaGetErrorString(err));
-            (void)cudaGetLastError();
-            return NULL;
-        }
-        cuda_model_drop_file_pages(offset + copied, n);
-        cuda_model_discard_source_pages(model_map, g_model_registered_size, offset + copied, n);
-        copied += n;
-        cuda_model_load_progress_note(g_model_range_bytes + copied);
-        chunk_idx++;
+static cuda_stream_expert_persistent_state
+    g_stream_expert_persistent_state;
+
+typedef struct {
+    cuda_stream_expert_persistent_key key;
+    cuda_stream_expert_persistent_key victim;
+    uint32_t slot;
+    int had_victim;
+} cuda_stream_expert_persistent_load;
+
+typedef struct {
+    cuda_stream_expert_persistent_state next;
+    std::vector<uint32_t> remap;
+    std::vector<cuda_stream_expert_persistent_load> loads;
+    uint32_t slot_base;
+    uint32_t weight_domain;
+    uint32_t unique_count;
+    uint32_t hit_count;
+    uint32_t miss_count;
+    uint32_t duplicate_count;
+    int built;
+} cuda_stream_expert_persistent_plan;
+
+static int cuda_stream_expert_persistent_add_u64(
+        uint64_t a, uint64_t b, uint64_t *out) {
+    if (!out || a > UINT64_MAX - b) return 0;
+    *out = a + b;
+    return 1;
+}
+
+static int cuda_stream_expert_persistent_mul_u64(
+        uint64_t a, uint64_t b, uint64_t *out) {
+    if (!out || (a != 0 && b > UINT64_MAX / a)) return 0;
+    *out = a * b;
+    return 1;
+}
+
+static int cuda_stream_expert_persistent_class_make(
+        cuda_stream_expert_persistent_class *out,
+        uint32_t capacity,
+        uint64_t gate_expert_bytes,
+        uint64_t down_expert_bytes) {
+    if (!out || capacity == 0 || gate_expert_bytes == 0 ||
+        down_expert_bytes == 0) {
+        return 0;
     }
-    err = cudaStreamSynchronize(g_model_upload_stream);
-    if (err != cudaSuccess) {
-        fprintf(stderr, "ds4: CUDA model range upload sync failed for %s: %s\n",
-                what ? what : "weights", cudaGetErrorString(err));
-        (void)cudaGetLastError();
-        return NULL;
+    uint64_t two_gate = 0;
+    uint64_t bytes_per_slot = 0;
+    uint64_t arena_bytes = 0;
+    if (!cuda_stream_expert_persistent_mul_u64(
+            gate_expert_bytes, 2u, &two_gate) ||
+        !cuda_stream_expert_persistent_add_u64(
+            two_gate, down_expert_bytes, &bytes_per_slot) ||
+        !cuda_stream_expert_persistent_mul_u64(
+            bytes_per_slot, capacity, &arena_bytes) ||
+        arena_bytes > SIZE_MAX) {
+        return 0;
     }
+    out->gate_expert_bytes = gate_expert_bytes;
+    out->down_expert_bytes = down_expert_bytes;
+    out->bytes_per_slot = bytes_per_slot;
+    out->arena_bytes = arena_bytes;
+    out->capacity = capacity;
+    return 1;
+}
 
-    g_model_ranges.push_back({model_map, offset, bytes, dev, NULL, NULL, 0, 0, 1});
-    g_model_range_by_offset[offset] = g_model_ranges.size() - 1u;
-    g_model_range_bytes += bytes;
-    cuda_model_load_progress_note(g_model_range_bytes);
-    if (getenv("DS4_CUDA_WEIGHT_CACHE_VERBOSE")) {
-        fprintf(stderr, "ds4: CUDA fd-cached %s %.2f MiB (total %.2f GiB)\n",
-                what ? what : "weights",
-                (double)bytes / 1048576.0,
-                (double)g_model_range_bytes / 1073741824.0);
+static int cuda_stream_expert_persistent_class_equal(
+        const cuda_stream_expert_persistent_class *a,
+        const cuda_stream_expert_persistent_class *b) {
+    return a && b &&
+           a->gate_expert_bytes == b->gate_expert_bytes &&
+           a->down_expert_bytes == b->down_expert_bytes &&
+           a->bytes_per_slot == b->bytes_per_slot &&
+           a->arena_bytes == b->arena_bytes &&
+           a->capacity == b->capacity;
+}
+
+static int cuda_stream_expert_persistent_key_equal(
+        const cuda_stream_expert_persistent_key *a,
+        const cuda_stream_expert_persistent_key *b) {
+    return a && b && a->model_map == b->model_map &&
+           a->model_size == b->model_size &&
+           a->gate_offset == b->gate_offset &&
+           a->up_offset == b->up_offset &&
+           a->down_offset == b->down_offset &&
+           a->gate_expert_bytes == b->gate_expert_bytes &&
+           a->down_expert_bytes == b->down_expert_bytes &&
+           a->layer == b->layer &&
+           a->n_total_expert == b->n_total_expert &&
+           a->expert_id == b->expert_id;
+}
+
+static int cuda_stream_expert_persistent_table_range_ok(
+        uint64_t offset, uint64_t expert_bytes,
+        uint32_t n_total_expert, uint64_t model_size) {
+    uint64_t table_bytes = 0;
+    return cuda_stream_expert_persistent_mul_u64(
+               expert_bytes, n_total_expert, &table_bytes) &&
+           offset <= model_size && table_bytes <= model_size - offset;
+}
+
+static int cuda_stream_expert_persistent_key_make(
+        cuda_stream_expert_persistent_key *out,
+        const ds4_gpu_stream_expert_table *table,
+        int32_t expert_id) {
+    if (!out || !table || !table->model_map || table->model_size == 0 ||
+        table->n_total_expert == 0 || table->gate_expert_bytes == 0 ||
+        table->down_expert_bytes == 0 || expert_id < 0 ||
+        (uint32_t)expert_id >= table->n_total_expert ||
+        !cuda_stream_expert_persistent_table_range_ok(
+            table->gate_offset, table->gate_expert_bytes,
+            table->n_total_expert, table->model_size) ||
+        !cuda_stream_expert_persistent_table_range_ok(
+            table->up_offset, table->gate_expert_bytes,
+            table->n_total_expert, table->model_size) ||
+        !cuda_stream_expert_persistent_table_range_ok(
+            table->down_offset, table->down_expert_bytes,
+            table->n_total_expert, table->model_size)) {
+        return 0;
     }
-    return (const char *)dev;
+    out->model_map = table->model_map;
+    out->model_size = table->model_size;
+    out->gate_offset = table->gate_offset;
+    out->up_offset = table->up_offset;
+    out->down_offset = table->down_offset;
+    out->gate_expert_bytes = table->gate_expert_bytes;
+    out->down_expert_bytes = table->down_expert_bytes;
+    out->layer = table->layer;
+    out->n_total_expert = table->n_total_expert;
+    out->expert_id = (uint32_t)expert_id;
+    return 1;
 }
 
-static int cuda_model_copy_chunked(const void *model_map, uint64_t model_size, uint64_t map_offset, uint64_t map_size) {
-    if (!model_map || model_size == 0 || map_offset > model_size || map_size > model_size - map_offset) return 0;
-    if (getenv("DS4_CUDA_NO_MODEL_COPY") != NULL ||
-        getenv("DS4_CUDA_DIRECT_MODEL") != NULL ||
-        getenv("DS4_CUDA_WEIGHT_CACHE") != NULL ||
-        getenv("DS4_CUDA_WEIGHT_PRELOAD") != NULL) {
+static int cuda_stream_expert_persistent_state_init(
+        cuda_stream_expert_persistent_state *state,
+        uint32_t capacity,
+        uint64_t gate_expert_bytes,
+        uint64_t down_expert_bytes) {
+    if (!state || !cuda_stream_expert_persistent_class_make(
+            &state->size_class, capacity,
+            gate_expert_bytes, down_expert_bytes)) {
         return 0;
     }
-    if (g_model_device_owned || g_model_registered) return 1;
-
-    void *dev = NULL;
-    const double t0 = cuda_wall_sec();
-    cudaError_t err = cudaMalloc(&dev, (size_t)model_size);
-    if (err != cudaSuccess) {
-        fprintf(stderr, "ds4: CUDA model allocation skipped: %s\n", cudaGetErrorString(err));
-        (void)cudaGetLastError();
+    try {
+        state->slots.assign(capacity, {});
+        state->free_slots.clear();
+        state->free_slots.reserve(capacity);
+        for (uint32_t slot = capacity; slot != 0; slot--) {
+            state->free_slots.push_back(slot - 1u);
+        }
+    } catch (...) {
+        state->slots.clear();
+        state->free_slots.clear();
+        memset(&state->size_class, 0, sizeof(state->size_class));
         return 0;
     }
+    state->lru_clock = 0;
+    return 1;
+}
 
-    fprintf(stderr, "ds4: CUDA chunk-copying %.2f GiB model image\n",
-            (double)model_size / 1073741824.0);
+static void cuda_stream_expert_persistent_state_clear(
+        cuda_stream_expert_persistent_state *state) {
+    if (!state) return;
+    state->slots.clear();
+    state->free_slots.clear();
+    memset(&state->size_class, 0, sizeof(state->size_class));
+    state->lru_clock = 0;
+}
 
-    const uint64_t chunk = cuda_model_copy_chunk_bytes();
-    void *stage = NULL;
-    err = cudaMallocHost(&stage, (size_t)chunk);
-    if (err != cudaSuccess) {
-        fprintf(stderr, "ds4: CUDA pinned model staging allocation failed: %s\n", cudaGetErrorString(err));
-        (void)cudaFree(dev);
-        (void)cudaGetLastError();
+static uint32_t cuda_stream_expert_persistent_state_count(
+        const cuda_stream_expert_persistent_state *state) {
+    if (!state || state->free_slots.size() > state->slots.size()) return 0;
+    const size_t count = state->slots.size() - state->free_slots.size();
+    return count <= UINT32_MAX ? (uint32_t)count : 0;
+}
+
+static int cuda_stream_expert_persistent_state_valid(
+        const cuda_stream_expert_persistent_state *state) {
+    if (!state || state->size_class.capacity == 0 ||
+        state->slots.size() != state->size_class.capacity ||
+        state->free_slots.size() > state->slots.size()) {
         return 0;
     }
-
-    if (map_offset > 0) {
-        uint64_t copied_header = 0;
-        while (copied_header < map_offset) {
-            const uint64_t n = (map_offset - copied_header < chunk) ? (map_offset - copied_header) : chunk;
-            memcpy(stage, (const char *)model_map + copied_header, (size_t)n);
-            err = cudaMemcpy((char *)dev + copied_header, stage, (size_t)n, cudaMemcpyHostToDevice);
-            if (err != cudaSuccess) {
-                fprintf(stderr, "ds4: CUDA model header copy failed: %s\n", cudaGetErrorString(err));
-                (void)cudaFreeHost(stage);
-                (void)cudaFree(dev);
-                (void)cudaGetLastError();
+    std::vector<uint8_t> free_seen;
+    try {
+        free_seen.assign(state->slots.size(), 0u);
+    } catch (...) {
+        return 0;
+    }
+    for (uint32_t slot : state->free_slots) {
+        if (slot >= state->slots.size() || free_seen[slot] ||
+            state->slots[slot].valid || state->slots[slot].pin_count != 0) {
+            return 0;
+        }
+        free_seen[slot] = 1u;
+    }
+    for (size_t slot = 0; slot < state->slots.size(); slot++) {
+        const cuda_stream_expert_persistent_entry *entry =
+            &state->slots[slot];
+        if (entry->valid) {
+            if (free_seen[slot] || entry->last_use == 0 ||
+                entry->last_use > state->lru_clock) {
                 return 0;
             }
-            copied_header += n;
+        } else if (!free_seen[slot] || entry->last_use != 0 ||
+                   entry->pin_count != 0) {
+            return 0;
         }
     }
+    return 1;
+}
 
-    uint64_t copied = 0;
-    double last_report = t0;
-    while (copied < map_size) {
-        const uint64_t n = (map_size - copied < chunk) ? (map_size - copied) : chunk;
-        const uint64_t off = map_offset + copied;
-        memcpy(stage, (const char *)model_map + off, (size_t)n);
-        err = cudaMemcpy((char *)dev + off, stage, (size_t)n, cudaMemcpyHostToDevice);
-        if (err != cudaSuccess) {
-            fprintf(stderr, "ds4: CUDA model chunk copy failed at %.2f GiB: %s\n",
-                    (double)copied / 1073741824.0, cudaGetErrorString(err));
-            (void)cudaFreeHost(stage);
-            (void)cudaFree(dev);
-            (void)cudaGetLastError();
+static int cuda_stream_expert_persistent_state_equal(
+        const cuda_stream_expert_persistent_state *a,
+        const cuda_stream_expert_persistent_state *b) {
+    if (!a || !b ||
+        !cuda_stream_expert_persistent_class_equal(
+            &a->size_class, &b->size_class) ||
+        a->lru_clock != b->lru_clock ||
+        a->free_slots != b->free_slots ||
+        a->slots.size() != b->slots.size()) {
+        return 0;
+    }
+    for (size_t i = 0; i < a->slots.size(); i++) {
+        const cuda_stream_expert_persistent_entry *ea = &a->slots[i];
+        const cuda_stream_expert_persistent_entry *eb = &b->slots[i];
+        if (ea->valid != eb->valid || ea->last_use != eb->last_use ||
+            ea->pin_count != eb->pin_count ||
+            (ea->valid && !cuda_stream_expert_persistent_key_equal(
+                              &ea->key, &eb->key))) {
             return 0;
         }
-        cuda_model_discard_source_pages(model_map, model_size, off, n);
-        copied += n;
-        const double now = cuda_wall_sec();
-        if (getenv("DS4_CUDA_MODEL_COPY_VERBOSE") != NULL && now - last_report >= 2.0) {
-            fprintf(stderr, "ds4: CUDA model chunk copy %.2f/%.2f GiB\n",
-                    (double)copied / 1073741824.0,
-                    (double)map_size / 1073741824.0);
-            last_report = now;
-        }
     }
+    return 1;
+}
 
-    (void)cudaFreeHost(stage);
-    g_model_device_base = (const char *)dev;
-    g_model_device_owned = 1;
-    g_model_hmm_direct = 0;
-    const double t1 = cuda_wall_sec();
-    fprintf(stderr,
-            "ds4: CUDA model chunk copy complete in %.3fs (%.2f GiB tensors)\n",
-            t1 - t0,
-            (double)map_size / 1073741824.0);
+static int cuda_stream_expert_persistent_state_copy(
+        cuda_stream_expert_persistent_state *dst,
+        const cuda_stream_expert_persistent_state *src) {
+    if (!dst || !src) return 0;
+    try {
+        *dst = *src;
+    } catch (...) {
+        return 0;
+    }
     return 1;
 }
 
-static void cuda_model_range_release_all(void) {
-    for (const cuda_model_range &r : g_model_ranges) {
-        if (r.host_registered && r.registered_base) {
-            (void)cudaHostUnregister(r.registered_base);
-        } else if (r.device_ptr && !r.arena_allocated && !r.borrowed) {
-            (void)cudaFree(r.device_ptr);
+static int cuda_stream_expert_persistent_find_key(
+        const cuda_stream_expert_persistent_state *state,
+        const cuda_stream_expert_persistent_key *key,
+        uint32_t *slot_out) {
+    if (!state || !key) return 0;
+    for (uint32_t slot = 0; slot < state->slots.size(); slot++) {
+        const cuda_stream_expert_persistent_entry *entry =
+            &state->slots[slot];
+        if (entry->valid && cuda_stream_expert_persistent_key_equal(
+                                &entry->key, key)) {
+            if (slot_out) *slot_out = slot;
+            return 1;
         }
     }
-    for (const cuda_model_arena &a : g_model_arenas) {
-        if (a.device_ptr) (void)cudaFree(a.device_ptr);
+    return 0;
+}
+
+static int cuda_stream_expert_persistent_touch(
+        cuda_stream_expert_persistent_state *state, uint32_t slot) {
+    if (!state || slot >= state->slots.size() ||
+        !state->slots[slot].valid || state->lru_clock == UINT64_MAX) {
+        return 0;
     }
-    g_model_arenas.clear();
-    g_model_ranges.clear();
-    g_model_range_by_offset.clear();
-    g_model_range_bytes = 0;
-    cuda_model_load_progress_reset();
+    state->slots[slot].last_use = ++state->lru_clock;
+    return 1;
 }
 
-static void cuda_derived_range_release_all(void) {
+static int cuda_stream_expert_persistent_pin_key(
+        cuda_stream_expert_persistent_state *state,
+        const cuda_stream_expert_persistent_key *key,
+        int pin) {
+    uint32_t slot = 0;
+    if (!state || !cuda_stream_expert_persistent_find_key(
+            state, key, &slot)) {
+        return 0;
+    }
+    if (pin) {
+        if (state->slots[slot].pin_count == UINT32_MAX) return 0;
+        state->slots[slot].pin_count++;
+    } else {
+        if (state->slots[slot].pin_count == 0) return 0;
+        state->slots[slot].pin_count--;
+    }
+    return 1;
+}
+
+static void cuda_stream_expert_persistent_note_reject(
+        std::atomic<uint64_t> *specific) {
+    g_stream_expert_persistent_rejects.fetch_add(
+        1, std::memory_order_relaxed);
+    if (specific) specific->fetch_add(1, std::memory_order_relaxed);
+}
+
+/* Build against a private state copy.  Until commit(), both LRU touches and
+ * victim selection are invisible to the live cache, so every rejection and
+ * every abandoned upload plan is a true rollback. */
+static int cuda_stream_expert_persistent_plan_build(
+        const cuda_stream_expert_persistent_state *state,
+        const ds4_gpu_stream_expert_table *table,
+        const int32_t *selected_ids,
+        uint32_t n_selected,
+        cuda_stream_expert_persistent_plan *plan) {
+    enum { dense_expert_stack_capacity = 384 };
+    g_stream_expert_persistent_plan_attempts.fetch_add(
+        1, std::memory_order_relaxed);
+    if (!state || !table || !selected_ids || n_selected == 0 || !plan ||
+        !cuda_stream_expert_persistent_state_valid(state)) {
+        cuda_stream_expert_persistent_note_reject(NULL);
+        return 0;
+    }
+
+    cuda_stream_expert_persistent_class request_class = {};
+    if (!cuda_stream_expert_persistent_class_make(
+            &request_class, state->size_class.capacity,
+            table->gate_expert_bytes, table->down_expert_bytes)) {
+        cuda_stream_expert_persistent_note_reject(
+            &g_stream_expert_persistent_overflow_rejects);
+        return 0;
+    }
+    if (!cuda_stream_expert_persistent_class_equal(
+            &state->size_class, &request_class)) {
+        cuda_stream_expert_persistent_note_reject(
+            &g_stream_expert_persistent_class_rejects);
+        return 0;
+    }
+
+    cuda_stream_expert_persistent_key key_template = {};
+    if (!cuda_stream_expert_persistent_key_make(&key_template, table, 0)) {
+        cuda_stream_expert_persistent_note_reject(
+            &g_stream_expert_persistent_overflow_rejects);
+        return 0;
+    }
+
+    std::vector<cuda_stream_expert_persistent_key> unique_keys;
+    std::vector<uint32_t> input_unique;
+    uint32_t expert_to_unique_stack[dense_expert_stack_capacity];
+    std::vector<uint32_t> expert_to_unique_heap;
+    uint32_t *expert_to_unique = NULL;
+    uint32_t duplicate_count = 0;
+    try {
+        /* Expert ids form a small dense domain.  Keep only one full key per
+         * expert instead of reserving n_selected keys and scanning them for
+         * every token/top-k pair. */
+        unique_keys.reserve(std::min(n_selected, table->n_total_expert));
+        input_unique.reserve(n_selected);
+        if (table->n_total_expert <= dense_expert_stack_capacity) {
+            std::fill_n(expert_to_unique_stack,
+                        table->n_total_expert,
+                        UINT32_MAX);
+            expert_to_unique = expert_to_unique_stack;
+        } else {
+            expert_to_unique_heap.assign(table->n_total_expert, UINT32_MAX);
+            expert_to_unique = expert_to_unique_heap.data();
+        }
+        for (uint32_t i = 0; i < n_selected; i++) {
+            const int32_t selected_id = selected_ids[i];
+            if (selected_id < 0 ||
+                (uint32_t)selected_id >= key_template.n_total_expert) {
+                cuda_stream_expert_persistent_note_reject(
+                    &g_stream_expert_persistent_overflow_rejects);
+                return 0;
+            }
+            const uint32_t expert_id = (uint32_t)selected_id;
+            uint32_t unique_index = expert_to_unique[expert_id];
+            if (unique_index == UINT32_MAX) {
+                if (unique_keys.size() >= UINT32_MAX) {
+                    cuda_stream_expert_persistent_note_reject(
+                        &g_stream_expert_persistent_overflow_rejects);
+                    return 0;
+                }
+                unique_index = (uint32_t)unique_keys.size();
+                cuda_stream_expert_persistent_key key = key_template;
+                key.expert_id = expert_id;
+                unique_keys.push_back(key);
+                expert_to_unique[expert_id] = unique_index;
+            } else {
+                /* The immutable key template was validated before the loop;
+                 * duplicates therefore need only verify the dense map's own
+                 * expert-id invariant. */
+                if (unique_index >= unique_keys.size() ||
+                    unique_keys[unique_index].expert_id != expert_id) {
+                    cuda_stream_expert_persistent_note_reject(NULL);
+                    return 0;
+                }
+                duplicate_count++;
+            }
+            input_unique.push_back(unique_index);
+        }
+    } catch (...) {
+        cuda_stream_expert_persistent_note_reject(NULL);
+        return 0;
+    }
+    if (unique_keys.size() > state->size_class.capacity) {
+        cuda_stream_expert_persistent_note_reject(
+            &g_stream_expert_persistent_budget_rejects);
+        return 0;
+    }
+
+    cuda_stream_expert_persistent_plan candidate = {};
+    if (!cuda_stream_expert_persistent_state_copy(
+            &candidate.next, state)) {
+        cuda_stream_expert_persistent_note_reject(NULL);
+        return 0;
+    }
+    std::vector<uint8_t> protected_slots;
+    uint8_t mismatched_expert_keys_stack[dense_expert_stack_capacity];
+    std::vector<uint8_t> mismatched_expert_keys_heap;
+    uint8_t *mismatched_expert_keys = NULL;
+    std::vector<uint32_t> unique_slots;
+    try {
+        protected_slots.assign(candidate.next.slots.size(), 0u);
+        if (unique_keys.size() <= dense_expert_stack_capacity) {
+            std::fill_n(mismatched_expert_keys_stack,
+                        unique_keys.size(),
+                        (uint8_t)0u);
+            mismatched_expert_keys = mismatched_expert_keys_stack;
+        } else {
+            mismatched_expert_keys_heap.assign(unique_keys.size(), 0u);
+            mismatched_expert_keys = mismatched_expert_keys_heap.data();
+        }
+        unique_slots.assign(unique_keys.size(), UINT32_MAX);
+        candidate.remap.resize(n_selected);
+        candidate.loads.reserve(unique_keys.size());
+    } catch (...) {
+        cuda_stream_expert_persistent_note_reject(NULL);
+        return 0;
+    }
+    /* Derive a request-local exact-key index with one increasing slot scan.
+     * It deliberately does not enter persistent state: commit/rollback stay
+     * unchanged, and keys from other models or layers cannot alias. */
+    for (uint32_t slot = 0; slot < candidate.next.slots.size(); slot++) {
+        const cuda_stream_expert_persistent_entry *entry =
+            &candidate.next.slots[slot];
+        if (entry->pin_count != 0) {
+            protected_slots[slot] = 1u;
+        }
+        if (!entry->valid ||
+            entry->key.expert_id >= key_template.n_total_expert) {
+            continue;
+        }
+        const uint32_t unique_index =
+            expert_to_unique[entry->key.expert_id];
+        if (unique_index == UINT32_MAX ||
+            unique_index >= unique_keys.size()) {
+            continue;
+        }
+        if (cuda_stream_expert_persistent_key_equal(
+                &entry->key, &unique_keys[unique_index])) {
+            /* Preserve the old linear lookup's deterministic first-slot
+             * behaviour if a damaged state ever contains duplicate keys. */
+            if (unique_slots[unique_index] == UINT32_MAX) {
+                unique_slots[unique_index] = slot;
+            }
+        } else {
+            mismatched_expert_keys[unique_index] = 1u;
+        }
+    }
+
+    uint32_t hit_count = 0;
+    uint32_t miss_count = 0;
+    uint32_t key_miss_count = 0;
+    uint32_t free_assignment_count = 0;
+    uint32_t eviction_count = 0;
+    /* Resolve every hit before considering a victim.  A request may put a
+     * cold miss before a resident key; a one-pass planner would otherwise be
+     * able to evict that later requested resident slot. */
+    for (uint32_t u = 0; u < unique_keys.size(); u++) {
+        const uint32_t slot = unique_slots[u];
+        if (slot != UINT32_MAX) {
+            if (slot >= candidate.next.slots.size() ||
+                !candidate.next.slots[slot].valid ||
+                !cuda_stream_expert_persistent_key_equal(
+                    &candidate.next.slots[slot].key, &unique_keys[u])) {
+                cuda_stream_expert_persistent_note_reject(NULL);
+                return 0;
+            }
+            protected_slots[slot] = 1u;
+            if (!cuda_stream_expert_persistent_touch(
+                    &candidate.next, slot)) {
+                cuda_stream_expert_persistent_note_reject(
+                    &g_stream_expert_persistent_overflow_rejects);
+                return 0;
+            }
+            unique_slots[u] = slot;
+            hit_count++;
+            continue;
+        }
+        miss_count++;
+        if (mismatched_expert_keys[u]) key_miss_count++;
+    }
+
+    /* Only slots outside the complete request hit-set and outside active
+     * leases are eligible LRU victims.  Newly assigned miss slots are also
+     * protected so later misses in the transaction cannot replace them. */
+    for (uint32_t u = 0; u < unique_keys.size(); u++) {
+        if (unique_slots[u] != UINT32_MAX) continue;
+        uint32_t slot = 0;
+        int had_victim = 0;
+        cuda_stream_expert_persistent_key victim = {};
+        if (!candidate.next.free_slots.empty()) {
+            slot = candidate.next.free_slots.back();
+            candidate.next.free_slots.pop_back();
+            if (slot >= candidate.next.slots.size() ||
+                candidate.next.slots[slot].valid ||
+                candidate.next.slots[slot].pin_count != 0) {
+                cuda_stream_expert_persistent_note_reject(NULL);
+                return 0;
+            }
+            free_assignment_count++;
+        } else {
+            uint64_t oldest = UINT64_MAX;
+            slot = UINT32_MAX;
+            for (uint32_t s = 0; s < candidate.next.slots.size(); s++) {
+                const cuda_stream_expert_persistent_entry *entry =
+                    &candidate.next.slots[s];
+                if (!entry->valid || protected_slots[s] ||
+                    entry->pin_count != 0) {
+                    continue;
+                }
+                if (slot == UINT32_MAX || entry->last_use < oldest ||
+                    (entry->last_use == oldest && s < slot)) {
+                    oldest = entry->last_use;
+                    slot = s;
+                }
+            }
+            if (slot == UINT32_MAX) {
+                cuda_stream_expert_persistent_note_reject(
+                    &g_stream_expert_persistent_protected_rejects);
+                return 0;
+            }
+            victim = candidate.next.slots[slot].key;
+            had_victim = 1;
+            eviction_count++;
+        }
+        cuda_stream_expert_persistent_entry *entry =
+            &candidate.next.slots[slot];
+        entry->key = unique_keys[u];
+        entry->valid = 1;
+        entry->pin_count = 0;
+        if (!cuda_stream_expert_persistent_touch(&candidate.next, slot)) {
+            cuda_stream_expert_persistent_note_reject(
+                &g_stream_expert_persistent_overflow_rejects);
+            return 0;
+        }
+        protected_slots[slot] = 1u;
+        unique_slots[u] = slot;
+        cuda_stream_expert_persistent_load load = {};
+        load.key = unique_keys[u];
+        load.victim = victim;
+        load.slot = slot;
+        load.had_victim = had_victim;
+        try {
+            candidate.loads.push_back(load);
+        } catch (...) {
+            cuda_stream_expert_persistent_note_reject(NULL);
+            return 0;
+        }
+    }
+
+    uint32_t min_slot = UINT32_MAX;
+    uint32_t max_slot = 0;
+    for (uint32_t slot : unique_slots) {
+        if (slot == UINT32_MAX || slot >= candidate.next.slots.size()) {
+            cuda_stream_expert_persistent_note_reject(NULL);
+            return 0;
+        }
+        min_slot = std::min(min_slot, slot);
+        max_slot = std::max(max_slot, slot);
+    }
+    if (min_slot == UINT32_MAX || max_slot < min_slot) {
+        cuda_stream_expert_persistent_note_reject(NULL);
+        return 0;
+    }
+    const uint64_t domain64 =
+        (uint64_t)max_slot - min_slot + 1u;
+    if (domain64 == 0 || domain64 > INT_MAX) {
+        cuda_stream_expert_persistent_note_reject(
+            &g_stream_expert_persistent_overflow_rejects);
+        return 0;
+    }
+    for (uint32_t i = 0; i < n_selected; i++) {
+        const uint32_t unique_index = input_unique[i];
+        if (unique_index >= unique_slots.size() ||
+            unique_slots[unique_index] < min_slot) {
+            cuda_stream_expert_persistent_note_reject(NULL);
+            return 0;
+        }
+        candidate.remap[i] = unique_slots[unique_index] - min_slot;
+        if (candidate.remap[i] >= domain64) {
+            cuda_stream_expert_persistent_note_reject(NULL);
+            return 0;
+        }
+    }
+    if (!cuda_stream_expert_persistent_state_valid(&candidate.next)) {
+        cuda_stream_expert_persistent_note_reject(NULL);
+        return 0;
+    }
+    candidate.slot_base = min_slot;
+    candidate.weight_domain = (uint32_t)domain64;
+    candidate.unique_count = (uint32_t)unique_keys.size();
+    candidate.hit_count = hit_count;
+    candidate.miss_count = miss_count;
+    candidate.duplicate_count = duplicate_count;
+    candidate.built = 1;
+    try {
+        *plan = std::move(candidate);
+    } catch (...) {
+        cuda_stream_expert_persistent_note_reject(NULL);
+        return 0;
+    }
+
+    g_stream_expert_persistent_plans_built.fetch_add(
+        1, std::memory_order_relaxed);
+    g_stream_expert_persistent_hits.fetch_add(
+        hit_count, std::memory_order_relaxed);
+    g_stream_expert_persistent_misses.fetch_add(
+        miss_count, std::memory_order_relaxed);
+    g_stream_expert_persistent_duplicates.fetch_add(
+        duplicate_count, std::memory_order_relaxed);
+    g_stream_expert_persistent_free_assignments.fetch_add(
+        free_assignment_count, std::memory_order_relaxed);
+    g_stream_expert_persistent_evictions.fetch_add(
+        eviction_count, std::memory_order_relaxed);
+    g_stream_expert_persistent_key_misses.fetch_add(
+        key_miss_count, std::memory_order_relaxed);
+    return 1;
+}
+
+static int cuda_stream_expert_persistent_plan_commit(
+        cuda_stream_expert_persistent_state *state,
+        cuda_stream_expert_persistent_plan *plan) {
+    if (!state || !plan || !plan->built ||
+        !cuda_stream_expert_persistent_state_valid(&plan->next)) {
+        return 0;
+    }
+    state->size_class = plan->next.size_class;
+    state->slots.swap(plan->next.slots);
+    state->free_slots.swap(plan->next.free_slots);
+    state->lru_clock = plan->next.lru_clock;
+    plan->built = 0;
+    g_stream_expert_persistent_commits.fetch_add(
+        1, std::memory_order_relaxed);
+    return 1;
+}
+
+static void cuda_stream_expert_persistent_plan_rollback(
+        cuda_stream_expert_persistent_plan *plan) {
+    if (!plan || !plan->built) return;
+    plan->built = 0;
+    plan->next.slots.clear();
+    plan->next.free_slots.clear();
+    plan->remap.clear();
+    plan->loads.clear();
+    g_stream_expert_persistent_rollbacks.fetch_add(
+        1, std::memory_order_relaxed);
+}
+
+static void cuda_stream_expert_persistent_resolve_policy(
+        int enable, int disable, int require, int stats, int oracle,
+        int *enabled_out, int *required_out, int *stats_out,
+        int *oracle_out) {
+    const int disabled = disable != 0;
+    if (enabled_out) {
+        *enabled_out = !disabled && (enable || require || oracle);
+    }
+    if (required_out) *required_out = !disabled && require;
+    if (stats_out) *stats_out = !disabled && stats;
+    if (oracle_out) *oracle_out = !disabled && oracle;
+}
+
+extern "C" int ds4_cuda_test_stream_expert_persistent_env_value(
+        const char *value) {
+    return cuda_stream_selected_env_value_enabled(value);
+}
+
+extern "C" int ds4_cuda_test_stream_expert_persistent_policy(
+        int enable, int disable, int require, int stats, int oracle,
+        int *enabled_out, int *required_out, int *stats_out,
+        int *oracle_out) {
+    if (!enabled_out || !required_out || !stats_out || !oracle_out) return 0;
+    cuda_stream_expert_persistent_resolve_policy(
+        enable, disable, require, stats, oracle,
+        enabled_out, required_out, stats_out, oracle_out);
+    return 1;
+}
+
+static void cuda_stream_expert_persistent_report_at_exit(void) {
+    fprintf(stderr,
+            "ds4: CUDA persistent expert planner: attempts=%llu built=%llu "
+            "commits=%llu rollbacks=%llu hits=%llu misses=%llu "
+            "duplicates=%llu free=%llu evictions=%llu rejects=%llu "
+            "budget_rejects=%llu class_rejects=%llu protected_rejects=%llu "
+            "key_misses=%llu overflow_rejects=%llu oracle=%llu/%llu\n",
+            (unsigned long long)g_stream_expert_persistent_plan_attempts.load(),
+            (unsigned long long)g_stream_expert_persistent_plans_built.load(),
+            (unsigned long long)g_stream_expert_persistent_commits.load(),
+            (unsigned long long)g_stream_expert_persistent_rollbacks.load(),
+            (unsigned long long)g_stream_expert_persistent_hits.load(),
+            (unsigned long long)g_stream_expert_persistent_misses.load(),
+            (unsigned long long)g_stream_expert_persistent_duplicates.load(),
+            (unsigned long long)g_stream_expert_persistent_free_assignments.load(),
+            (unsigned long long)g_stream_expert_persistent_evictions.load(),
+            (unsigned long long)g_stream_expert_persistent_rejects.load(),
+            (unsigned long long)g_stream_expert_persistent_budget_rejects.load(),
+            (unsigned long long)g_stream_expert_persistent_class_rejects.load(),
+            (unsigned long long)g_stream_expert_persistent_protected_rejects.load(),
+            (unsigned long long)g_stream_expert_persistent_key_misses.load(),
+            (unsigned long long)g_stream_expert_persistent_overflow_rejects.load(),
+            (unsigned long long)g_stream_expert_persistent_oracle_runs.load(),
+            (unsigned long long)g_stream_expert_persistent_oracle_failures.load());
+}
+
+static void cuda_stream_expert_persistent_init(void) {
+    const int enable = cuda_stream_selected_env_flag(
+        "DS4_CUDA_ENABLE_STREAMING_EXPERT_PERSISTENT_CACHE");
+    const int disable = cuda_stream_selected_env_flag(
+        "DS4_CUDA_DISABLE_STREAMING_EXPERT_PERSISTENT_CACHE") ||
+        cuda_stream_selected_env_flag(
+            "DS4_CUDA_NO_STREAMING_EXPERT_PERSISTENT_CACHE");
+    const int require = cuda_stream_selected_env_flag(
+        "DS4_CUDA_REQUIRE_STREAMING_EXPERT_PERSISTENT_CACHE");
+    const int stats = cuda_stream_selected_env_flag(
+        "DS4_CUDA_STREAMING_EXPERT_PERSISTENT_CACHE_STATS");
+    const int oracle = cuda_stream_selected_env_flag(
+        "DS4_CUDA_STREAMING_EXPERT_PERSISTENT_CACHE_ORACLE");
+    cuda_stream_expert_persistent_resolve_policy(
+        enable, disable, require, stats, oracle,
+        &g_stream_expert_persistent_enabled,
+        &g_stream_expert_persistent_required,
+        &g_stream_expert_persistent_stats,
+        &g_stream_expert_persistent_oracle);
+    if ((enable || require || stats || oracle) &&
+        !g_stream_expert_persistent_report_registered) {
+        g_stream_expert_persistent_report_registered = 1;
+        (void)atexit(cuda_stream_expert_persistent_report_at_exit);
+    }
+    if (g_stream_expert_persistent_enabled) {
+        fprintf(stderr,
+                "ds4: CUDA persistent expert planner enabled%s%s\n",
+                g_stream_expert_persistent_required ? " (required)" : "",
+                g_stream_expert_persistent_oracle ? " with oracle" : "");
+    } else if (disable && (enable || require || oracle)) {
+        fprintf(stderr,
+                "ds4: CUDA persistent expert planner disabled by rollback "
+                "override\n");
+    }
+}
+
+static int cuda_stream_expert_persistent_requested(void) {
+    std::call_once(g_stream_expert_persistent_once,
+                   cuda_stream_expert_persistent_init);
+    return g_stream_expert_persistent_enabled;
+}
+
+static int cuda_stream_expert_persistent_require_requested(void) {
+    std::call_once(g_stream_expert_persistent_once,
+                   cuda_stream_expert_persistent_init);
+    return g_stream_expert_persistent_required;
+}
+
+extern "C" void ds4_cuda_stream_expert_persistent_get_report(
+        ds4_cuda_stream_expert_persistent_report *report) {
+    if (!report) return;
+    std::call_once(g_stream_expert_persistent_once,
+                   cuda_stream_expert_persistent_init);
+    cuda_stream_expert_persistent_report_layout out = {};
+    out.plan_attempts = g_stream_expert_persistent_plan_attempts.load();
+    out.plans_built = g_stream_expert_persistent_plans_built.load();
+    out.commits = g_stream_expert_persistent_commits.load();
+    out.rollbacks = g_stream_expert_persistent_rollbacks.load();
+    out.hits = g_stream_expert_persistent_hits.load();
+    out.misses = g_stream_expert_persistent_misses.load();
+    out.duplicates = g_stream_expert_persistent_duplicates.load();
+    out.free_assignments =
+        g_stream_expert_persistent_free_assignments.load();
+    out.evictions = g_stream_expert_persistent_evictions.load();
+    out.rejects = g_stream_expert_persistent_rejects.load();
+    out.budget_rejects =
+        g_stream_expert_persistent_budget_rejects.load();
+    out.class_rejects = g_stream_expert_persistent_class_rejects.load();
+    out.protected_rejects =
+        g_stream_expert_persistent_protected_rejects.load();
+    out.key_misses = g_stream_expert_persistent_key_misses.load();
+    out.overflow_rejects =
+        g_stream_expert_persistent_overflow_rejects.load();
+    out.oracle_runs = g_stream_expert_persistent_oracle_runs.load();
+    out.oracle_failures =
+        g_stream_expert_persistent_oracle_failures.load();
+    out.arena_allocations =
+        g_stream_expert_persistent_arena_allocations.load();
+    out.arena_reuses = g_stream_expert_persistent_arena_reuses.load();
+    out.arena_releases = g_stream_expert_persistent_arena_releases.load();
+    out.arena_failures = g_stream_expert_persistent_arena_failures.load();
+    out.arena_oracle_runs =
+        g_stream_expert_persistent_arena_oracle_runs.load();
+    out.arena_oracle_failures =
+        g_stream_expert_persistent_arena_oracle_failures.load();
+    out.epochs_attempted =
+        g_stream_expert_persistent_epochs_attempted.load();
+    out.epochs_published =
+        g_stream_expert_persistent_epochs_published.load();
+    out.all_hit_epochs =
+        g_stream_expert_persistent_all_hit_epochs.load();
+    out.miss_epochs = g_stream_expert_persistent_miss_epochs.load();
+    out.miss_experts = g_stream_expert_persistent_miss_experts.load();
+    out.weight_bytes_uploaded =
+        g_stream_expert_persistent_weight_bytes.load();
+    out.remap_bytes_uploaded =
+        g_stream_expert_persistent_remap_bytes.load();
+    out.upload_failures =
+        g_stream_expert_persistent_upload_failures.load();
+    out.fallbacks = g_stream_expert_persistent_fallbacks.load();
+    out.slot_invalidations =
+        g_stream_expert_persistent_slot_invalidations.load();
+    out.poisons = g_stream_expert_persistent_poisons.load();
+    out.persistent_dispatches =
+        g_stream_expert_persistent_dispatches.load();
+    out.transient_dispatches =
+        g_stream_expert_transient_dispatches.load();
+    out.runtime_oracle_runs =
+        g_stream_expert_persistent_runtime_oracle_runs.load();
+    out.runtime_oracle_failures =
+        g_stream_expert_persistent_runtime_oracle_failures.load();
+    out.enabled = g_stream_expert_persistent_enabled;
+    out.required = g_stream_expert_persistent_required;
+    out.stats = g_stream_expert_persistent_stats;
+    out.oracle = g_stream_expert_persistent_oracle;
+    memcpy(report, &out, sizeof(out));
+}
+
+static int cuda_stream_expert_persistent_test_table_make(
+        ds4_gpu_stream_expert_table *table,
+        const void *model_map,
+        uint32_t layer,
+        uint32_t n_total_expert,
+        uint64_t gate_expert_bytes,
+        uint64_t down_expert_bytes) {
+    if (!table || !model_map || n_total_expert == 0) return 0;
+    uint64_t gate_table_bytes = 0;
+    uint64_t down_table_bytes = 0;
+    uint64_t down_offset = 0;
+    uint64_t model_size = 0;
+    if (!cuda_stream_expert_persistent_mul_u64(
+            gate_expert_bytes, n_total_expert, &gate_table_bytes) ||
+        !cuda_stream_expert_persistent_mul_u64(
+            down_expert_bytes, n_total_expert, &down_table_bytes) ||
+        !cuda_stream_expert_persistent_mul_u64(
+            gate_table_bytes, 2u, &down_offset) ||
+        !cuda_stream_expert_persistent_add_u64(
+            down_offset, down_table_bytes, &model_size)) {
+        return 0;
+    }
+    memset(table, 0, sizeof(*table));
+    table->model_map = model_map;
+    table->model_size = model_size;
+    table->layer = layer;
+    table->n_total_expert = n_total_expert;
+    table->gate_offset = 0;
+    table->up_offset = gate_table_bytes;
+    table->down_offset = down_offset;
+    table->gate_expert_bytes = gate_expert_bytes;
+    table->down_expert_bytes = down_expert_bytes;
+    return 1;
+}
+
+static int cuda_stream_expert_persistent_test_basic(void) {
+    unsigned char model_marker = 0;
+    ds4_gpu_stream_expert_table table = {};
+    cuda_stream_expert_persistent_state state = {};
+    cuda_stream_expert_persistent_state before = {};
+    if (!cuda_stream_expert_persistent_test_table_make(
+            &table, &model_marker, 7u, 8u, 16u, 8u) ||
+        !cuda_stream_expert_persistent_state_init(&state, 4u, 16u, 8u) ||
+        !cuda_stream_expert_persistent_state_copy(&before, &state)) {
+        return 0;
+    }
+
+    const int32_t cold_ids[3] = {2, 3, 2};
+    cuda_stream_expert_persistent_plan cold = {};
+    if (!cuda_stream_expert_persistent_plan_build(
+            &state, &table, cold_ids, 3u, &cold) ||
+        !cuda_stream_expert_persistent_state_equal(&state, &before) ||
+        cold.unique_count != 2u || cold.hit_count != 0u ||
+        cold.miss_count != 2u || cold.duplicate_count != 1u ||
+        cold.loads.size() != 2u || cold.loads[0].slot != 0u ||
+        cold.loads[1].slot != 1u || cold.slot_base != 0u ||
+        cold.weight_domain != 2u || cold.remap.size() != 3u ||
+        cold.remap[0] != 0u || cold.remap[1] != 1u ||
+        cold.remap[2] != 0u ||
+        !cuda_stream_expert_persistent_plan_commit(&state, &cold) ||
+        !cuda_stream_expert_persistent_state_valid(&state)) {
+        return 0;
+    }
+
+    const int32_t fill_ids[2] = {4, 5};
+    cuda_stream_expert_persistent_plan fill = {};
+    if (!cuda_stream_expert_persistent_plan_build(
+            &state, &table, fill_ids, 2u, &fill) ||
+        fill.loads.size() != 2u || fill.loads[0].slot != 2u ||
+        fill.loads[1].slot != 3u ||
+        !cuda_stream_expert_persistent_plan_commit(&state, &fill)) {
+        return 0;
+    }
+
+    if (!cuda_stream_expert_persistent_state_copy(&before, &state)) return 0;
+    const int32_t sparse_ids[3] = {3, 5, 3};
+    cuda_stream_expert_persistent_plan sparse = {};
+    if (!cuda_stream_expert_persistent_plan_build(
+            &state, &table, sparse_ids, 3u, &sparse) ||
+        !cuda_stream_expert_persistent_state_equal(&state, &before) ||
+        sparse.unique_count != 2u || sparse.hit_count != 2u ||
+        sparse.miss_count != 0u || sparse.duplicate_count != 1u ||
+        !sparse.loads.empty() || sparse.slot_base != 1u ||
+        sparse.weight_domain != 3u || sparse.remap.size() != 3u ||
+        sparse.remap[0] != 0u || sparse.remap[1] != 2u ||
+        sparse.remap[2] != 0u ||
+        !cuda_stream_expert_persistent_plan_commit(&state, &sparse) ||
+        !cuda_stream_expert_persistent_state_valid(&state)) {
+        return 0;
+    }
+    return 1;
+}
+
+static int cuda_stream_expert_persistent_test_protection(void) {
+    unsigned char model_marker = 0;
+    ds4_gpu_stream_expert_table table = {};
+    cuda_stream_expert_persistent_state state = {};
+    if (!cuda_stream_expert_persistent_test_table_make(
+            &table, &model_marker, 11u, 8u, 16u, 8u) ||
+        !cuda_stream_expert_persistent_state_init(&state, 3u, 16u, 8u)) {
+        return 0;
+    }
+    const int32_t initial_ids[3] = {0, 1, 2};
+    cuda_stream_expert_persistent_plan initial = {};
+    if (!cuda_stream_expert_persistent_plan_build(
+            &state, &table, initial_ids, 3u, &initial) ||
+        !cuda_stream_expert_persistent_plan_commit(&state, &initial)) {
+        return 0;
+    }
+
+    cuda_stream_expert_persistent_key key0 = {};
+    cuda_stream_expert_persistent_key key1 = {};
+    cuda_stream_expert_persistent_key key2 = {};
+    if (!cuda_stream_expert_persistent_key_make(&key0, &table, 0) ||
+        !cuda_stream_expert_persistent_key_make(&key1, &table, 1) ||
+        !cuda_stream_expert_persistent_key_make(&key2, &table, 2)) {
+        return 0;
+    }
+    cuda_stream_expert_persistent_state before = {};
+    if (!cuda_stream_expert_persistent_state_copy(&before, &state)) return 0;
+
+    /* The hit is deliberately ordered after the miss.  Planning all hits
+     * first must protect resident slot zero before the miss selects a victim. */
+    const int32_t miss_then_hit[2] = {3, 0};
+    cuda_stream_expert_persistent_plan requested = {};
+    if (!cuda_stream_expert_persistent_plan_build(
+            &state, &table, miss_then_hit, 2u, &requested) ||
+        requested.hit_count != 1u || requested.miss_count != 1u ||
+        requested.loads.size() != 1u || requested.loads[0].slot != 1u ||
+        !requested.loads[0].had_victim ||
+        requested.loads[0].victim.expert_id != 1u ||
+        requested.slot_base != 0u || requested.weight_domain != 2u ||
+        requested.remap.size() != 2u || requested.remap[0] != 1u ||
+        requested.remap[1] != 0u ||
+        !cuda_stream_expert_persistent_state_equal(&state, &before)) {
+        return 0;
+    }
+    cuda_stream_expert_persistent_plan_rollback(&requested);
+    if (!cuda_stream_expert_persistent_state_equal(&state, &before) ||
+        !cuda_stream_expert_persistent_pin_key(&state, &key0, 1) ||
+        !cuda_stream_expert_persistent_state_copy(&before, &state)) {
+        return 0;
+    }
+
+    /* Slot zero is the LRU, but its active lease protects it.  The planner
+     * must choose the next-oldest unpinned slot, then rollback without an LRU
+     * or free-list mutation. */
+    const int32_t miss_id[1] = {3};
+    cuda_stream_expert_persistent_plan replacement = {};
+    if (!cuda_stream_expert_persistent_plan_build(
+            &state, &table, miss_id, 1u, &replacement) ||
+        replacement.loads.size() != 1u ||
+        replacement.loads[0].slot != 1u ||
+        !replacement.loads[0].had_victim ||
+        replacement.loads[0].victim.expert_id != 1u ||
+        !cuda_stream_expert_persistent_state_equal(&state, &before)) {
+        return 0;
+    }
+    cuda_stream_expert_persistent_plan_rollback(&replacement);
+    if (!cuda_stream_expert_persistent_state_equal(&state, &before)) return 0;
+
+    if (!cuda_stream_expert_persistent_pin_key(&state, &key1, 1) ||
+        !cuda_stream_expert_persistent_pin_key(&state, &key2, 1) ||
+        !cuda_stream_expert_persistent_state_copy(&before, &state)) {
+        return 0;
+    }
+    cuda_stream_expert_persistent_plan blocked = {};
+    if (cuda_stream_expert_persistent_plan_build(
+            &state, &table, miss_id, 1u, &blocked) ||
+        !cuda_stream_expert_persistent_state_equal(&state, &before)) {
+        return 0;
+    }
+    return 1;
+}
+
+static int cuda_stream_expert_persistent_test_rejections(void) {
+    unsigned char model_a = 0;
+    unsigned char model_b = 0;
+    ds4_gpu_stream_expert_table table = {};
+    cuda_stream_expert_persistent_state state = {};
+    if (!cuda_stream_expert_persistent_test_table_make(
+            &table, &model_a, 3u, 8u, 16u, 8u) ||
+        !cuda_stream_expert_persistent_state_init(&state, 2u, 16u, 8u)) {
+        return 0;
+    }
+    const int32_t initial_ids[2] = {0, 1};
+    cuda_stream_expert_persistent_plan initial = {};
+    if (!cuda_stream_expert_persistent_plan_build(
+            &state, &table, initial_ids, 2u, &initial) ||
+        !cuda_stream_expert_persistent_plan_commit(&state, &initial)) {
+        return 0;
+    }
+    cuda_stream_expert_persistent_state before = {};
+    if (!cuda_stream_expert_persistent_state_copy(&before, &state)) return 0;
+
+    ds4_gpu_stream_expert_table wrong_class = {};
+    const int32_t one_id[1] = {0};
+    cuda_stream_expert_persistent_plan rejected = {};
+    if (!cuda_stream_expert_persistent_test_table_make(
+            &wrong_class, &model_a, 3u, 8u, 32u, 8u) ||
+        cuda_stream_expert_persistent_plan_build(
+            &state, &wrong_class, one_id, 1u, &rejected) ||
+        !cuda_stream_expert_persistent_state_equal(&state, &before)) {
+        return 0;
+    }
+    /* Exact class identity matters even when 2*gate+down has the same total
+     * bytes per slot (16/8 and 12/16 are both 40 bytes). */
+    ds4_gpu_stream_expert_table same_total_wrong_class = {};
+    if (!cuda_stream_expert_persistent_test_table_make(
+            &same_total_wrong_class, &model_a, 3u, 8u, 12u, 16u) ||
+        cuda_stream_expert_persistent_plan_build(
+            &state, &same_total_wrong_class, one_id, 1u, &rejected) ||
+        !cuda_stream_expert_persistent_state_equal(&state, &before)) {
+        return 0;
+    }
+
+    const int32_t over_budget[3] = {0, 1, 2};
+    if (cuda_stream_expert_persistent_plan_build(
+            &state, &table, over_budget, 3u, &rejected) ||
+        !cuda_stream_expert_persistent_state_equal(&state, &before)) {
+        return 0;
+    }
+
+    /* Same expert number in another model/layer/table is a miss, never an
+     * aliasing hit.  The discarded transaction must preserve the old key. */
+    ds4_gpu_stream_expert_table other_key = {};
+    if (!cuda_stream_expert_persistent_test_table_make(
+            &other_key, &model_b, 4u, 8u, 16u, 8u)) {
+        return 0;
+    }
+    const uint64_t key_misses_before =
+        g_stream_expert_persistent_key_misses.load();
+    cuda_stream_expert_persistent_plan key_plan = {};
+    if (!cuda_stream_expert_persistent_plan_build(
+            &state, &other_key, one_id, 1u, &key_plan) ||
+        key_plan.hit_count != 0u || key_plan.miss_count != 1u ||
+        key_plan.loads.size() != 1u || !key_plan.loads[0].had_victim ||
+        key_plan.loads[0].key.model_map != &model_b ||
+        key_plan.loads[0].victim.model_map != &model_a ||
+        g_stream_expert_persistent_key_misses.load() !=
+            key_misses_before + 1u ||
+        !cuda_stream_expert_persistent_state_equal(&state, &before)) {
+        return 0;
+    }
+    cuda_stream_expert_persistent_plan_rollback(&key_plan);
+    if (!cuda_stream_expert_persistent_state_equal(&state, &before)) return 0;
+
+    ds4_gpu_stream_expert_table bad_range = table;
+    bad_range.gate_offset = UINT64_MAX - 7u;
+    if (cuda_stream_expert_persistent_plan_build(
+            &state, &bad_range, one_id, 1u, &rejected) ||
+        !cuda_stream_expert_persistent_state_equal(&state, &before)) {
+        return 0;
+    }
+
+    cuda_stream_expert_persistent_class impossible = {};
+    if (cuda_stream_expert_persistent_class_make(
+            &impossible, 2u, UINT64_MAX, 1u)) {
+        return 0;
+    }
+
+    /* A wrapped LRU clock is fail-closed on the private transaction. */
+    cuda_stream_expert_persistent_state wrapped = {};
+    cuda_stream_expert_persistent_state wrapped_before = {};
+    if (!cuda_stream_expert_persistent_state_copy(&wrapped, &state)) return 0;
+    wrapped.lru_clock = UINT64_MAX;
+    if (!cuda_stream_expert_persistent_state_copy(
+            &wrapped_before, &wrapped)) {
+        return 0;
+    }
+    if (cuda_stream_expert_persistent_plan_build(
+            &wrapped, &table, one_id, 1u, &rejected) ||
+        !cuda_stream_expert_persistent_state_equal(
+            &wrapped, &wrapped_before)) {
+        return 0;
+    }
+    return 1;
+}
+
+static int cuda_stream_expert_persistent_test_dense_index(void) {
+    unsigned char model_a = 0;
+    unsigned char model_b = 0;
+    ds4_gpu_stream_expert_table table_a = {};
+    ds4_gpu_stream_expert_table table_b = {};
+    ds4_gpu_stream_expert_table table_c = {};
+    cuda_stream_expert_persistent_state state = {};
+    if (!cuda_stream_expert_persistent_test_table_make(
+            &table_a, &model_a, 3u, 8u, 16u, 8u) ||
+        !cuda_stream_expert_persistent_test_table_make(
+            &table_b, &model_a, 4u, 8u, 16u, 8u) ||
+        !cuda_stream_expert_persistent_test_table_make(
+            &table_c, &model_b, 3u, 8u, 16u, 8u) ||
+        !cuda_stream_expert_persistent_state_init(&state, 4u, 16u, 8u)) {
+        return 0;
+    }
+
+    const int32_t ids_a[2] = {0, 1};
+    cuda_stream_expert_persistent_plan fill_a = {};
+    if (!cuda_stream_expert_persistent_plan_build(
+            &state, &table_a, ids_a, 2u, &fill_a) ||
+        !cuda_stream_expert_persistent_plan_commit(&state, &fill_a)) {
+        return 0;
+    }
+    const int32_t ids_b[2] = {0, 2};
+    cuda_stream_expert_persistent_plan fill_b = {};
+    if (!cuda_stream_expert_persistent_plan_build(
+            &state, &table_b, ids_b, 2u, &fill_b) ||
+        !cuda_stream_expert_persistent_plan_commit(&state, &fill_b)) {
+        return 0;
+    }
+
+    cuda_stream_expert_persistent_state before = {};
+    if (!cuda_stream_expert_persistent_state_copy(&before, &state)) return 0;
+    const uint64_t key_misses_before =
+        g_stream_expert_persistent_key_misses.load();
+
+    /* Slot zero and slot two deliberately contain expert zero from different
+     * layers of the same model.  The dense expert index must still choose the
+     * exact full key, preserve first-seen dedup order, and leave the live
+     * state untouched. */
+    const int32_t repeated_a[6] = {0, 0, 1, 0, 1, 1};
+    cuda_stream_expert_persistent_plan hit_a = {};
+    if (!cuda_stream_expert_persistent_plan_build(
+            &state, &table_a, repeated_a, 6u, &hit_a) ||
+        hit_a.unique_count != 2u || hit_a.hit_count != 2u ||
+        hit_a.miss_count != 0u || hit_a.duplicate_count != 4u ||
+        !hit_a.loads.empty() || hit_a.slot_base != 0u ||
+        hit_a.weight_domain != 2u || hit_a.remap.size() != 6u ||
+        hit_a.remap[0] != 0u || hit_a.remap[1] != 0u ||
+        hit_a.remap[2] != 1u || hit_a.remap[3] != 0u ||
+        hit_a.remap[4] != 1u || hit_a.remap[5] != 1u ||
+        g_stream_expert_persistent_key_misses.load() != key_misses_before ||
+        !cuda_stream_expert_persistent_state_equal(&state, &before)) {
+        return 0;
+    }
+    cuda_stream_expert_persistent_plan_rollback(&hit_a);
+
+    const int32_t repeated_b[3] = {0, 2, 0};
+    cuda_stream_expert_persistent_plan hit_b = {};
+    if (!cuda_stream_expert_persistent_plan_build(
+            &state, &table_b, repeated_b, 3u, &hit_b) ||
+        hit_b.unique_count != 2u || hit_b.hit_count != 2u ||
+        hit_b.miss_count != 0u || hit_b.duplicate_count != 1u ||
+        !hit_b.loads.empty() || hit_b.slot_base != 2u ||
+        hit_b.weight_domain != 2u || hit_b.remap.size() != 3u ||
+        hit_b.remap[0] != 0u || hit_b.remap[1] != 1u ||
+        hit_b.remap[2] != 0u ||
+        g_stream_expert_persistent_key_misses.load() != key_misses_before ||
+        !cuda_stream_expert_persistent_state_equal(&state, &before)) {
+        return 0;
+    }
+    cuda_stream_expert_persistent_plan_rollback(&hit_b);
+
+    /* A third full key with the same dense expert id is a real miss.  Count
+     * it once even though two resident entries share that expert number. */
+    const int32_t id_c[1] = {0};
+    cuda_stream_expert_persistent_plan miss_c = {};
+    if (!cuda_stream_expert_persistent_plan_build(
+            &state, &table_c, id_c, 1u, &miss_c) ||
+        miss_c.hit_count != 0u || miss_c.miss_count != 1u ||
+        miss_c.loads.size() != 1u || !miss_c.loads[0].had_victim ||
+        g_stream_expert_persistent_key_misses.load() !=
+            key_misses_before + 1u ||
+        !cuda_stream_expert_persistent_state_equal(&state, &before)) {
+        return 0;
+    }
+    cuda_stream_expert_persistent_plan_rollback(&miss_c);
+    return cuda_stream_expert_persistent_state_equal(&state, &before);
+}
+
+static int cuda_stream_expert_persistent_test_prefill_index(void) {
+    enum {
+        test_n_expert = 384,
+        test_n_selected = 8192 * 6,
+    };
+    unsigned char model_marker = 0;
+    ds4_gpu_stream_expert_table table = {};
+    cuda_stream_expert_persistent_state state = {};
+    cuda_stream_expert_persistent_state before = {};
+    std::vector<int32_t> selected;
+    try {
+        selected.resize(test_n_selected);
+    } catch (...) {
+        return 0;
+    }
+    /* Five is coprime to 384, so the first 384 entries visit every expert
+     * exactly once and subsequent entries exercise dense duplicate lookup. */
+    for (uint32_t i = 0; i < test_n_selected; i++) {
+        selected[i] = (int32_t)((i * 5u) % test_n_expert);
+    }
+    if (!cuda_stream_expert_persistent_test_table_make(
+            &table, &model_marker, 9u, test_n_expert, 16u, 8u) ||
+        !cuda_stream_expert_persistent_state_init(
+            &state, test_n_expert, 16u, 8u) ||
+        !cuda_stream_expert_persistent_state_copy(&before, &state)) {
+        return 0;
+    }
+
+    cuda_stream_expert_persistent_plan prefill = {};
+    if (!cuda_stream_expert_persistent_plan_build(
+            &state, &table, selected.data(), test_n_selected, &prefill) ||
+        prefill.unique_count != test_n_expert ||
+        prefill.hit_count != 0u || prefill.miss_count != test_n_expert ||
+        prefill.duplicate_count != test_n_selected - test_n_expert ||
+        prefill.loads.size() != test_n_expert ||
+        prefill.remap.size() != test_n_selected ||
+        prefill.slot_base != 0u ||
+        prefill.weight_domain != test_n_expert ||
+        !cuda_stream_expert_persistent_state_equal(&state, &before)) {
+        return 0;
+    }
+    for (uint32_t i = 0; i < test_n_selected; i++) {
+        if (prefill.remap[i] != i % test_n_expert) return 0;
+    }
+    for (uint32_t u = 0; u < test_n_expert; u++) {
+        if (prefill.loads[u].slot != u ||
+            prefill.loads[u].key.expert_id != (u * 5u) % test_n_expert) {
+            return 0;
+        }
+    }
+    cuda_stream_expert_persistent_plan_rollback(&prefill);
+    if (!cuda_stream_expert_persistent_state_equal(&state, &before)) return 0;
+
+    const int32_t invalid_low[1] = {-1};
+    const int32_t invalid_high[1] = {test_n_expert};
+    cuda_stream_expert_persistent_plan rejected = {};
+    if (cuda_stream_expert_persistent_plan_build(
+            &state, &table, invalid_low, 1u, &rejected) ||
+        cuda_stream_expert_persistent_plan_build(
+            &state, &table, invalid_high, 1u, &rejected) ||
+        !cuda_stream_expert_persistent_state_equal(&state, &before)) {
+        return 0;
+    }
+    return 1;
+}
+
+extern "C" int ds4_cuda_test_stream_expert_persistent_planner(void) {
+    g_stream_expert_persistent_oracle_runs.fetch_add(
+        1, std::memory_order_relaxed);
+    const int ok = cuda_stream_expert_persistent_test_basic() &&
+                   cuda_stream_expert_persistent_test_protection() &&
+                   cuda_stream_expert_persistent_test_rejections() &&
+                   cuda_stream_expert_persistent_test_dense_index() &&
+                   cuda_stream_expert_persistent_test_prefill_index();
+    if (!ok) {
+        g_stream_expert_persistent_oracle_failures.fetch_add(
+            1, std::memory_order_relaxed);
+    }
+    return ok;
+}
+
+struct ds4_cuda_iq2_ssd_grouped_report;
+typedef struct ds4_cuda_iq2_ssd_grouped_report
+    ds4_cuda_iq2_ssd_grouped_report;
+typedef struct {
+    uint64_t candidates;
+    uint64_t eligible;
+    uint64_t attempts;
+    uint64_t completed;
+    uint64_t not_applicable;
+    uint64_t safe_fallbacks;
+    uint64_t failures;
+    uint64_t required_failures;
+    uint64_t upload_waits;
+    uint64_t lease_waits;
+    uint64_t lease_records;
+    uint64_t lease_drains;
+    int enabled;
+    int required;
+    int stats;
+} cuda_iq2_ssd_grouped_report_layout;
+
+static void cuda_iq2_ssd_grouped_resolve_policy(
+        int enable, int disable, int require, int stats,
+        int *enabled_out, int *required_out, int *stats_out) {
+    const int disabled = disable != 0;
+    if (enabled_out) *enabled_out = !disabled && (enable || require);
+    if (required_out) *required_out = !disabled && require;
+    if (stats_out) *stats_out = !disabled && stats;
+}
+
+extern "C" int ds4_cuda_test_iq2_ssd_grouped_policy(
+        int enable, int disable, int require, int stats,
+        int *enabled_out, int *required_out, int *stats_out) {
+    if (!enabled_out || !required_out || !stats_out) return 0;
+    cuda_iq2_ssd_grouped_resolve_policy(
+        enable, disable, require, stats,
+        enabled_out, required_out, stats_out);
+    return 1;
+}
+
+/* Keep the production gate decomposed so host-only policy tests can prove
+ * that every exclusion remains fail-safe without spoofing CUDA properties.
+ * Raw-layout validation itself is checked separately against exact GGUF
+ * strides in routed_moe_launch(). */
+static int cuda_iq2_ssd_grouped_eligible_values(
+        int enabled, int ssd_streaming, int single_gpu, int gb10,
+        int quality, int owned_filtered, int capture, int mmq,
+        uint32_t n_tokens, uint32_t n_expert, int top6_unique,
+        int raw_layout, int binding_valid) {
+    return enabled && ssd_streaming && single_gpu && gb10 && !quality &&
+           !owned_filtered && !capture && mmq && n_tokens >= 32u &&
+           n_expert == 6u && top6_unique && raw_layout && binding_valid;
+}
+
+extern "C" int ds4_cuda_test_iq2_ssd_grouped_eligibility(
+        int enabled, int ssd_streaming, int single_gpu, int gb10,
+        int quality, int owned_filtered, int capture, int mmq,
+        uint32_t n_tokens, uint32_t n_expert, int top6_unique,
+        int raw_layout, int binding_valid) {
+    return cuda_iq2_ssd_grouped_eligible_values(
+        enabled, ssd_streaming, single_gpu, gb10, quality,
+        owned_filtered, capture, mmq, n_tokens, n_expert,
+        top6_unique, raw_layout, binding_valid);
+}
+
+static int cuda_iq2_ssd_grouped_candidate_values(
+        int iq2_path, int ssd_streaming, int allow_streaming,
+        int owned_filtered, uint32_t n_tokens, uint32_t n_expert,
+        int top6_unique, int raw_layout, int binding_valid) {
+    return iq2_path && ssd_streaming && allow_streaming &&
+           !owned_filtered && n_tokens >= 32u && n_expert == 6u &&
+           top6_unique && raw_layout && binding_valid;
+}
+
+extern "C" int ds4_cuda_test_iq2_ssd_grouped_candidate(
+        int iq2_path, int ssd_streaming, int allow_streaming,
+        int owned_filtered, uint32_t n_tokens, uint32_t n_expert,
+        int top6_unique, int raw_layout, int binding_valid) {
+    return cuda_iq2_ssd_grouped_candidate_values(
+        iq2_path, ssd_streaming, allow_streaming, owned_filtered,
+        n_tokens, n_expert, top6_unique, raw_layout, binding_valid);
+}
+
+static void cuda_iq2_ssd_grouped_report_at_exit(void) {
+    fprintf(stderr,
+            "ds4: CUDA IQ2 SSD grouped MMQ: candidates=%llu eligible=%llu "
+            "attempts=%llu completed=%llu not_applicable=%llu "
+            "safe_fallbacks=%llu failures=%llu required_failures=%llu "
+            "upload_waits=%llu lease_waits=%llu lease_records=%llu "
+            "lease_drains=%llu\n",
+            (unsigned long long)g_iq2_ssd_grouped_candidates.load(),
+            (unsigned long long)g_iq2_ssd_grouped_eligible.load(),
+            (unsigned long long)g_iq2_ssd_grouped_attempts.load(),
+            (unsigned long long)g_iq2_ssd_grouped_completed.load(),
+            (unsigned long long)g_iq2_ssd_grouped_not_applicable.load(),
+            (unsigned long long)g_iq2_ssd_grouped_safe_fallbacks.load(),
+            (unsigned long long)g_iq2_ssd_grouped_failures.load(),
+            (unsigned long long)g_iq2_ssd_grouped_required_failures.load(),
+            (unsigned long long)g_iq2_ssd_grouped_upload_waits.load(),
+            (unsigned long long)g_iq2_ssd_grouped_lease_waits.load(),
+            (unsigned long long)g_iq2_ssd_grouped_lease_records.load(),
+            (unsigned long long)g_iq2_ssd_grouped_lease_drains.load());
+}
+
+static void cuda_iq2_ssd_grouped_init(void) {
+    const int enable = cuda_stream_selected_env_flag(
+        "DS4_CUDA_ENABLE_IQ2_XXS_SSD_PREFILL_MMQ");
+    const int disable = cuda_stream_selected_env_flag(
+        "DS4_CUDA_DISABLE_IQ2_XXS_SSD_PREFILL_MMQ") ||
+        cuda_stream_selected_env_flag(
+            "DS4_CUDA_NO_IQ2_XXS_SSD_PREFILL_MMQ");
+    const int require = cuda_stream_selected_env_flag(
+        "DS4_CUDA_REQUIRE_IQ2_XXS_SSD_PREFILL_MMQ");
+    const int stats = cuda_stream_selected_env_flag(
+        "DS4_CUDA_IQ2_XXS_SSD_PREFILL_MMQ_STATS");
+    cuda_iq2_ssd_grouped_resolve_policy(
+        enable, disable, require, stats,
+        &g_iq2_ssd_grouped_enabled,
+        &g_iq2_ssd_grouped_required,
+        &g_iq2_ssd_grouped_stats);
+    if ((enable || require || stats) &&
+        !g_iq2_ssd_grouped_report_registered) {
+        g_iq2_ssd_grouped_report_registered = 1;
+        (void)atexit(cuda_iq2_ssd_grouped_report_at_exit);
+    }
+    if (g_iq2_ssd_grouped_enabled) {
+        fprintf(stderr,
+                "ds4: CUDA IQ2_XXS/Q2_K SSD grouped prefill MMQ enabled%s\n",
+                g_iq2_ssd_grouped_required ? " (required)" : "");
+    } else if (disable && (enable || require)) {
+        fprintf(stderr,
+                "ds4: CUDA IQ2 SSD grouped MMQ disabled by rollback "
+                "override\n");
+    }
+}
+
+static int cuda_iq2_ssd_grouped_enabled(void) {
+    std::call_once(g_iq2_ssd_grouped_once, cuda_iq2_ssd_grouped_init);
+    return g_iq2_ssd_grouped_enabled;
+}
+
+static int cuda_iq2_ssd_grouped_required(void) {
+    std::call_once(g_iq2_ssd_grouped_once, cuda_iq2_ssd_grouped_init);
+    return g_iq2_ssd_grouped_required;
+}
+
+extern "C" void ds4_cuda_iq2_ssd_grouped_get_report(
+        ds4_cuda_iq2_ssd_grouped_report *report) {
+    if (!report) return;
+    cuda_iq2_ssd_grouped_report_layout out = {};
+    out.candidates = g_iq2_ssd_grouped_candidates.load();
+    out.eligible = g_iq2_ssd_grouped_eligible.load();
+    out.attempts = g_iq2_ssd_grouped_attempts.load();
+    out.completed = g_iq2_ssd_grouped_completed.load();
+    out.not_applicable = g_iq2_ssd_grouped_not_applicable.load();
+    out.safe_fallbacks = g_iq2_ssd_grouped_safe_fallbacks.load();
+    out.failures = g_iq2_ssd_grouped_failures.load();
+    out.required_failures = g_iq2_ssd_grouped_required_failures.load();
+    out.upload_waits = g_iq2_ssd_grouped_upload_waits.load();
+    out.lease_waits = g_iq2_ssd_grouped_lease_waits.load();
+    out.lease_records = g_iq2_ssd_grouped_lease_records.load();
+    out.lease_drains = g_iq2_ssd_grouped_lease_drains.load();
+    out.enabled = cuda_iq2_ssd_grouped_enabled();
+    out.required = cuda_iq2_ssd_grouped_required();
+    out.stats = g_iq2_ssd_grouped_stats;
+    memcpy(report, &out, sizeof(out));
+}
+
+static void cuda_stream_selected_batch_io_resolve_policy(
+        int enable, int disable, int require, int oracle,
+        int *enabled_out, int *required_out, int *oracle_out) {
+    const int disabled = disable != 0;
+    if (enabled_out) {
+        *enabled_out = !disabled && (enable || require || oracle);
+    }
+    if (required_out) *required_out = !disabled && require;
+    if (oracle_out) *oracle_out = !disabled && oracle;
+}
+
+extern "C" int ds4_cuda_test_stream_selected_batch_policy(
+        int enable, int disable, int require, int oracle,
+        int *enabled_out, int *required_out, int *oracle_out) {
+    if (!enabled_out || !required_out || !oracle_out) return 0;
+    cuda_stream_selected_batch_io_resolve_policy(
+        enable, disable, require, oracle,
+        enabled_out, required_out, oracle_out);
+    return 1;
+}
+
+static void cuda_stream_selected_batch_io_report_at_exit(void) {
+    fprintf(stderr,
+            "ds4: CUDA selected-expert batched I/O: candidates=%llu "
+            "attempts=%llu completed=%llu legacy=%llu safe_fallbacks=%llu "
+            "failures=%llu required_failures=%llu oracle_runs=%llu "
+            "oracle_failures=%llu tasks=%llu segments=%llu reads=%llu "
+            "bytes=%.2f MiB\n",
+            (unsigned long long)g_stream_selected_batch_io_candidates,
+            (unsigned long long)g_stream_selected_batch_io_attempts,
+            (unsigned long long)g_stream_selected_batch_io_completed,
+            (unsigned long long)g_stream_selected_batch_io_legacy,
+            (unsigned long long)g_stream_selected_batch_io_safe_fallbacks,
+            (unsigned long long)g_stream_selected_batch_io_failures,
+            (unsigned long long)g_stream_selected_batch_io_required_failures,
+            (unsigned long long)g_stream_selected_batch_io_oracle_runs,
+            (unsigned long long)g_stream_selected_batch_io_oracle_failures,
+            (unsigned long long)g_stream_selected_batch_io_tasks,
+            (unsigned long long)g_stream_selected_batch_io_segments,
+            (unsigned long long)g_stream_selected_batch_io_reads,
+            (double)g_stream_selected_batch_io_bytes / 1048576.0);
+}
+
+static void cuda_stream_selected_batch_io_init(void) {
+    const int enable = cuda_stream_selected_env_flag(
+        "DS4_CUDA_ENABLE_STREAMING_SELECTED_BATCH_IO");
+    const int disable = cuda_stream_selected_env_flag(
+        "DS4_CUDA_DISABLE_STREAMING_SELECTED_BATCH_IO") ||
+        cuda_stream_selected_env_flag(
+            "DS4_CUDA_NO_STREAMING_SELECTED_BATCH_IO");
+    const int require = cuda_stream_selected_env_flag(
+        "DS4_CUDA_REQUIRE_STREAMING_SELECTED_BATCH_IO");
+    const int oracle = cuda_stream_selected_env_flag(
+        "DS4_CUDA_STREAMING_SELECTED_BATCH_IO_ORACLE");
+    cuda_stream_selected_batch_io_resolve_policy(
+        enable, disable, require, oracle,
+        &g_stream_selected_batch_io_enabled,
+        &g_stream_selected_batch_io_required,
+        &g_stream_selected_batch_io_oracle);
+    if ((enable || require || oracle) &&
+        !g_stream_selected_batch_io_report_registered) {
+        g_stream_selected_batch_io_report_registered = 1;
+        (void)atexit(cuda_stream_selected_batch_io_report_at_exit);
+    }
+    if (g_stream_selected_batch_io_enabled) {
+        fprintf(stderr,
+                "ds4: CUDA selected-expert batched I/O enabled%s%s\n",
+                g_stream_selected_batch_io_required ? " (required)" : "",
+                g_stream_selected_batch_io_oracle ?
+                    " with full byte oracle" : "");
+    } else if (disable && (enable || require || oracle)) {
+        fprintf(stderr,
+                "ds4: CUDA selected-expert batched I/O disabled by rollback"
+                " override\n");
+    }
+}
+
+static int cuda_stream_selected_batch_io_requested(void) {
+    std::call_once(g_stream_selected_batch_io_once,
+                   cuda_stream_selected_batch_io_init);
+    return g_stream_selected_batch_io_enabled;
+}
+
+static int cuda_stream_selected_batch_io_require_requested(void) {
+    std::call_once(g_stream_selected_batch_io_once,
+                   cuda_stream_selected_batch_io_init);
+    return g_stream_selected_batch_io_required;
+}
+
+static int cuda_stream_selected_batch_io_oracle_requested(void) {
+    std::call_once(g_stream_selected_batch_io_once,
+                   cuda_stream_selected_batch_io_init);
+    return g_stream_selected_batch_io_oracle;
+}
+
+extern "C" void ds4_cuda_stream_selected_batch_io_get_report(
+        ds4_cuda_stream_selected_batch_io_report *report) {
+    if (!report) return;
+    cuda_stream_selected_batch_io_report_layout out = {};
+    out.candidates = g_stream_selected_batch_io_candidates;
+    out.attempts = g_stream_selected_batch_io_attempts;
+    out.completed = g_stream_selected_batch_io_completed;
+    out.legacy_batches = g_stream_selected_batch_io_legacy;
+    out.safe_fallbacks = g_stream_selected_batch_io_safe_fallbacks;
+    out.failures = g_stream_selected_batch_io_failures;
+    out.required_failures =
+        g_stream_selected_batch_io_required_failures;
+    out.oracle_runs = g_stream_selected_batch_io_oracle_runs;
+    out.oracle_failures = g_stream_selected_batch_io_oracle_failures;
+    out.tasks = g_stream_selected_batch_io_tasks;
+    out.segments = g_stream_selected_batch_io_segments;
+    out.reads = g_stream_selected_batch_io_reads;
+    out.bytes = g_stream_selected_batch_io_bytes;
+    out.enabled = cuda_stream_selected_batch_io_requested();
+    out.required = cuda_stream_selected_batch_io_require_requested();
+    out.oracle = cuda_stream_selected_batch_io_oracle_requested();
+    memcpy(report, &out, sizeof(out));
+}
+
+struct ds4_cuda_stream_selected_event_pipeline_report;
+typedef struct ds4_cuda_stream_selected_event_pipeline_report
+    ds4_cuda_stream_selected_event_pipeline_report;
+typedef struct {
+    uint64_t candidates;
+    uint64_t signals;
+    uint64_t readbacks;
+    uint64_t uploads;
+    uint64_t compute_waits;
+    uint64_t safe_fallbacks;
+    uint64_t failures;
+    uint64_t required_failures;
+    uint64_t oracle_runs;
+    uint64_t oracle_failures;
+    int enabled;
+    int required;
+    int oracle;
+} cuda_stream_selected_event_pipeline_report_layout;
+
+static void cuda_stream_selected_event_pipeline_resolve_policy(
+        int enable, int disable, int require, int oracle,
+        int *enabled_out, int *required_out, int *oracle_out) {
+    const int disabled = disable != 0;
+    if (enabled_out) *enabled_out = !disabled && (enable || require || oracle);
+    if (required_out) *required_out = !disabled && require;
+    if (oracle_out) *oracle_out = !disabled && oracle;
+}
+
+extern "C" int ds4_cuda_test_stream_selected_event_pipeline_policy(
+        int enable, int disable, int require, int oracle,
+        int *enabled_out, int *required_out, int *oracle_out) {
+    if (!enabled_out || !required_out || !oracle_out) return 0;
+    cuda_stream_selected_event_pipeline_resolve_policy(
+        enable, disable, require, oracle,
+        enabled_out, required_out, oracle_out);
+    return 1;
+}
+
+static void cuda_stream_selected_event_pipeline_report_at_exit(void) {
+    fprintf(stderr,
+            "ds4: CUDA selected-expert event pipeline: candidates=%llu "
+            "signals=%llu readbacks=%llu uploads=%llu compute_waits=%llu "
+            "safe_fallbacks=%llu failures=%llu required_failures=%llu "
+            "oracle_runs=%llu oracle_failures=%llu\n",
+            (unsigned long long)g_stream_selected_event_candidates.load(),
+            (unsigned long long)g_stream_selected_event_signals.load(),
+            (unsigned long long)g_stream_selected_event_readbacks.load(),
+            (unsigned long long)g_stream_selected_event_uploads.load(),
+            (unsigned long long)g_stream_selected_event_compute_waits.load(),
+            (unsigned long long)g_stream_selected_event_safe_fallbacks.load(),
+            (unsigned long long)g_stream_selected_event_failures.load(),
+            (unsigned long long)g_stream_selected_event_required_failures.load(),
+            (unsigned long long)g_stream_selected_event_oracle_runs.load(),
+            (unsigned long long)g_stream_selected_event_oracle_failures.load());
+}
+
+static void cuda_stream_selected_event_pipeline_init(void) {
+    const int enable = cuda_stream_selected_env_flag(
+        "DS4_CUDA_ENABLE_STREAMING_SELECTED_EVENT_PIPELINE");
+    const int disable = cuda_stream_selected_env_flag(
+        "DS4_CUDA_DISABLE_STREAMING_SELECTED_EVENT_PIPELINE") ||
+        cuda_stream_selected_env_flag(
+            "DS4_CUDA_NO_STREAMING_SELECTED_EVENT_PIPELINE");
+    const int require = cuda_stream_selected_env_flag(
+        "DS4_CUDA_REQUIRE_STREAMING_SELECTED_EVENT_PIPELINE");
+    const int oracle = cuda_stream_selected_env_flag(
+        "DS4_CUDA_STREAMING_SELECTED_EVENT_PIPELINE_ORACLE");
+    const int stats = cuda_stream_selected_env_flag(
+        "DS4_CUDA_STREAMING_SELECTED_EVENT_PIPELINE_STATS");
+    cuda_stream_selected_event_pipeline_resolve_policy(
+        enable, disable, require, oracle,
+        &g_stream_selected_event_pipeline_enabled,
+        &g_stream_selected_event_pipeline_required,
+        &g_stream_selected_event_pipeline_oracle);
+    if ((enable || require || oracle || stats) &&
+        !g_stream_selected_event_pipeline_report_registered) {
+        g_stream_selected_event_pipeline_report_registered = 1;
+        (void)atexit(cuda_stream_selected_event_pipeline_report_at_exit);
+    }
+    if (g_stream_selected_event_pipeline_enabled) {
+        fprintf(stderr,
+                "ds4: CUDA selected-expert event pipeline enabled%s%s\n",
+                g_stream_selected_event_pipeline_required ? " (required)" : "",
+                g_stream_selected_event_pipeline_oracle ?
+                    " with synchronous oracle" : "");
+    } else if (disable && (enable || require || oracle)) {
+        fprintf(stderr,
+                "ds4: CUDA selected-expert event pipeline disabled by "
+                "rollback override\n");
+    }
+}
+
+extern "C" int ds4_gpu_cuda_stream_selected_event_pipeline_enabled(void) {
+    std::call_once(g_stream_selected_event_pipeline_once,
+                   cuda_stream_selected_event_pipeline_init);
+    return g_stream_selected_event_pipeline_enabled;
+}
+
+extern "C" int ds4_gpu_cuda_stream_selected_event_pipeline_required(void) {
+    std::call_once(g_stream_selected_event_pipeline_once,
+                   cuda_stream_selected_event_pipeline_init);
+    return g_stream_selected_event_pipeline_required;
+}
+
+static int cuda_stream_selected_event_pipeline_oracle_requested(void) {
+    std::call_once(g_stream_selected_event_pipeline_once,
+                   cuda_stream_selected_event_pipeline_init);
+    return g_stream_selected_event_pipeline_oracle;
+}
+
+extern "C" void ds4_gpu_cuda_stream_selected_event_note_candidate(void) {
+    g_stream_selected_event_candidates.fetch_add(1, std::memory_order_relaxed);
+}
+
+extern "C" void ds4_gpu_cuda_stream_selected_event_note_fallback(void) {
+    g_stream_selected_event_safe_fallbacks.fetch_add(
+        1, std::memory_order_relaxed);
+}
+
+extern "C" void ds4_gpu_cuda_stream_selected_event_note_failure(
+        int required) {
+    g_stream_selected_event_failures.fetch_add(1, std::memory_order_relaxed);
+    if (required) {
+        g_stream_selected_event_required_failures.fetch_add(
+            1, std::memory_order_relaxed);
+    }
+}
+
+extern "C" void ds4_cuda_stream_selected_event_pipeline_get_report(
+        ds4_cuda_stream_selected_event_pipeline_report *report) {
+    if (!report) return;
+    cuda_stream_selected_event_pipeline_report_layout out = {};
+    out.candidates = g_stream_selected_event_candidates.load();
+    out.signals = g_stream_selected_event_signals.load();
+    out.readbacks = g_stream_selected_event_readbacks.load();
+    out.uploads = g_stream_selected_event_uploads.load();
+    out.compute_waits = g_stream_selected_event_compute_waits.load();
+    out.safe_fallbacks = g_stream_selected_event_safe_fallbacks.load();
+    out.failures = g_stream_selected_event_failures.load();
+    out.required_failures = g_stream_selected_event_required_failures.load();
+    out.oracle_runs = g_stream_selected_event_oracle_runs.load();
+    out.oracle_failures = g_stream_selected_event_oracle_failures.load();
+    out.enabled = ds4_gpu_cuda_stream_selected_event_pipeline_enabled();
+    out.required = ds4_gpu_cuda_stream_selected_event_pipeline_required();
+    out.oracle = cuda_stream_selected_event_pipeline_oracle_requested();
+    memcpy(report, &out, sizeof(out));
+}
+
+static void cuda_stream_selected_event_pipeline_release(void) {
+    int previous_device = -1;
+    (void)cudaGetDevice(&previous_device);
+    if (g_stream_selected_event_owner_device >= 0) {
+        (void)cudaSetDevice(g_stream_selected_event_owner_device);
+    }
+    if (g_stream_selected_upload_stream) {
+        (void)cudaStreamSynchronize(g_stream_selected_upload_stream);
+    }
+    if (g_stream_selected_readback_stream) {
+        (void)cudaStreamSynchronize(g_stream_selected_readback_stream);
+        (void)cudaStreamDestroy(g_stream_selected_readback_stream);
+        g_stream_selected_readback_stream = NULL;
+    }
+    if (g_stream_selected_readback_stage) {
+        (void)cudaFreeHost(g_stream_selected_readback_stage);
+        g_stream_selected_readback_stage = NULL;
+    }
+    g_stream_selected_readback_stage_capacity = 0;
+    if (g_stream_selected_compute_ready_event) {
+        (void)cudaEventDestroy(g_stream_selected_compute_ready_event);
+        g_stream_selected_compute_ready_event = NULL;
+    }
+    if (g_stream_selected_readback_done_event) {
+        (void)cudaEventDestroy(g_stream_selected_readback_done_event);
+        g_stream_selected_readback_done_event = NULL;
+    }
+    if (g_stream_selected_upload_done_event) {
+        (void)cudaEventDestroy(g_stream_selected_upload_done_event);
+        g_stream_selected_upload_done_event = NULL;
+    }
+    g_stream_selected_event_owner_device = -1;
+    uint64_t compute_value = ++g_stream_selected_compute_event_value;
+    if (compute_value == 0) ++g_stream_selected_compute_event_value;
+    uint64_t upload_value = ++g_stream_selected_upload_event_value;
+    if (upload_value == 0) ++g_stream_selected_upload_event_value;
+    if (previous_device >= 0) (void)cudaSetDevice(previous_device);
+}
+
+static int cuda_stream_selected_event_pipeline_ensure(void) {
+    if (g_n_gpus != 1) return 0;
+    int device = -1;
+    if (cudaGetDevice(&device) != cudaSuccess) {
+        (void)cudaGetLastError();
+        return 0;
+    }
+    if (g_stream_selected_event_owner_device >= 0 &&
+        g_stream_selected_event_owner_device != device) {
+        cuda_stream_selected_event_pipeline_release();
+    }
+    if (g_stream_selected_readback_stream &&
+        g_stream_selected_compute_ready_event &&
+        g_stream_selected_readback_done_event &&
+        g_stream_selected_upload_done_event) {
+        return 1;
+    }
+    cuda_stream_selected_event_pipeline_release();
+    g_stream_selected_event_owner_device = device;
+    cudaError_t err = cudaStreamCreateWithFlags(
+        &g_stream_selected_readback_stream, cudaStreamNonBlocking);
+    if (err == cudaSuccess) {
+        err = cudaEventCreateWithFlags(
+            &g_stream_selected_compute_ready_event, cudaEventDisableTiming);
+    }
+    if (err == cudaSuccess) {
+        err = cudaEventCreateWithFlags(
+            &g_stream_selected_readback_done_event, cudaEventDisableTiming);
+    }
+    if (err == cudaSuccess) {
+        err = cudaEventCreateWithFlags(
+            &g_stream_selected_upload_done_event, cudaEventDisableTiming);
+    }
+    if (err != cudaSuccess) {
+        fprintf(stderr,
+                "ds4: CUDA selected event resource creation failed: %s\n",
+                cudaGetErrorString(err));
+        (void)cudaGetLastError();
+        cuda_stream_selected_event_pipeline_release();
+        return 0;
+    }
+    return 1;
+}
+
+/* A compact selected cache is a transient binding, not ordinary scratch:
+ * the next SSD load overwrites the same gate/up/down arrays and remap.  The
+ * reusable consume event is recorded after the last grouped-MMQ consumer and
+ * imported into the upload stream before the following H2D epoch.  Allocation
+ * growth and teardown use the host drain because cudaFree cannot be ordered
+ * behind an event in the upload stream. */
+static int cuda_stream_selected_consume_ensure_locked(void) {
+    if (g_n_gpus != 1 || g_stream_selected_consume_poisoned) return 0;
+    const int owner = g_gpu[0].device_id;
+    if (cudaSetDevice(owner) != cudaSuccess) {
+        (void)cudaGetLastError();
+        return 0;
+    }
+    if (g_stream_selected_consume_done_event &&
+        g_stream_selected_consume_owner_device == owner) {
+        return 1;
+    }
+    if (g_stream_selected_consume_pending &&
+        g_stream_selected_consume_done_event) {
+        const cudaError_t sync_err =
+            cudaEventSynchronize(g_stream_selected_consume_done_event);
+        if (sync_err != cudaSuccess) {
+            fprintf(stderr,
+                    "ds4: CUDA compact-cache consume drain failed while "
+                    "changing owner: %s\n",
+                    cudaGetErrorString(sync_err));
+            (void)cudaGetLastError();
+            return 0;
+        }
+        g_stream_selected_consume_pending = 0;
+        g_iq2_ssd_grouped_lease_drains.fetch_add(
+            1, std::memory_order_relaxed);
+    }
+    if (g_stream_selected_consume_done_event) {
+        (void)cudaEventDestroy(g_stream_selected_consume_done_event);
+        g_stream_selected_consume_done_event = NULL;
+    }
+    const cudaError_t err = cudaEventCreateWithFlags(
+        &g_stream_selected_consume_done_event, cudaEventDisableTiming);
+    if (err != cudaSuccess) {
+        fprintf(stderr,
+                "ds4: CUDA compact-cache consume event creation failed: %s\n",
+                cudaGetErrorString(err));
+        (void)cudaGetLastError();
+        g_stream_selected_consume_owner_device = -1;
+        return 0;
+    }
+    g_stream_selected_consume_owner_device = owner;
+    return 1;
+}
+
+static int cuda_stream_selected_consume_prepare(void) {
+    std::lock_guard<std::mutex> lock(g_stream_selected_consume_mutex);
+    return cuda_stream_selected_consume_ensure_locked();
+}
+
+/* Exclusive host-side writer lease for the transient binding.  The flag is
+ * held for the entire load, publication, release or abort transaction, not
+ * just while valid is toggled.  That closes both directions of the TOCTOU:
+ * readers cannot enter after a loader observes zero readers, and a reader
+ * cannot observe partially replaced metadata while a writer is active. */
+class cuda_stream_selected_writer_guard {
+public:
+    explicit cuda_stream_selected_writer_guard(
+            std::atomic<int> *waiting_probe = nullptr) {
+        std::unique_lock<std::mutex> lock(
+            g_stream_selected_consume_mutex);
+        if (waiting_probe) {
+            waiting_probe->store(1, std::memory_order_release);
+        }
+        g_stream_selected_consume_cv.wait(lock, [] {
+            return !g_stream_selected_writer_active &&
+                   g_stream_selected_consume_host_readers == 0u;
+        });
+        g_stream_selected_writer_active = 1;
+        cuda_stream_selected_cache_invalidate();
+        held_ = true;
+    }
+
+    cuda_stream_selected_writer_guard(
+        const cuda_stream_selected_writer_guard &) = delete;
+    cuda_stream_selected_writer_guard &operator=(
+        const cuda_stream_selected_writer_guard &) = delete;
+
+    void publish_valid() {
+        std::lock_guard<std::mutex> lock(
+            g_stream_selected_consume_mutex);
+        if (!held_) return;
+        g_stream_selected_cache.valid = 1;
+        g_stream_selected_writer_active = 0;
+        held_ = false;
+        g_stream_selected_consume_cv.notify_all();
+    }
+
+    ~cuda_stream_selected_writer_guard() {
+        std::lock_guard<std::mutex> lock(
+            g_stream_selected_consume_mutex);
+        if (!held_) return;
+        cuda_stream_selected_cache_invalidate();
+        g_stream_selected_writer_active = 0;
+        held_ = false;
+        g_stream_selected_consume_cv.notify_all();
+    }
+
+private:
+    bool held_ = false;
+};
+
+static int cuda_stream_selected_consume_drain(void) {
+    std::lock_guard<std::mutex> lock(g_stream_selected_consume_mutex);
+    if (g_stream_selected_consume_poisoned) {
+        if (g_stream_selected_consume_owner_device < 0 ||
+            cudaSetDevice(g_stream_selected_consume_owner_device) !=
+                cudaSuccess) {
+            (void)cudaGetLastError();
+            return 0;
+        }
+        const cudaError_t device_err = cudaDeviceSynchronize();
+        if (device_err != cudaSuccess) {
+            fprintf(stderr,
+                    "ds4: CUDA compact-cache poisoned drain failed: %s\n",
+                    cudaGetErrorString(device_err));
+            (void)cudaGetLastError();
+            return 0;
+        }
+        g_stream_selected_consume_poisoned = 0;
+        g_stream_selected_consume_pending = 0;
+        g_iq2_ssd_grouped_lease_drains.fetch_add(
+            1, std::memory_order_relaxed);
+        return 1;
+    }
+    if (!g_stream_selected_consume_pending) return 1;
+    if (!g_stream_selected_consume_done_event ||
+        g_stream_selected_consume_owner_device < 0 ||
+        cudaSetDevice(g_stream_selected_consume_owner_device) != cudaSuccess) {
+        (void)cudaGetLastError();
+        return 0;
+    }
+    const cudaError_t err =
+        cudaEventSynchronize(g_stream_selected_consume_done_event);
+    if (err != cudaSuccess) {
+        fprintf(stderr,
+                "ds4: CUDA compact-cache consume drain failed: %s\n",
+                cudaGetErrorString(err));
+        (void)cudaGetLastError();
+        return 0;
+    }
+    g_stream_selected_consume_pending = 0;
+    g_iq2_ssd_grouped_lease_drains.fetch_add(
+        1, std::memory_order_relaxed);
+    return 1;
+}
+
+/* Caller owns cuda_stream_selected_writer_guard, so no reader can acquire
+ * these pointers while they are drained and freed. */
+static int cuda_stream_selected_cache_free_storage_writer(void) {
+    cuda_stream_selected_transient_storage *storage =
+        &g_stream_selected_transient_storage;
+    if (!g_stream_selected_writer_active ||
+        !cuda_stream_selected_consume_drain() ||
+        !cuda_stream_selected_upload_drain_checked()) {
+        /* Never free a compact binding whose last consumer could not be
+         * drained.  A bounded leak is safer than a cross-stream UAF when the
+         * CUDA context is already reporting an unrecoverable error. */
+        cuda_stream_selected_cache_invalidate();
+        storage->poisoned = 1;
+        return 0;
+    }
+    const int has_storage = storage->gate || storage->up ||
+        storage->down || storage->remap;
+    int previous_device = -1;
+    if (has_storage) (void)cudaGetDevice(&previous_device);
+    if (has_storage &&
+        (storage->owner_device < 0 ||
+         cudaSetDevice(storage->owner_device) != cudaSuccess)) {
+        (void)cudaGetLastError();
+        cuda_stream_selected_cache_invalidate();
+        storage->poisoned = 1;
+        if (previous_device >= 0) (void)cudaSetDevice(previous_device);
+        return 0;
+    }
+    if (storage->gate && cudaFree(storage->gate) != cudaSuccess) {
+        (void)cudaGetLastError();
+        storage->poisoned = 1;
+        cuda_stream_selected_cache_invalidate();
+        if (previous_device >= 0) (void)cudaSetDevice(previous_device);
+        return 0;
+    }
+    storage->gate = NULL;
+    storage->gate_capacity = 0;
+    if (storage->up && cudaFree(storage->up) != cudaSuccess) {
+        (void)cudaGetLastError();
+        storage->poisoned = 1;
+        cuda_stream_selected_cache_invalidate();
+        if (previous_device >= 0) (void)cudaSetDevice(previous_device);
+        return 0;
+    }
+    storage->up = NULL;
+    storage->up_capacity = 0;
+    if (storage->down && cudaFree(storage->down) != cudaSuccess) {
+        (void)cudaGetLastError();
+        storage->poisoned = 1;
+        cuda_stream_selected_cache_invalidate();
+        if (previous_device >= 0) (void)cudaSetDevice(previous_device);
+        return 0;
+    }
+    storage->down = NULL;
+    storage->down_capacity = 0;
+    if (storage->remap && cudaFree(storage->remap) != cudaSuccess) {
+        (void)cudaGetLastError();
+        storage->poisoned = 1;
+        cuda_stream_selected_cache_invalidate();
+        if (previous_device >= 0) (void)cudaSetDevice(previous_device);
+        return 0;
+    }
+    storage->remap = NULL;
+    storage->remap_capacity = 0;
+    storage->owner_device = -1;
+    storage->poisoned = 0;
+    if (previous_device >= 0) (void)cudaSetDevice(previous_device);
+    memset(&g_stream_selected_cache, 0, sizeof(g_stream_selected_cache));
+    g_stream_selected_cache.logical_tier = -1;
+    return 1;
+}
+
+static void cuda_stream_selected_cache_release(void) {
+    cuda_stream_selected_writer_guard writer;
+    (void)cuda_stream_selected_cache_free_storage_writer();
+}
+
+static int cuda_stream_selected_cache_bind_transient_storage(
+        int logical_tier,
+        uint64_t gate_bytes,
+        uint64_t down_bytes,
+        uint64_t remap_bytes) {
+    const cuda_stream_selected_transient_storage *storage =
+        &g_stream_selected_transient_storage;
+    if (storage->poisoned || !storage->gate || !storage->up ||
+        !storage->down || !storage->remap ||
+        storage->gate_capacity < gate_bytes ||
+        storage->up_capacity < gate_bytes ||
+        storage->down_capacity < down_bytes ||
+        storage->remap_capacity < remap_bytes) {
+        return 0;
+    }
+    g_stream_selected_cache.storage_kind =
+        CUDA_STREAM_SELECTED_STORAGE_TRANSIENT;
+    g_stream_selected_cache.gate_ptr = storage->gate;
+    g_stream_selected_cache.up_ptr = storage->up;
+    g_stream_selected_cache.down_ptr = storage->down;
+    g_stream_selected_cache.gate_capacity = storage->gate_capacity;
+    g_stream_selected_cache.up_capacity = storage->up_capacity;
+    g_stream_selected_cache.down_capacity = storage->down_capacity;
+    g_stream_selected_cache.slot_selected_ptr = storage->remap;
+    g_stream_selected_cache.slot_selected_capacity =
+        storage->remap_capacity;
+    g_stream_selected_cache.slot_selected_tensor.ptr = storage->remap;
+    g_stream_selected_cache.slot_selected_tensor.bytes = remap_bytes;
+    g_stream_selected_cache.slot_selected_tensor.owner = 0;
+    g_stream_selected_cache.slot_selected_tensor.device_id = logical_tier;
+    return 1;
+}
+
+/* Invalidate the public binding under one writer epoch, retire its transient
+ * owner, and only then retire the persistent owner it may alias in phase 3. */
+static int cuda_stream_expert_storage_release_writer(int reset_class) {
+    if (!g_stream_selected_writer_active ||
+        !cuda_stream_selected_cache_free_storage_writer()) {
+        return 0;
+    }
+    return cuda_stream_expert_persistent_arena_release(reset_class);
+}
+
+static int cuda_stream_expert_storage_release(int reset_class) {
+    cuda_stream_selected_writer_guard writer;
+    return cuda_stream_expert_storage_release_writer(reset_class);
+}
+
+static void cuda_stream_selected_consume_release(void) {
+    std::lock_guard<std::mutex> lock(g_stream_selected_consume_mutex);
+    if (g_stream_selected_consume_owner_device >= 0) {
+        (void)cudaSetDevice(g_stream_selected_consume_owner_device);
+    }
+    if (g_stream_selected_consume_pending &&
+        g_stream_selected_consume_done_event) {
+        (void)cudaEventSynchronize(g_stream_selected_consume_done_event);
+    }
+    if (g_stream_selected_consume_done_event) {
+        (void)cudaEventDestroy(g_stream_selected_consume_done_event);
+        g_stream_selected_consume_done_event = NULL;
+    }
+    g_stream_selected_consume_owner_device = -1;
+    g_stream_selected_consume_pending = 0;
+    g_stream_selected_consume_poisoned = 0;
+    g_stream_selected_consume_generation = 0;
+}
+
+static int cuda_stream_selected_consume_wait_on_upload(void) {
+    std::lock_guard<std::mutex> lock(g_stream_selected_consume_mutex);
+    if (g_stream_selected_consume_poisoned) return 0;
+    if (!g_stream_selected_consume_pending) return 1;
+    if (!g_stream_selected_consume_done_event ||
+        !g_stream_selected_upload_stream ||
+        g_stream_selected_consume_owner_device < 0 ||
+        g_stream_selected_upload_owner_device !=
+            g_stream_selected_consume_owner_device ||
+        cudaSetDevice(g_stream_selected_consume_owner_device) != cudaSuccess) {
+        (void)cudaGetLastError();
+        return 0;
+    }
+    const cudaError_t err = cudaStreamWaitEvent(
+        g_stream_selected_upload_stream,
+        g_stream_selected_consume_done_event, 0);
+    if (err != cudaSuccess) {
+        fprintf(stderr,
+                "ds4: CUDA compact-cache upload lease wait failed: %s\n",
+                cudaGetErrorString(err));
+        (void)cudaGetLastError();
+        return 0;
+    }
+    g_iq2_ssd_grouped_lease_waits.fetch_add(
+        1, std::memory_order_relaxed);
+    return 1;
+}
+
+static int cuda_stream_selected_consume_record(
+        uint64_t binding_generation, cudaStream_t stream) {
+    std::lock_guard<std::mutex> lock(g_stream_selected_consume_mutex);
+    if (!cuda_stream_selected_consume_ensure_locked()) return 0;
+    const cudaError_t err = cudaEventRecord(
+        g_stream_selected_consume_done_event, stream);
+    if (err != cudaSuccess) {
+        fprintf(stderr,
+                "ds4: CUDA compact-cache consume event record failed: %s\n",
+                cudaGetErrorString(err));
+        (void)cudaGetLastError();
+        /* A failed record must never expose an unfenced binding to the next
+         * loader.  Draining the exact consumer stream is the safe fallback. */
+        const cudaError_t sync_err = cudaStreamSynchronize(stream);
+        if (sync_err != cudaSuccess) {
+            fprintf(stderr,
+                    "ds4: CUDA compact-cache consumer drain failed: %s\n",
+                    cudaGetErrorString(sync_err));
+            (void)cudaGetLastError();
+            const cudaError_t device_err = cudaDeviceSynchronize();
+            if (device_err != cudaSuccess) {
+                fprintf(stderr,
+                        "ds4: CUDA compact-cache device drain failed: %s\n",
+                        cudaGetErrorString(device_err));
+                (void)cudaGetLastError();
+                g_stream_selected_consume_poisoned = 1;
+            }
+        }
+        g_stream_selected_consume_pending = 0;
+        return 0;
+    }
+    g_stream_selected_consume_generation = binding_generation;
+    g_stream_selected_consume_pending = 1;
+    g_iq2_ssd_grouped_lease_records.fetch_add(
+        1, std::memory_order_relaxed);
+    return 1;
+}
+
+static int cuda_stream_selected_readback_stage_ensure(uint64_t bytes) {
+    if (bytes == 0 || bytes > SIZE_MAX) return 0;
+    if (g_stream_selected_readback_stage &&
+        g_stream_selected_readback_stage_capacity >= bytes) {
+        return 1;
+    }
+    if (g_stream_selected_readback_stream &&
+        cudaStreamSynchronize(g_stream_selected_readback_stream) !=
+            cudaSuccess) {
+        (void)cudaGetLastError();
+        return 0;
+    }
+    if (g_stream_selected_readback_stage) {
+        (void)cudaFreeHost(g_stream_selected_readback_stage);
+        g_stream_selected_readback_stage = NULL;
+        g_stream_selected_readback_stage_capacity = 0;
+    }
+    cudaError_t err = cudaMallocHost(
+        &g_stream_selected_readback_stage, (size_t)bytes);
+    if (err != cudaSuccess) {
+        fprintf(stderr,
+                "ds4: CUDA selected readback staging allocation failed: %s\n",
+                cudaGetErrorString(err));
+        (void)cudaGetLastError();
+        return 0;
+    }
+    g_stream_selected_readback_stage_capacity = bytes;
+    return 1;
+}
+
+/* Build a deterministic source-ordered plan without changing compact-slot
+ * order.  Adjacent source spans can share one bounded read even when their
+ * device destinations are discontiguous.  Overlaps are rejected because the
+ * selected-expert list has already been deduplicated. */
+static int cuda_stream_selected_copy_plan(
+        const std::vector<cuda_stream_selected_copy_task> &tasks,
+        uint64_t model_size,
+        uint64_t chunk,
+        std::vector<cuda_stream_selected_copy_task> *sorted_out,
+        std::vector<cuda_stream_selected_copy_segment> *segments_out,
+        std::vector<cuda_stream_selected_copy_group> *groups_out) {
+    if (!sorted_out || !segments_out || !groups_out ||
+        tasks.empty() || chunk == 0) {
+        return 0;
+    }
+
+    try {
+        *sorted_out = tasks;
+        std::stable_sort(
+            sorted_out->begin(), sorted_out->end(),
+            [](const cuda_stream_selected_copy_task &a,
+               const cuda_stream_selected_copy_task &b) {
+                return a.offset != b.offset ? a.offset < b.offset :
+                       a.ordinal < b.ordinal;
+            });
+        segments_out->clear();
+        groups_out->clear();
+        segments_out->reserve(tasks.size());
+        groups_out->reserve(tasks.size());
+    } catch (...) {
+        return 0;
+    }
+    if (sorted_out->size() >= UINT32_MAX) return 0;
+
+    uint64_t previous_end = 0;
+    for (uint32_t i = 0; i < sorted_out->size(); i++) {
+        const cuda_stream_selected_copy_task &task = (*sorted_out)[i];
+        if (!task.dst || task.bytes == 0 || task.offset > model_size ||
+            task.bytes > model_size - task.offset ||
+            task.bytes > (uint64_t)SIZE_MAX ||
+            (i != 0 && task.offset < previous_end)) {
+            return 0;
+        }
+        previous_end = task.offset + task.bytes;
+
+        uint64_t done = 0;
+        while (done < task.bytes) {
+            const uint64_t n = task.bytes - done < chunk ?
+                               task.bytes - done : chunk;
+            try {
+                segments_out->push_back(
+                    { i, done, task.offset + done, n });
+            } catch (...) {
+                return 0;
+            }
+            done += n;
+        }
+    }
+    if (segments_out->empty() ||
+        segments_out->size() >= UINT32_MAX) return 0;
+
+    uint32_t begin = 0;
+    while (begin < segments_out->size()) {
+        const cuda_stream_selected_copy_segment &first =
+            (*segments_out)[begin];
+        uint64_t group_bytes = first.bytes;
+        uint32_t end = begin + 1u;
+        while (end < segments_out->size()) {
+            const cuda_stream_selected_copy_segment &next =
+                (*segments_out)[end];
+            if (next.offset != first.offset + group_bytes ||
+                next.bytes > chunk - group_bytes) {
+                break;
+            }
+            group_bytes += next.bytes;
+            end++;
+        }
+        try {
+            groups_out->push_back(
+                { begin, end, first.offset, group_bytes });
+        } catch (...) {
+            return 0;
+        }
+        begin = end;
+    }
+    return !groups_out->empty();
+}
+
+static int cuda_stream_selected_upload_fail(const char *what) {
+    if (g_stream_selected_upload_stream) {
+        const cudaError_t sync_err =
+            cudaStreamSynchronize(g_stream_selected_upload_stream);
+        if (sync_err != cudaSuccess) {
+            fprintf(stderr,
+                    "ds4: CUDA selected batched I/O abort sync failed for %s: %s\n",
+                    what ? what : "expert batch",
+                    cudaGetErrorString(sync_err));
+            (void)cudaGetLastError();
+        }
+    }
+    return 0;
+}
+
+static int cuda_stream_selected_copy_oracle(
+        const std::vector<cuda_stream_selected_copy_task> &tasks,
+        const void *model_map,
+        uint64_t model_size,
+        const int32_t *remap_src,
+        const int32_t *remap_dst,
+        uint64_t remap_count,
+        int report) {
+    if (!model_map || tasks.empty() ||
+        (remap_count != 0 && (!remap_src || !remap_dst)) ||
+        remap_count > SIZE_MAX / sizeof(int32_t)) {
+        return 0;
+    }
+
+    uint64_t largest_task = 0;
+    for (const cuda_stream_selected_copy_task &task : tasks) {
+        if (!task.dst || task.offset > model_size || task.bytes == 0 ||
+            task.bytes > model_size - task.offset) {
+            return 0;
+        }
+        if (task.bytes > largest_task) largest_task = task.bytes;
+    }
+    const uint64_t scratch_bytes = largest_task < 4u * 1024u * 1024u ?
+                                   largest_task : 4u * 1024u * 1024u;
+    if (scratch_bytes == 0 || scratch_bytes > SIZE_MAX) return 0;
+
+    std::vector<unsigned char> got;
+    try {
+        got.resize((size_t)scratch_bytes);
+    } catch (...) {
+        if (report) {
+            fprintf(stderr,
+                    "ds4: CUDA selected batched I/O byte oracle allocation failed\n");
+        }
+        return 0;
+    }
+
+    for (const cuda_stream_selected_copy_task &task : tasks) {
+        uint64_t done = 0;
+        while (done < task.bytes) {
+            const uint64_t n = task.bytes - done < scratch_bytes ?
+                               task.bytes - done : scratch_bytes;
+            const cudaError_t err = cudaMemcpy(
+                got.data(), task.dst + done, (size_t)n,
+                cudaMemcpyDeviceToHost);
+            if (err != cudaSuccess) {
+                if (report) {
+                    fprintf(stderr,
+                            "ds4: CUDA selected batched I/O byte oracle read "
+                            "failed at model offset %llu: %s\n",
+                            (unsigned long long)(task.offset + done),
+                            cudaGetErrorString(err));
+                }
+                (void)cudaGetLastError();
+                return 0;
+            }
+            const unsigned char *expected =
+                (const unsigned char *)model_map + task.offset + done;
+            if (memcmp(got.data(), expected, (size_t)n) != 0) {
+                if (report) {
+                    size_t mismatch = 0;
+                    while (mismatch < (size_t)n &&
+                           got[mismatch] == expected[mismatch]) {
+                        mismatch++;
+                    }
+                    fprintf(stderr,
+                            "ds4: CUDA selected batched I/O byte oracle "
+                            "mismatch at model offset %llu "
+                            "(expected=0x%02x got=0x%02x)\n",
+                            (unsigned long long)(task.offset + done + mismatch),
+                            (unsigned)expected[mismatch],
+                            (unsigned)got[mismatch]);
+                }
+                return 0;
+            }
+            done += n;
+        }
+    }
+
+    if (remap_count != 0) {
+        std::vector<int32_t> got_remap;
+        try {
+            got_remap.resize((size_t)remap_count);
+        } catch (...) {
+            return 0;
+        }
+        const size_t remap_bytes =
+            (size_t)remap_count * sizeof(got_remap[0]);
+        const cudaError_t err = cudaMemcpy(
+            got_remap.data(), remap_dst, remap_bytes,
+            cudaMemcpyDeviceToHost);
+        if (err != cudaSuccess ||
+            memcmp(got_remap.data(), remap_src, remap_bytes) != 0) {
+            if (report) {
+                fprintf(stderr,
+                        "ds4: CUDA selected batched I/O remap byte oracle %s\n",
+                        err == cudaSuccess ? "mismatch" :
+                        cudaGetErrorString(err));
+            }
+            if (err != cudaSuccess) (void)cudaGetLastError();
+            return 0;
+        }
+    }
+    return 1;
+}
+
+/* Queue every unique gate/up/down span plus the selected-id remap into one
+ * upload epoch.  The ring events protect staging reuse.  Ordinary callers
+ * retain the final stream synchronization; the decode worker may instead
+ * request an upload-done event which the compute stream consumes before the
+ * routed kernels.  submitted_out distinguishes safe pre-enqueue fallback
+ * from a partial upload, which must always fail closed. */
+static int cuda_model_copy_tasks_to_device_streamed(
+        const std::vector<cuda_stream_selected_copy_task> &tasks,
+        const void *model_map,
+        uint64_t model_size,
+        int32_t *remap_dst,
+        const int32_t *remap_src,
+        uint64_t remap_count,
+        int run_oracle,
+        uint64_t chunk_override,
+        int *submitted_out,
+        uint64_t *upload_event_out,
+        const char *what) {
+    if (submitted_out) *submitted_out = 0;
+    if (upload_event_out) *upload_event_out = 0;
+    if (!model_map || tasks.empty() || !remap_dst || !remap_src ||
+        remap_count == 0 || remap_count > SIZE_MAX / sizeof(int32_t)) {
+        return 0;
+    }
+
+    const uint64_t chunk = chunk_override != 0 ?
+                           chunk_override : cuda_model_copy_chunk_bytes();
+    std::vector<cuda_stream_selected_copy_task> sorted;
+    std::vector<cuda_stream_selected_copy_segment> segments;
+    std::vector<cuda_stream_selected_copy_group> groups;
+    if (!cuda_stream_selected_copy_plan(tasks, model_size, chunk,
+                                        &sorted, &segments, &groups)) {
+        fprintf(stderr,
+                "ds4: CUDA selected batched I/O could not build a valid "
+                "copy plan\n");
+        return 0;
+    }
+
+    /* Both diagnostic oracles deliberately retain a synchronous publication
+     * boundary.  This keeps their reference observation deterministic and
+     * makes the ordinary event path the only mode which returns a token. */
+    const int async_publish = upload_event_out != NULL && !run_oracle &&
+        !cuda_stream_selected_event_pipeline_oracle_requested();
+    uint64_t stage_bytes = 0;
+    if (!cuda_host_stage_bytes_for_chunk(chunk, g_model_direct_align,
+                                         &stage_bytes) ||
+        !cuda_stream_selected_stage_pool_alloc(stage_bytes)) {
+        return 0;
+    }
+    if (!cuda_stream_selected_consume_wait_on_upload()) {
+        fprintf(stderr,
+                "ds4: CUDA selected batch refused to overwrite an active "
+                "compact-cache lease\n");
+        return 0;
+    }
+
+    /* Pool allocation is first: its cold-start/re-size path releases every
+     * selected-upload staging allocation, including the persistent remap.
+     * Only publish/copy the remap after that destructive boundary. */
+    if (async_publish &&
+        (!cuda_stream_selected_event_pipeline_ensure() ||
+         !cuda_stream_selected_remap_stage_ensure(remap_count))) {
+        return 0;
+    }
+    const int32_t *remap_upload_src = remap_src;
+    if (async_publish) {
+        memcpy(g_stream_selected_remap_stage, remap_src,
+               (size_t)remap_count * sizeof(remap_src[0]));
+        remap_upload_src = g_stream_selected_remap_stage;
+    }
+
+    const bool profile = cuda_stream_selected_env_flag(
+        "DS4_CUDA_STREAMING_SELECTED_BATCH_IO_PROFILE");
+    const double t0 = profile ? cuda_wall_sec() : 0.0;
+    uint64_t submitted_bytes = 0;
+    const bool use_fd =
+        g_model_fd >= 0 &&
+        (g_model_fd_host_base == NULL || g_model_fd_host_base == model_map);
+
+    if (use_fd) {
+        for (uint32_t gi = 0; gi < groups.size(); gi++) {
+            const cuda_stream_selected_copy_group &group = groups[gi];
+            const uint64_t bi = gi % 4u;
+            if (gi >= 4u) {
+                const cudaError_t err =
+                    cudaEventSynchronize(g_stream_selected_stage_event[bi]);
+                if (err != cudaSuccess) {
+                    fprintf(stderr,
+                            "ds4: CUDA selected batched I/O staging wait "
+                            "failed for %s: %s\n",
+                            what ? what : "expert batch",
+                            cudaGetErrorString(err));
+                    (void)cudaGetLastError();
+                    return cuda_stream_selected_upload_fail(what);
+                }
+            }
+
+            const char *payload = NULL;
+            if (!cuda_model_stage_read(g_stream_selected_stage[bi],
+                                       g_stream_selected_stage_bytes,
+                                       group.offset, group.bytes, &payload)) {
+                fprintf(stderr,
+                        "ds4: CUDA selected batched I/O read failed for %s "
+                        "at model offset %llu: %s\n",
+                        what ? what : "expert batch",
+                        (unsigned long long)group.offset,
+                        strerror(errno));
+                return cuda_stream_selected_upload_fail(what);
+            }
+
+            for (uint32_t si = group.segment_begin;
+                 si < group.segment_end; si++) {
+                const cuda_stream_selected_copy_segment &segment =
+                    segments[si];
+                const cuda_stream_selected_copy_task &task =
+                    sorted[segment.task_index];
+                const uint64_t stage_offset = segment.offset - group.offset;
+                const cudaError_t err = cudaMemcpyAsync(
+                    task.dst + segment.task_offset,
+                    payload + stage_offset,
+                    (size_t)segment.bytes,
+                    cudaMemcpyHostToDevice,
+                    g_stream_selected_upload_stream);
+                if (err != cudaSuccess) {
+                    fprintf(stderr,
+                            "ds4: CUDA selected batched I/O copy failed for "
+                            "%s at model offset %llu: %s\n",
+                            what ? what : "expert batch",
+                            (unsigned long long)segment.offset,
+                            cudaGetErrorString(err));
+                    (void)cudaGetLastError();
+                    return cuda_stream_selected_upload_fail(what);
+                }
+                if (submitted_out) *submitted_out = 1;
+                if (segment.bytes > UINT64_MAX - submitted_bytes) {
+                    return cuda_stream_selected_upload_fail(what);
+                }
+                submitted_bytes += segment.bytes;
+            }
+            const cudaError_t event_err = cudaEventRecord(
+                g_stream_selected_stage_event[bi],
+                g_stream_selected_upload_stream);
+            if (event_err != cudaSuccess) {
+                fprintf(stderr,
+                        "ds4: CUDA selected batched I/O staging record failed "
+                        "for %s: %s\n",
+                        what ? what : "expert batch",
+                        cudaGetErrorString(event_err));
+                (void)cudaGetLastError();
+                return cuda_stream_selected_upload_fail(what);
+            }
+            cuda_model_drop_file_pages(group.offset, group.bytes);
+            cuda_model_discard_source_pages(model_map, model_size,
+                                            group.offset, group.bytes);
+        }
+    } else {
+        for (const cuda_stream_selected_copy_task &task : sorted) {
+            const cudaError_t err = cudaMemcpyAsync(
+                task.dst,
+                (const char *)model_map + task.offset,
+                (size_t)task.bytes,
+                cudaMemcpyHostToDevice,
+                g_stream_selected_upload_stream);
+            if (err != cudaSuccess) {
+                fprintf(stderr,
+                        "ds4: CUDA selected batched host copy failed for %s "
+                        "at model offset %llu: %s\n",
+                        what ? what : "expert batch",
+                        (unsigned long long)task.offset,
+                        cudaGetErrorString(err));
+                (void)cudaGetLastError();
+                return cuda_stream_selected_upload_fail(what);
+            }
+            if (submitted_out) *submitted_out = 1;
+            if (task.bytes > UINT64_MAX - submitted_bytes) {
+                return cuda_stream_selected_upload_fail(what);
+            }
+            submitted_bytes += task.bytes;
+        }
+    }
+
+    const size_t remap_bytes =
+        (size_t)remap_count * sizeof(remap_upload_src[0]);
+    cudaError_t err = cudaMemcpyAsync(
+        remap_dst, remap_upload_src, remap_bytes,
+        cudaMemcpyHostToDevice, g_stream_selected_upload_stream);
+    if (err != cudaSuccess) {
+        fprintf(stderr,
+                "ds4: CUDA selected batched I/O remap copy failed: %s\n",
+                cudaGetErrorString(err));
+        (void)cudaGetLastError();
+        return cuda_stream_selected_upload_fail(what);
+    }
+    if (submitted_out) *submitted_out = 1;
+
+    if (async_publish) {
+        err = cudaEventRecord(g_stream_selected_upload_done_event,
+                              g_stream_selected_upload_stream);
+        if (err != cudaSuccess) {
+            fprintf(stderr,
+                    "ds4: CUDA selected upload event record failed for %s: %s\n",
+                    what ? what : "expert batch", cudaGetErrorString(err));
+            (void)cudaGetLastError();
+            return cuda_stream_selected_upload_fail(what);
+        }
+        uint64_t value = ++g_stream_selected_upload_event_value;
+        if (value == 0) value = ++g_stream_selected_upload_event_value;
+        *upload_event_out = value;
+        g_stream_selected_event_uploads.fetch_add(
+            1, std::memory_order_relaxed);
+    } else {
+        err = cudaStreamSynchronize(g_stream_selected_upload_stream);
+        if (err != cudaSuccess) {
+            fprintf(stderr,
+                    "ds4: CUDA selected batched I/O final sync failed for %s: %s\n",
+                    what ? what : "expert batch", cudaGetErrorString(err));
+            (void)cudaGetLastError();
+            return 0;
+        }
+    }
+
+    if (run_oracle) {
+        g_stream_selected_batch_io_oracle_runs++;
+        if (!cuda_stream_selected_copy_oracle(
+                tasks, model_map, model_size, remap_src, remap_dst,
+                remap_count, /*report=*/1)) {
+            g_stream_selected_batch_io_oracle_failures++;
+            fprintf(stderr,
+                    "ds4: CUDA selected batched I/O failed the byte oracle; "
+                    "compact table rejected\n");
+            return 0;
+        }
+    }
+
+    g_stream_selected_batch_io_tasks += tasks.size();
+    g_stream_selected_batch_io_segments += segments.size();
+    g_stream_selected_batch_io_reads += use_fd ? groups.size() : sorted.size();
+    if (submitted_bytes <= UINT64_MAX - g_stream_selected_batch_io_bytes) {
+        g_stream_selected_batch_io_bytes += submitted_bytes;
+    } else {
+        g_stream_selected_batch_io_bytes = UINT64_MAX;
+    }
+    if (profile) {
+        const double t1 = cuda_wall_sec();
+        fprintf(stderr,
+                "ds4: CUDA selected batched I/O tasks=%zu segments=%zu "
+                "reads=%zu bytes=%.2f MiB total=%.3f ms%s\n",
+                tasks.size(), segments.size(),
+                use_fd ? groups.size() : sorted.size(),
+                (double)submitted_bytes / 1048576.0,
+                (t1 - t0) * 1000.0,
+                run_oracle ? " (includes oracle)" : "");
+    }
+    return 1;
+}
+
+extern "C" int ds4_cuda_test_stream_selected_batch_plan(void) {
+    try {
+        const uint64_t layout_align = 64u;
+        const uint64_t layout_usable = 513u;
+        size_t layout_allocation = 0;
+        std::vector<unsigned char> layout_storage(
+                (size_t)(layout_usable + 2u * layout_align));
+        const uintptr_t layout_base = (uintptr_t)layout_storage.data();
+        const size_t to_boundary = (size_t)(
+            (layout_align - layout_base % layout_align) % layout_align);
+        void *layout_raw = layout_storage.data() + to_boundary + 1u;
+        void *layout_aligned = NULL;
+        uint64_t overflow_stage = 0;
+        if (!cuda_host_stage_allocation_bytes(
+                layout_usable, layout_align, &layout_allocation) ||
+            layout_allocation != layout_usable + layout_align - 1u ||
+            !cuda_host_stage_aligned_view(
+                layout_raw, layout_allocation, layout_usable,
+                layout_align, &layout_aligned) ||
+            (uintptr_t)layout_raw % layout_align != 1u ||
+            (uintptr_t)layout_aligned % layout_align != 0u ||
+            (char *)layout_aligned + layout_usable !=
+                (char *)layout_raw + layout_allocation ||
+            cuda_host_stage_aligned_view(
+                layout_raw, layout_allocation - 1u, layout_usable,
+                layout_align, &layout_aligned) ||
+            cuda_host_stage_allocation_bytes(
+                UINT64_MAX, 2u, &layout_allocation) ||
+            cuda_host_stage_bytes_for_chunk(
+                UINT64_MAX, 2u, &overflow_stage)) {
+            return 0;
+        }
+        memset(layout_aligned, 0xa5, (size_t)layout_usable);
+
+        std::vector<char> storage(64u * 1024u);
+        std::vector<cuda_stream_selected_copy_task> tasks = {
+            { storage.data() + 0u,     8192u, 4096u, 0u },
+            { storage.data() + 4096u,     0u, 4096u, 1u },
+            { storage.data() + 8192u,  4096u, 4096u, 2u },
+            { storage.data() + 12288u, 12288u, 4096u, 3u },
+            { storage.data() + 16384u, 32768u, 4096u, 4u },
+        };
+        std::vector<cuda_stream_selected_copy_task> sorted;
+        std::vector<cuda_stream_selected_copy_segment> segments;
+        std::vector<cuda_stream_selected_copy_group> groups;
+        if (!cuda_stream_selected_copy_plan(
+                tasks, storage.size(), 12288u,
+                &sorted, &segments, &groups) ||
+            sorted.size() != 5u || segments.size() != 5u ||
+            groups.size() != 3u ||
+            sorted[0].ordinal != 1u || sorted[1].ordinal != 2u ||
+            sorted[2].ordinal != 0u || sorted[3].ordinal != 3u ||
+            sorted[4].ordinal != 4u ||
+            groups[0].offset != 0u || groups[0].bytes != 12288u ||
+            groups[0].segment_begin != 0u ||
+            groups[0].segment_end != 3u) {
+            return 0;
+        }
+
+        tasks = {
+            { storage.data(),          0u, 20000u, 0u },
+            { storage.data() + 20000u, 20000u, 4000u, 1u },
+        };
+        if (!cuda_stream_selected_copy_plan(
+                tasks, storage.size(), 8192u,
+                &sorted, &segments, &groups) ||
+            segments.size() != 4u || groups.size() != 3u ||
+            groups[2].offset != 16384u || groups[2].bytes != 7616u) {
+            return 0;
+        }
+
+        tasks = {
+            { storage.data(),          0u, 4096u, 0u },
+            { storage.data() + 4096u, 2048u, 4096u, 1u },
+        };
+        if (cuda_stream_selected_copy_plan(
+                tasks, storage.size(), 8192u,
+                &sorted, &segments, &groups)) {
+            return 0;
+        }
+
+        tasks = {
+            { storage.data(), UINT64_MAX - 1u, 2u, 0u },
+        };
+        if (cuda_stream_selected_copy_plan(
+                tasks, UINT64_MAX, 8192u,
+                &sorted, &segments, &groups)) {
+            return 0;
+        }
+    } catch (...) {
+        return 0;
+    }
+    return 1;
+}
+
+/* Exercise aligned staging, one-read scatter, ring wrap, remap upload and a
+ * deliberate byte-oracle failure on a real CUDA device.  The test restores
+ * every temporary file-global before returning. */
+extern "C" int ds4_cuda_test_stream_selected_batch_copy(void) {
+    if (g_n_gpus < 1) return 0;
+    int saved_device = -1;
+    (void)cudaGetDevice(&saved_device);
+    const int saved_logical_tier = g_current_logical_tier;
+    const auto restore_device = [saved_device, saved_logical_tier]() {
+        if (saved_device >= 0) (void)cudaSetDevice(saved_device);
+        g_current_logical_tier = saved_logical_tier;
+    };
+    if (ds4_gpu_set_current_device(0) != 0) {
+        restore_device();
+        return 0;
+    }
+    if (g_model_stage_bytes != 0 || g_model_upload_stream ||
+        g_stream_selected_stage_bytes != 0 ||
+        g_stream_selected_upload_stream) {
+        restore_device();
+        return 0;
+    }
+
+    const size_t model_bytes = 16u * 1024u;
+    std::vector<unsigned char> source;
+    try {
+        source.resize(model_bytes);
+    } catch (...) {
+        restore_device();
+        return 0;
+    }
+    for (size_t i = 0; i < source.size(); i++) {
+        source[i] = (unsigned char)((i * 37u + 11u) & 0xffu);
+    }
+
+    FILE *fp = tmpfile();
+    if (!fp || fwrite(source.data(), 1, source.size(), fp) != source.size() ||
+        fflush(fp) != 0) {
+        if (fp) fclose(fp);
+        restore_device();
+        return 0;
+    }
+    void *map = mmap(NULL, model_bytes, PROT_READ, MAP_PRIVATE,
+                     fileno(fp), 0);
+    if (map == MAP_FAILED) {
+        fclose(fp);
+        restore_device();
+        return 0;
+    }
+
+    char *dst[6] = { NULL, NULL, NULL, NULL, NULL, NULL };
+    int32_t *remap_dst = NULL;
+    int ok = 1;
+    for (uint32_t i = 0; i < 6u; i++) {
+        if (cudaMalloc((void **)&dst[i], 4096u) != cudaSuccess) ok = 0;
+    }
+    const int32_t remap_src[6] = { 2, 0, 1, 2, 2, 0 };
+    if (cudaMalloc((void **)&remap_dst, sizeof(remap_src)) != cudaSuccess) {
+        ok = 0;
+    }
+
+    const int saved_fd = g_model_fd;
+    const void *saved_fd_host_base = g_model_fd_host_base;
+    const int saved_direct_fd = g_model_direct_fd;
+    const uint64_t saved_direct_align = g_model_direct_align;
+    const uint64_t saved_file_size = g_model_file_size;
+    g_model_fd = fileno(fp);
+    g_model_fd_host_base = map;
+    g_model_direct_fd = -1;
+    g_model_direct_align = 257u;
+    g_model_file_size = model_bytes;
+
+    if (ok) {
+        uint64_t stage_bytes = 0;
+        size_t allocation_bytes = 0;
+        ok = cuda_host_stage_bytes_for_chunk(
+                 4096u, g_model_direct_align, &stage_bytes) &&
+             cuda_host_stage_allocation_bytes(
+                 stage_bytes, g_model_direct_align, &allocation_bytes) &&
+             cuda_model_stage_pool_alloc(stage_bytes) &&
+             g_model_stage_bytes == stage_bytes &&
+             g_model_stage_align == g_model_direct_align;
+        for (uint32_t i = 0; ok && i < 4u; i++) {
+            const size_t delta = (size_t)(
+                (char *)g_model_stage[i] -
+                (char *)g_model_stage_raw[i]);
+            ok = (uintptr_t)g_model_stage[i] % g_model_direct_align == 0u &&
+                 delta <= allocation_bytes &&
+                 stage_bytes <= allocation_bytes - delta;
+        }
+        cuda_model_stage_pool_release();
+    }
+
+    if (ok) {
+        try {
+            std::vector<cuda_stream_selected_copy_task> tasks = {
+                { dst[0], 8192u, 4096u, 0u },
+                { dst[1],    0u, 4096u, 1u },
+                { dst[2], 4096u, 4096u, 2u },
+            };
+            /* Force the production cold-start order.  stage_pool_alloc()
+             * must run before the persistent remap is allocated/copied; the
+             * inverse order used to free the remap and enqueue from a stale
+             * host pointer on the first async batch. */
+            cuda_stream_selected_stage_release();
+            int submitted = 0;
+            uint64_t cold_upload_event = 0;
+            ok = cuda_model_copy_tasks_to_device_streamed(
+                tasks, map, model_bytes,
+                remap_dst, remap_src, 6u,
+                /*run_oracle=*/0,
+                /*chunk_override=*/12288u,
+                &submitted, &cold_upload_event,
+                "selected batch cold-start event selftest") &&
+                submitted &&
+                ds4_gpu_stream_expert_cache_wait_selected_upload(
+                    cold_upload_event,
+                    "selected batch cold-start event selftest") &&
+                cudaStreamSynchronize(cuda_decode_stream()) == cudaSuccess &&
+                cuda_stream_selected_copy_oracle(
+                    tasks, map, model_bytes,
+                    remap_src, remap_dst, 6u, /*report=*/1);
+            submitted = 0;
+            ok = ok && cuda_model_copy_tasks_to_device_streamed(
+                tasks, map, model_bytes,
+                remap_dst, remap_src, 6u,
+                /*run_oracle=*/1,
+                /*chunk_override=*/12288u,
+                &submitted, /*upload_event_out=*/NULL,
+                "selected batch selftest") && submitted;
+            if (ok) {
+                size_t selected_allocation = 0;
+                ok = cuda_host_stage_allocation_bytes(
+                    g_stream_selected_stage_bytes,
+                    g_stream_selected_stage_align,
+                    &selected_allocation);
+                void *first_ring_raw[4];
+                for (uint32_t i = 0; ok && i < 4u; i++) {
+                    first_ring_raw[i] = g_stream_selected_stage_raw[i];
+                    const size_t delta = (size_t)(
+                        (char *)g_stream_selected_stage[i] -
+                        (char *)g_stream_selected_stage_raw[i]);
+                    ok = (uintptr_t)g_stream_selected_stage[i] %
+                             g_model_direct_align == 0u &&
+                         delta <= selected_allocation &&
+                         g_stream_selected_stage_bytes <=
+                             selected_allocation - delta;
+                }
+                const int32_t wrap_remap[6] = { 5, 4, 3, 2, 1, 0 };
+                std::vector<cuda_stream_selected_copy_task> wrap_tasks = {
+                    { dst[0],     0u, 1024u, 0u },
+                    { dst[1],  2048u, 1024u, 1u },
+                    { dst[2],  4096u, 1024u, 2u },
+                    { dst[3],  6144u, 1024u, 3u },
+                    { dst[4],  8192u, 1024u, 4u },
+                    { dst[5], 10240u, 1024u, 5u },
+                };
+                std::vector<cuda_stream_selected_copy_task> wrap_sorted;
+                std::vector<cuda_stream_selected_copy_segment> wrap_segments;
+                std::vector<cuda_stream_selected_copy_group> wrap_groups;
+                submitted = 0;
+                ok = ok && cuda_stream_selected_copy_plan(
+                         wrap_tasks, model_bytes, 4096u,
+                         &wrap_sorted, &wrap_segments, &wrap_groups) &&
+                     wrap_groups.size() == 6u &&
+                     cuda_model_copy_tasks_to_device_streamed(
+                         wrap_tasks, map, model_bytes,
+                         remap_dst, wrap_remap, 6u,
+                         /*run_oracle=*/1,
+                         /*chunk_override=*/4096u,
+                         &submitted,
+                         /*upload_event_out=*/NULL,
+                         "selected batch ring-wrap selftest") &&
+                     submitted;
+                for (uint32_t i = 0; ok && i < 4u; i++) {
+                    ok = first_ring_raw[i] ==
+                         g_stream_selected_stage_raw[i];
+                }
+                if (ok) {
+                    const unsigned char corrupt =
+                        (unsigned char)(source[8192u] ^ 0xffu);
+                    ok = cudaMemcpy(dst[4], &corrupt, sizeof(corrupt),
+                                    cudaMemcpyHostToDevice) == cudaSuccess &&
+                         !cuda_stream_selected_copy_oracle(
+                             wrap_tasks, map, model_bytes,
+                             wrap_remap, remap_dst, 6u, /*report=*/0);
+                }
+            }
+        } catch (...) {
+            ok = 0;
+        }
+    }
+
+    g_model_fd = saved_fd;
+    g_model_fd_host_base = saved_fd_host_base;
+    g_model_direct_fd = saved_direct_fd;
+    g_model_direct_align = saved_direct_align;
+    g_model_file_size = saved_file_size;
+    cuda_model_stage_pool_release();
+    if (g_model_upload_stream) {
+        (void)cudaStreamDestroy(g_model_upload_stream);
+        g_model_upload_stream = NULL;
+    }
+    cuda_stream_selected_stage_release();
+    for (uint32_t i = 0; i < 6u; i++) {
+        if (dst[i]) (void)cudaFree(dst[i]);
+    }
+    if (remap_dst) (void)cudaFree(remap_dst);
+    (void)munmap(map, model_bytes);
+    (void)fclose(fp);
+    restore_device();
+    return ok;
+}
+
+static int cuda_model_copy_to_device_streamed(
+        char *dst,
+        const void *model_map,
+        uint64_t model_size,
+        uint64_t offset,
+        uint64_t bytes,
+        const char *what) {
+    if (!dst || !model_map || offset > model_size ||
+        bytes > model_size - offset) {
+        return 0;
+    }
+    if (bytes == 0) return 1;
+    if (g_model_fd < 0 ||
+        (g_model_fd_host_base != NULL && model_map != g_model_fd_host_base)) {
+        return cuda_ok(cudaMemcpy(dst,
+                                  (const char *)model_map + offset,
+                                  (size_t)bytes,
+                                  cudaMemcpyHostToDevice),
+                       what ? what : "stream selected expert copy");
+    }
+
+    const uint64_t chunk = cuda_model_copy_chunk_bytes();
+    uint64_t stage_bytes = 0;
+    if (!cuda_host_stage_bytes_for_chunk(chunk, g_model_direct_align,
+                                         &stage_bytes) ||
+        !cuda_stream_selected_stage_pool_alloc(stage_bytes) ||
+        !cuda_stream_selected_consume_wait_on_upload()) return 0;
+
+    uint64_t copied = 0;
+    uint64_t chunk_idx = 0;
+    while (copied < bytes) {
+        const uint64_t n = bytes - copied < chunk ? bytes - copied : chunk;
+        const uint64_t bi = chunk_idx % 4u;
+        cudaError_t err;
+        if (chunk_idx >= 4u) {
+            err = cudaEventSynchronize(g_stream_selected_stage_event[bi]);
+            if (err != cudaSuccess) {
+                fprintf(stderr,
+                        "ds4: CUDA streaming selected staging wait failed for %s: %s\n",
+                        what ? what : "expert", cudaGetErrorString(err));
+                (void)cudaGetLastError();
+                return cuda_stream_selected_upload_fail(what);
+            }
+        }
+        const char *payload = NULL;
+        if (!cuda_model_stage_read(g_stream_selected_stage[bi],
+                                   g_stream_selected_stage_bytes,
+                                   offset + copied, n, &payload)) {
+            fprintf(stderr,
+                    "ds4: CUDA streaming selected read failed for %s at %.2f MiB: %s\n",
+                    what ? what : "expert", (double)copied / 1048576.0,
+                    strerror(errno));
+            return cuda_stream_selected_upload_fail(what);
+        }
+        err = cudaMemcpyAsync(dst + copied, payload, (size_t)n,
+                              cudaMemcpyHostToDevice,
+                              g_stream_selected_upload_stream);
+        if (err != cudaSuccess) {
+            fprintf(stderr,
+                    "ds4: CUDA streaming selected copy failed for %s at %.2f MiB: %s\n",
+                    what ? what : "expert", (double)copied / 1048576.0,
+                    cudaGetErrorString(err));
+            (void)cudaGetLastError();
+            return cuda_stream_selected_upload_fail(what);
+        }
+        err = cudaEventRecord(g_stream_selected_stage_event[bi],
+                              g_stream_selected_upload_stream);
+        if (err != cudaSuccess) {
+            fprintf(stderr,
+                    "ds4: CUDA streaming selected staging record failed for %s: %s\n",
+                    what ? what : "expert", cudaGetErrorString(err));
+            (void)cudaGetLastError();
+            return cuda_stream_selected_upload_fail(what);
+        }
+        cuda_model_drop_file_pages(offset + copied, n);
+        cuda_model_discard_source_pages(model_map, model_size,
+                                        offset + copied, n);
+        copied += n;
+        chunk_idx++;
+    }
+
+    const cudaError_t err =
+        cudaStreamSynchronize(g_stream_selected_upload_stream);
+    if (err != cudaSuccess) {
+        fprintf(stderr,
+                "ds4: CUDA streaming selected upload sync failed for %s: %s\n",
+                what ? what : "expert", cudaGetErrorString(err));
+        (void)cudaGetLastError();
+        return 0;
+    }
+    return 1;
+}
+
+static uint64_t cuda_model_cache_limit_bytes(void) {
+    uint64_t gb = 0;
+    const char *env = getenv("DS4_CUDA_WEIGHT_CACHE_LIMIT_GB");
+    if (env && env[0]) {
+        char *end = NULL;
+        unsigned long long v = strtoull(env, &end, 10);
+        if (end != env) gb = (uint64_t)v;
+    }
+    if (gb == 0) return UINT64_MAX;
+    return gb * 1073741824ull;
+}
+
+static uint64_t cuda_model_arena_chunk_bytes(uint64_t need) {
+    uint64_t mb = 1792;
+    const char *env = getenv("DS4_CUDA_WEIGHT_ARENA_CHUNK_MB");
+    if (env && env[0]) {
+        char *end = NULL;
+        unsigned long long v = strtoull(env, &end, 10);
+        if (end != env && v > 0) mb = (uint64_t)v;
+    }
+    if (mb < 256) mb = 256;
+    if (mb > 8192) mb = 8192;
+    uint64_t bytes = mb * 1048576ull;
+    if (bytes < need) {
+        const uint64_t align = 256ull * 1048576ull;
+        bytes = (need + align - 1u) & ~(align - 1u);
+    }
+    return bytes;
+}
+
+static char *cuda_model_arena_alloc(uint64_t bytes, const char *what) {
+    if (bytes == 0) return NULL;
+    if (g_model_cache_full) return NULL;
+    const uint64_t align = 256u;
+    const uint64_t aligned = (bytes + align - 1u) & ~(align - 1u);
+
+    for (cuda_model_arena &a : g_model_arenas) {
+        const uint64_t used = (a.used + align - 1u) & ~(align - 1u);
+        if (used <= a.bytes && aligned <= a.bytes - used) {
+            char *ptr = a.device_ptr + used;
+            a.used = used + aligned;
+            return ptr;
+        }
+    }
+
+    const uint64_t limit = cuda_model_cache_limit_bytes();
+    if (g_model_range_bytes > limit || aligned > limit - g_model_range_bytes) return NULL;
+
+    const uint64_t chunk = cuda_model_arena_chunk_bytes(aligned);
+    void *dev = NULL;
+    cudaError_t err = cudaMalloc(&dev, (size_t)chunk);
+    if (err != cudaSuccess) {
+        fprintf(stderr, "ds4: CUDA model arena alloc failed for %s (%.2f MiB chunk): %s\n",
+                what ? what : "weights",
+                (double)chunk / 1048576.0,
+                cudaGetErrorString(err));
+        (void)cudaGetLastError();
+        g_model_cache_full = 1;
+        return NULL;
+    }
+    g_model_arenas.push_back({(char *)dev, chunk, aligned});
+    if (getenv("DS4_CUDA_WEIGHT_CACHE_VERBOSE")) {
+        uint64_t arena_bytes = 0;
+        for (const cuda_model_arena &a : g_model_arenas) arena_bytes += a.bytes;
+        fprintf(stderr, "ds4: CUDA model arena allocated %.2f MiB (arenas %.2f GiB)\n",
+                (double)chunk / 1048576.0,
+                (double)arena_bytes / 1073741824.0);
+    }
+    return (char *)dev;
+}
+
+static const char *cuda_model_range_ptr_from_fd(
+        const void *model_map,
+        uint64_t offset,
+        uint64_t bytes,
+        const char *what) {
+    if (g_model_fd < 0 || bytes == 0) return NULL;
+    if (g_model_fd_host_base != NULL && model_map != g_model_fd_host_base) return NULL;
+    const uint64_t limit = cuda_model_cache_limit_bytes();
+    if (g_model_range_bytes > limit || bytes > limit - g_model_range_bytes) {
+        if (getenv("DS4_CUDA_WEIGHT_CACHE_VERBOSE")) {
+            fprintf(stderr, "ds4: CUDA direct %s %.2f MiB (cache budget %.2f GiB exhausted)\n",
+                    what ? what : "weights",
+                    (double)bytes / 1048576.0,
+                    (double)limit / 1073741824.0);
+        }
+        return cuda_model_ptr(model_map, offset);
+    }
+
+    char *dev = cuda_model_arena_alloc(bytes, what);
+    if (!dev) {
+        if (getenv("DS4_CUDA_STRICT_WEIGHT_CACHE") != NULL) return NULL;
+        return cuda_model_ptr(model_map, offset);
+    }
+    cudaError_t err = cudaSuccess;
+
+    const uint64_t chunk = cuda_model_copy_chunk_bytes();
+    uint64_t stage_bytes = 0;
+    if (!cuda_host_stage_bytes_for_chunk(chunk, g_model_direct_align,
+                                         &stage_bytes) ||
+        !cuda_model_stage_pool_alloc(stage_bytes)) return NULL;
+
+    uint64_t copied = 0;
+    uint64_t chunk_idx = 0;
+    while (copied < bytes) {
+        const uint64_t n = (bytes - copied < chunk) ? (bytes - copied) : chunk;
+        const uint64_t bi = chunk_idx % 4u;
+        if (chunk_idx >= 4u) {
+            err = cudaEventSynchronize(g_model_stage_event[bi]);
+            if (err != cudaSuccess) {
+                fprintf(stderr, "ds4: CUDA model staging wait failed for %s: %s\n",
+                        what ? what : "weights", cudaGetErrorString(err));
+                (void)cudaGetLastError();
+                (void)cuda_model_upload_fail(what);
+                return NULL;
+            }
+        }
+        const char *payload = NULL;
+        if (!cuda_model_stage_read(g_model_stage[bi], g_model_stage_bytes,
+                                   offset + copied, n, &payload)) {
+            fprintf(stderr, "ds4: CUDA model range read failed for %s at %.2f MiB: %s\n",
+                    what ? what : "weights",
+                    (double)copied / 1048576.0,
+                    strerror(errno));
+            (void)cuda_model_upload_fail(what);
+            return NULL;
+        }
+        err = cudaMemcpyAsync(dev + copied, payload, (size_t)n,
+                              cudaMemcpyHostToDevice, g_model_upload_stream);
+        if (err != cudaSuccess) {
+            fprintf(stderr, "ds4: CUDA model range copy failed for %s at %.2f MiB: %s\n",
+                    what ? what : "weights",
+                    (double)copied / 1048576.0,
+                    cudaGetErrorString(err));
+            (void)cudaGetLastError();
+            (void)cuda_model_upload_fail(what);
+            return NULL;
+        }
+        err = cudaEventRecord(g_model_stage_event[bi], g_model_upload_stream);
+        if (err != cudaSuccess) {
+            fprintf(stderr, "ds4: CUDA model staging record failed for %s: %s\n",
+                    what ? what : "weights", cudaGetErrorString(err));
+            (void)cudaGetLastError();
+            (void)cuda_model_upload_fail(what);
+            return NULL;
+        }
+        cuda_model_drop_file_pages(offset + copied, n);
+        cuda_model_discard_source_pages(model_map, g_model_registered_size, offset + copied, n);
+        copied += n;
+        cuda_model_load_progress_note(g_model_range_bytes + copied);
+        chunk_idx++;
+    }
+    err = cudaStreamSynchronize(g_model_upload_stream);
+    if (err != cudaSuccess) {
+        fprintf(stderr, "ds4: CUDA model range upload sync failed for %s: %s\n",
+                what ? what : "weights", cudaGetErrorString(err));
+        (void)cudaGetLastError();
+        return NULL;
+    }
+
+    g_model_ranges.push_back({model_map, offset, bytes, dev, NULL, NULL, 0, 0, 1});
+    g_model_range_by_offset[offset] = g_model_ranges.size() - 1u;
+    g_model_range_bytes += bytes;
+    cuda_model_load_progress_note(g_model_range_bytes);
+    if (getenv("DS4_CUDA_WEIGHT_CACHE_VERBOSE")) {
+        fprintf(stderr, "ds4: CUDA fd-cached %s %.2f MiB (total %.2f GiB)\n",
+                what ? what : "weights",
+                (double)bytes / 1048576.0,
+                (double)g_model_range_bytes / 1073741824.0);
+    }
+    return (const char *)dev;
+}
+
+static int cuda_model_copy_chunked(const void *model_map, uint64_t model_size, uint64_t map_offset, uint64_t map_size) {
+    if (!model_map || model_size == 0 || map_offset > model_size || map_size > model_size - map_offset) return 0;
+    if (getenv("DS4_CUDA_NO_MODEL_COPY") != NULL ||
+        getenv("DS4_CUDA_DIRECT_MODEL") != NULL ||
+        getenv("DS4_CUDA_WEIGHT_CACHE") != NULL ||
+        getenv("DS4_CUDA_WEIGHT_PRELOAD") != NULL) {
+        return 0;
+    }
+    if (g_model_device_owned || g_model_registered) return 1;
+
+    void *dev = NULL;
+    const double t0 = cuda_wall_sec();
+    cudaError_t err = cudaMalloc(&dev, (size_t)model_size);
+    if (err != cudaSuccess) {
+        fprintf(stderr, "ds4: CUDA model allocation skipped: %s\n", cudaGetErrorString(err));
+        (void)cudaGetLastError();
+        return 0;
+    }
+
+    fprintf(stderr, "ds4: CUDA chunk-copying %.2f GiB model image\n",
+            (double)model_size / 1073741824.0);
+
+    const uint64_t chunk = cuda_model_copy_chunk_bytes();
+    void *stage = NULL;
+    err = cudaMallocHost(&stage, (size_t)chunk);
+    if (err != cudaSuccess) {
+        fprintf(stderr, "ds4: CUDA pinned model staging allocation failed: %s\n", cudaGetErrorString(err));
+        (void)cudaFree(dev);
+        (void)cudaGetLastError();
+        return 0;
+    }
+
+    if (map_offset > 0) {
+        uint64_t copied_header = 0;
+        while (copied_header < map_offset) {
+            const uint64_t n = (map_offset - copied_header < chunk) ? (map_offset - copied_header) : chunk;
+            memcpy(stage, (const char *)model_map + copied_header, (size_t)n);
+            err = cudaMemcpy((char *)dev + copied_header, stage, (size_t)n, cudaMemcpyHostToDevice);
+            if (err != cudaSuccess) {
+                fprintf(stderr, "ds4: CUDA model header copy failed: %s\n", cudaGetErrorString(err));
+                (void)cudaFreeHost(stage);
+                (void)cudaFree(dev);
+                (void)cudaGetLastError();
+                return 0;
+            }
+            copied_header += n;
+        }
+    }
+
+    uint64_t copied = 0;
+    double last_report = t0;
+    while (copied < map_size) {
+        const uint64_t n = (map_size - copied < chunk) ? (map_size - copied) : chunk;
+        const uint64_t off = map_offset + copied;
+        memcpy(stage, (const char *)model_map + off, (size_t)n);
+        err = cudaMemcpy((char *)dev + off, stage, (size_t)n, cudaMemcpyHostToDevice);
+        if (err != cudaSuccess) {
+            fprintf(stderr, "ds4: CUDA model chunk copy failed at %.2f GiB: %s\n",
+                    (double)copied / 1073741824.0, cudaGetErrorString(err));
+            (void)cudaFreeHost(stage);
+            (void)cudaFree(dev);
+            (void)cudaGetLastError();
+            return 0;
+        }
+        cuda_model_discard_source_pages(model_map, model_size, off, n);
+        copied += n;
+        const double now = cuda_wall_sec();
+        if (getenv("DS4_CUDA_MODEL_COPY_VERBOSE") != NULL && now - last_report >= 2.0) {
+            fprintf(stderr, "ds4: CUDA model chunk copy %.2f/%.2f GiB\n",
+                    (double)copied / 1073741824.0,
+                    (double)map_size / 1073741824.0);
+            last_report = now;
+        }
+    }
+
+    (void)cudaFreeHost(stage);
+    g_model_device_base = (const char *)dev;
+    g_model_device_owned = 1;
+    g_model_hmm_direct = 0;
+    const double t1 = cuda_wall_sec();
+    fprintf(stderr,
+            "ds4: CUDA model chunk copy complete in %.3fs (%.2f GiB tensors)\n",
+            t1 - t0,
+            (double)map_size / 1073741824.0);
+    return 1;
+}
+
+static void cuda_model_range_release_all(void) {
+    for (const cuda_model_range &r : g_model_ranges) {
+        if (r.host_registered && r.registered_base) {
+            (void)cudaHostUnregister(r.registered_base);
+        } else if (r.device_ptr && !r.arena_allocated && !r.borrowed) {
+            (void)cudaFree(r.device_ptr);
+        }
+    }
+    for (const cuda_model_arena &a : g_model_arenas) {
+        if (a.device_ptr) (void)cudaFree(a.device_ptr);
+    }
+    g_model_arenas.clear();
+    g_model_ranges.clear();
+    g_model_range_by_offset.clear();
+    g_model_range_bytes = 0;
+    cuda_model_load_progress_reset();
+}
+
+static void cuda_f16_pair_chunk32_release_all(void) {
+    int previous_device = -1;
+    (void)cudaGetDevice(&previous_device);
+    for (const cuda_f16_pair_chunk32_range &r : g_f16_pair_chunk32_ranges) {
+        if (!r.device_ptr) continue;
+        (void)cudaSetDevice(r.device_id);
+        (void)cudaFree(r.device_ptr);
+    }
+    if (previous_device >= 0) (void)cudaSetDevice(previous_device);
+    g_f16_pair_chunk32_ranges.clear();
+    g_f16_pair_chunk32_disabled_after_oom = 0;
+}
+
+static void cuda_derived_range_release_all(void) {
     ds4_mmq_set_aligned_q81_scratch(NULL, 0);
     if (g_aligned_q81_scratch) {
         (void)cudaFree(g_aligned_q81_scratch);
@@ -2575,7 +7816,11 @@ static int cublas_ok(cublasStatus_t st, const char *what) {
 }
 
 extern "C" int ds4_gpu_init_multi(const ds4_gpu_config *cfg) {
+    g_stream_expert_persistent_runtime_ready = 0;
+    cuda_q8_fold_invalidate_all();
+    ds4_mmq_set_gb10_optimizations(0);
     if (!cfg || cfg->n_gpus < 1 || cfg->n_gpus > DS4_MAX_GPUS) return 0;
+    memset(g_cuda_is_gb10, 0, sizeof(g_cuda_is_gb10));
     cuda_xdev_env_refresh();
     cuda_decode_dispatch_env_refresh();
     g_current_logical_tier = -1;
@@ -2598,6 +7843,8 @@ extern "C" int ds4_gpu_init_multi(const ds4_gpu_config *cfg) {
         if (!cuda_ok(cudaSetDevice(c->device_id), "init set device")) return 0;
         cudaDeviceProp prop;
         if (cudaGetDeviceProperties(&prop, c->device_id) == cudaSuccess) {
+            g_cuda_is_gb10[i] =
+                prop.major == 12 && prop.minor == 1 && prop.integrated;
             fprintf(stderr, "ds4: CUDA backend initialized on %s (sm_%d%d) dev=%d\n",
                     prop.name, prop.major, prop.minor, c->device_id);
         }
@@ -2625,6 +7872,11 @@ extern "C" int ds4_gpu_init_multi(const ds4_gpu_config *cfg) {
         c->scratch      = NULL;
         c->scratch_bytes = 0;
     }
+    int all_gb10 = cfg->n_gpus == 1;
+    for (int i = 0; i < cfg->n_gpus; i++) {
+        if (!g_cuda_is_gb10[i]) all_gb10 = 0;
+    }
+    ds4_mmq_set_gb10_optimizations(all_gb10);
 
     /* NxN peer-access matrix.
      *
@@ -2762,6 +8014,7 @@ extern "C" int ds4_gpu_init_multi(const ds4_gpu_config *cfg) {
     }
 
     g_cublas_ready = 1;
+    g_stream_expert_persistent_runtime_ready = g_n_gpus == 1;
     return 1;
 }
 
@@ -2774,7 +8027,19 @@ extern "C" int ds4_gpu_init(void) {
 }
 
 extern "C" void ds4_gpu_cleanup(void) {
+    g_cuda_test_q4_mmq_strict = 0;
+    g_stream_expert_persistent_runtime_ready = 0;
     (void)cudaDeviceSynchronize();
+    /* The resident q_b GEMMs may still reference cache-owned allocations.
+     * Retire them while the tier contexts and physical-device mapping are
+     * intact; release performs its own quiescence check. */
+    (void)ds4_gpu_release_q4_attn_q_b_f16_sidecars();
+    /* The transient arena is backend-owned rather than graph-owned.  Release
+     * it only after quiescence and before tearing down the device contexts. */
+    (void)cuda_q4_attn_q_b_transient_f16_scratch_release();
+    cuda_q4_attn_q_b_f16_set_multi_model_policy(0);
+    cuda_decode_graphs_shutdown();
+    cuda_q8_fold_release_all();
     g_current_logical_tier = -1;
 
     /* Multi-GPU teardown: events, streams, cublas handles, scratch
@@ -2827,8 +8092,14 @@ extern "C" void ds4_gpu_cleanup(void) {
             }
         }
     }
-    cuda_stream_selected_cache_release();
+    /* The selected cache may still be the destination of an event-published
+     * upload.  Drain and retire its auxiliary streams before freeing device
+     * storage or invalidating the owner-device context. */
+    (void)cuda_stream_expert_storage_release(1);
     cuda_stream_selected_stage_release();
+    cuda_stream_selected_event_pipeline_release();
+    cuda_stream_selected_consume_release();
+    ds4_mmq_set_gb10_optimizations(0);
     g_n_gpus = 0;
     g_cublas_ready = 0;
 
@@ -2849,6 +8120,7 @@ extern "C" void ds4_gpu_cleanup(void) {
     /* Continue with legacy global teardown below. */
 
     cuda_model_range_release_all();
+    cuda_f16_pair_chunk32_release_all();
     cuda_derived_range_release_all();
     cuda_q8_f16_cache_release_all();
     g_q8_f16_disabled_after_oom = 0;
@@ -2864,18 +8136,7 @@ extern "C" void ds4_gpu_cleanup(void) {
         g_cuda_tmp = NULL;
         g_cuda_tmp_bytes = 0;
     }
-    for (size_t i = 0; i < 4; i++) {
-        if (g_model_stage_event[i]) {
-            (void)cudaEventDestroy(g_model_stage_event[i]);
-            g_model_stage_event[i] = NULL;
-        }
-        if (g_model_stage_raw[i]) {
-            (void)cudaFreeHost(g_model_stage_raw[i]);
-            g_model_stage_raw[i] = NULL;
-            g_model_stage[i] = NULL;
-        }
-    }
-    g_model_stage_bytes = 0;
+    cuda_model_stage_pool_release();
     if (g_model_upload_stream) {
         (void)cudaStreamDestroy(g_model_upload_stream);
         g_model_upload_stream = NULL;
@@ -2894,6 +8155,10 @@ extern "C" void ds4_gpu_cleanup(void) {
     g_model_range_mapping_supported = 1;
     g_model_hmm_direct = 0;
     g_model_fd = -1;
+    g_model_fd_host_base = NULL;
+    g_support_host_base = NULL;
+    g_support_host_size = 0;
+    g_support_offset_bias = 0;
     if (g_model_direct_fd >= 0) {
         (void)close(g_model_direct_fd);
         g_model_direct_fd = -1;
@@ -3107,6 +8372,11 @@ extern "C" uint64_t ds4_gpu_tensor_bytes(const ds4_gpu_tensor *tensor) {
     return tensor ? tensor->bytes : 0;
 }
 
+extern "C" uintptr_t ds4_gpu_tensor_storage_key(
+        const ds4_gpu_tensor *tensor) {
+    return tensor ? (uintptr_t)tensor->ptr : (uintptr_t)0;
+}
+
 extern "C" void *ds4_gpu_tensor_contents(ds4_gpu_tensor *tensor) {
     if (!tensor) return NULL;
     /* Full-device sync preserves legacy semantics. */
@@ -3689,19 +8959,32 @@ extern "C" int ds4_gpu_pack_slot_rows_f32_tensor(
 }
 
 extern "C" int ds4_gpu_begin_commands(void) { return 1; }
-extern "C" int ds4_gpu_flush_commands(void) { return cuda_ok(cudaDeviceSynchronize(), "flush"); }
+extern "C" int ds4_gpu_flush_commands(void) {
+    if (!cuda_ok(cudaDeviceSynchronize(), "flush")) return 0;
+    return cuda_q4_attn_q_b_f16_consume_pending_evict();
+}
 extern "C" int ds4_gpu_end_commands(void) {
+    int ok = 0;
     if (g_cuda_end_stream_sync) {
-        return cuda_ok(cudaStreamSynchronize(0), "end commands stream");
+        ok = cuda_ok(cudaStreamSynchronize(0), "end commands stream");
+    } else {
+        ok = cuda_ok(cudaDeviceSynchronize(), "end commands");
     }
-    return cuda_ok(cudaDeviceSynchronize(), "end commands");
+    return ok && cuda_q4_attn_q_b_f16_consume_pending_evict();
+}
+extern "C" int ds4_gpu_synchronize(void) {
+    if (!cuda_ok(cudaDeviceSynchronize(), "synchronize")) return 0;
+    return cuda_q4_attn_q_b_f16_consume_pending_evict();
 }
-extern "C" int ds4_gpu_synchronize(void) { return cuda_ok(cudaDeviceSynchronize(), "synchronize"); }
 
 extern "C" int ds4_gpu_set_model_map(const void *model_map, uint64_t model_size) {
     if (!model_map || model_size == 0) return 0;
     if (g_model_host_base == model_map && g_model_registered_size == model_size) return 1;
-    cuda_stream_selected_cache_release();
+    cuda_q8_fold_invalidate_all();
+    if (!cuda_stream_expert_storage_release(1)) return 0;
+    if (!ds4_gpu_release_q4_attn_q_b_f16_sidecars()) return 0;
+    cuda_q4_attn_q_b_f16_set_multi_model_policy(0);
+    cuda_f16_pair_chunk32_release_all();
     cuda_model_range_release_all();
     cuda_q8_f16_cache_release_all();
     g_q8_f16_disabled_after_oom = 0;
@@ -3862,6 +9145,46 @@ extern "C" int ds4_gpu_set_aux_model_map_range(
     return 1;
 }
 
+extern "C" int ds4_gpu_prepare_support_model(
+        const void *model_map,
+        uint64_t model_size,
+        uint64_t map_offset,
+        uint64_t map_size,
+        uint64_t max_tensor_bytes) {
+    (void)max_tensor_bytes;
+    if (!model_map || model_size == 0 || map_offset > model_size ||
+        map_size == 0 || map_size > model_size - map_offset) {
+        return 0;
+    }
+    if (g_model_fd < 0 || g_model_fd_host_base != model_map) {
+        fprintf(stderr,
+                "ds4: CUDA support model fd does not match its mmap\n");
+        return 0;
+    }
+    /* A resident support GGUF creates a genuine multi-model working set.
+     * Serialize the mode transition with sidecar builders, retire any
+     * existing single-model sidecars, then publish the conservative policy
+     * before allocating the support payload. */
+    {
+        std::lock_guard<std::mutex> build_lock(
+            g_q4_attn_q_b_f16_build_mutex);
+        if (!cuda_q4_attn_q_b_f16_release_under_build_lock()) return 0;
+        std::lock_guard<std::mutex> cache_lock(
+            g_q4_attn_q_b_f16_cache_mutex);
+        g_q4_attn_q_b_f16_multi_model_active = 1;
+    }
+    /* Keep the target mmap active and install the support payload as one
+     * host-base-keyed device range. Dynamic target SSD remaps then cannot
+     * unregister or reinterpret the secondary GGUF. The caller has selected
+     * the support fd before entering this function. */
+    const char *ptr = cuda_model_range_ptr(model_map,
+                                           map_offset,
+                                           map_size,
+                                           "DSpark support model");
+    return ptr != NULL &&
+           cuda_model_range_is_cached(model_map, map_offset, map_size);
+}
+
 /* Register the mmap'd host model pointer for selective-cache lookups WITHOUT
  * triggering any device-side copy. Used by multi-GPU placement scaffolding's
  * multi-tier path so DS4_CUDA_COPY_MODEL cannot reintroduce a full-model
@@ -3873,8 +9196,12 @@ extern "C" int ds4_gpu_set_aux_model_map_range(
 extern "C" int ds4_gpu_register_model_map_no_copy(const void *model_map, uint64_t model_size) {
     if (!model_map || model_size == 0) return 0;
     if (g_model_host_base == model_map && g_model_registered_size == model_size) return 1;
+    cuda_q8_fold_invalidate_all();
 
-    cuda_stream_selected_cache_release();
+    if (!cuda_stream_expert_storage_release(1)) return 0;
+    if (!ds4_gpu_release_q4_attn_q_b_f16_sidecars()) return 0;
+    cuda_q4_attn_q_b_f16_set_multi_model_policy(0);
+    cuda_f16_pair_chunk32_release_all();
     cuda_model_range_release_all();
     cuda_q8_f16_cache_release_all();
     g_q8_f16_disabled_after_oom = 0;
@@ -4529,12 +9856,11 @@ extern "C" int ds4_gpu_build_derived_artifacts(
     g_derived_artifact_bytes = built_bytes;
     g_derived_artifact_build_secs = cuda_wall_sec() - t0;
     if (!g_aligned_q81_scratch) {
-        const size_t scratch_bytes = 96u * 1024u * 1024u;
         cudaError_t scratch_err = cudaMalloc(&g_aligned_q81_scratch,
-                                             scratch_bytes);
+                                             CUDA_ALIGNED_Q81_SCRATCH_BYTES);
         if (scratch_err == cudaSuccess) {
             ds4_mmq_set_aligned_q81_scratch(g_aligned_q81_scratch,
-                                             scratch_bytes);
+                                             CUDA_ALIGNED_Q81_SCRATCH_BYTES);
         } else {
             g_aligned_q81_scratch = NULL;
             (void)cudaGetLastError();
@@ -4607,6 +9933,12 @@ extern "C" void ds4_gpu_print_memory_report(const char *label) {
 }
 
 extern "C" void ds4_gpu_set_quality(bool quality) {
+    if (quality && !g_quality_mode &&
+        !ds4_gpu_release_q4_attn_q_b_f16_sidecars()) {
+        fprintf(stderr,
+                "ds4: CUDA could not safely release Q4 attn_q_b F16 "
+                "sidecars while enabling quality mode\n");
+    }
     g_quality_mode = quality ? 1 : 0;
     const cublasMath_t math_mode =
         (g_quality_mode || getenv("DS4_CUDA_NO_TF32") != NULL)
@@ -4787,6 +10119,99 @@ __global__ static void matmul_f16_small_out_hx_ordered_chunks_kernel(
     }
 }
 
+/* One-row DS4 HC prelude: unweighted RMSNorm (16384 floats) followed by the
+ * narrow 24-row F16 mixer.  Every block recreates the 256-thread reduction of
+ * rms_norm_plain_batch8_kernel, then two warps reproduce the contiguous
+ * 32-chunk accumulation and lane-ordered final sum used by the established
+ * one-token F16 matvec.  Recomputing the tiny norm in twelve blocks keeps all
+ * mixer rows parallel while avoiding the normalized 64 KiB device round trip
+ * and one launch. */
+__global__ static void hc_rms_norm_mix_f16_kernel(
+        float *out,
+        const float *x,
+        const __half *w,
+        float eps,
+        int round_x_to_f16) {
+    constexpr uint32_t N = 16384u;
+    constexpr uint32_t OUT_DIM = 24u;
+    constexpr uint32_t NORM_THREADS = 256u;
+    constexpr uint32_t ROWS_PER_BLOCK = 2u;
+    constexpr uint32_t MATVEC_THREADS = 32u;
+    constexpr uint32_t MATVEC_CHUNK = N / MATVEC_THREADS;
+
+    const uint32_t tid = threadIdx.x;
+    float norm_sum = 0.0f;
+    if (tid < NORM_THREADS) {
+#pragma unroll 1
+        for (uint32_t i = tid; i < N; i += 2048u) {
+            const float v0 = x[i];
+            const float v1 = x[i + 256u];
+            const float v2 = x[i + 512u];
+            const float v3 = x[i + 768u];
+            const float v4 = x[i + 1024u];
+            const float v5 = x[i + 1280u];
+            const float v6 = x[i + 1536u];
+            const float v7 = x[i + 1792u];
+            norm_sum += v0 * v0;
+            norm_sum += v1 * v1;
+            norm_sum += v2 * v2;
+            norm_sum += v3 * v3;
+            norm_sum += v4 * v4;
+            norm_sum += v5 * v5;
+            norm_sum += v6 * v6;
+            norm_sum += v7 * v7;
+        }
+    }
+
+    __shared__ float norm_partial[NORM_THREADS];
+    __shared__ float mv_partial[ROWS_PER_BLOCK * MATVEC_THREADS];
+    if (tid < NORM_THREADS) norm_partial[tid] = norm_sum;
+    __syncthreads();
+    for (uint32_t stride = NORM_THREADS >> 1u; stride > 0u; stride >>= 1u) {
+        if (tid < stride) {
+            norm_partial[tid] += norm_partial[tid + stride];
+        }
+        __syncthreads();
+    }
+    const float scale = rsqrtf(norm_partial[0] / (float)N + eps);
+
+    if (tid < ROWS_PER_BLOCK * MATVEC_THREADS) {
+        const uint32_t local_row = tid / MATVEC_THREADS;
+        const uint32_t lane = tid % MATVEC_THREADS;
+        const uint32_t row = blockIdx.x * ROWS_PER_BLOCK + local_row;
+        float sum = 0.0f;
+        if (row < OUT_DIM) {
+            const uint32_t k0 = lane * MATVEC_CHUNK;
+            const uint32_t k1 = k0 + MATVEC_CHUNK;
+            const __half *wr = w + (uint64_t)row * N;
+#pragma unroll 1
+            for (uint32_t i = k0; i < k1; i++) {
+                float xv = x[i] * scale;
+                if (round_x_to_f16) {
+                    xv = __half2float(__float2half(xv));
+                }
+                sum += __half2float(wr[i]) * xv;
+            }
+        }
+        mv_partial[local_row * MATVEC_THREADS + lane] = sum;
+    }
+    __syncthreads();
+
+    if (tid < ROWS_PER_BLOCK * MATVEC_THREADS &&
+        (tid % MATVEC_THREADS) == 0u) {
+        const uint32_t local_row = tid / MATVEC_THREADS;
+        const uint32_t row = blockIdx.x * ROWS_PER_BLOCK + local_row;
+        if (row < OUT_DIM) {
+            float total = 0.0f;
+#pragma unroll 1
+            for (uint32_t lane = 0; lane < MATVEC_THREADS; lane++) {
+                total += mv_partial[local_row * MATVEC_THREADS + lane];
+            }
+            out[row] = total;
+        }
+    }
+}
+
 __global__ static void matmul_f16_small_out_batch_kernel(
         float *out,
         const __half *w,
@@ -4883,6 +10308,290 @@ __global__ static void matmul_f16_pair_ordered_chunks_kernel(
     }
 }
 
+/* Decode compressor projection with the state-store epilogue folded into
+ * the ordered F16 pair matvec.  The dot products and final lane-ordered sums
+ * are verbatim matmul_f16_pair_ordered_chunks_kernel; only lane 0 performs
+ * the independent state writes that otherwise require a second launch. */
+__global__ static void matmul_f16_pair_compressor_store_ordered_chunks_kernel(
+        float *out_kv,
+        float *out_score,
+        float *state_kv,
+        float *state_score,
+        const __half *w_kv,
+        const __half *w_score,
+        const float *x,
+        const void *ape,
+        uint32_t ape_type,
+        uint64_t in_dim,
+        uint32_t width,
+        uint32_t ratio,
+        uint32_t pos) {
+    const uint32_t row = blockIdx.x;
+    if (row >= width) return;
+
+    __shared__ float partial_kv[32];
+    __shared__ float partial_score[32];
+    const uint32_t tid = threadIdx.x;
+    float sum_kv = 0.0f;
+    float sum_score = 0.0f;
+    const uint64_t chunk = (in_dim + 31u) / 32u;
+    const uint64_t k0 = (uint64_t)tid * chunk;
+    uint64_t k1 = k0 + chunk;
+    if (k1 > in_dim) k1 = in_dim;
+    const __half *wr_kv = w_kv + (uint64_t)row * in_dim;
+    const __half *wr_score = w_score + (uint64_t)row * in_dim;
+    for (uint64_t i = k0; i < k1; i++) {
+        const float xv = x[i];
+        sum_kv += __half2float(wr_kv[i]) * xv;
+        sum_score += __half2float(wr_score[i]) * xv;
+    }
+    partial_kv[tid] = sum_kv;
+    partial_score[tid] = sum_score;
+    __syncthreads();
+    if (tid == 0u) {
+        float total_kv = 0.0f;
+        float total_score = 0.0f;
+        for (uint32_t i = 0; i < 32u; i++) {
+            total_kv += partial_kv[i];
+            total_score += partial_score[i];
+        }
+        out_kv[row] = total_kv;
+        out_score[row] = total_score;
+        const uint32_t pos_mod = pos % ratio;
+        const uint32_t dst_row = ratio == 4u ? ratio + pos_mod : pos_mod;
+        const uint64_t ape_index = (uint64_t)pos_mod * width + row;
+        const float ape_value = ape_type == 1u
+            ? __half2float(((const __half *)ape)[ape_index])
+            : ((const float *)ape)[ape_index];
+        state_kv[(uint64_t)dst_row * width + row] = total_kv;
+        state_score[(uint64_t)dst_row * width + row] =
+            total_score + ape_value;
+    }
+}
+
+__global__ static void f16_pair_chunk32_repack_kernel(
+        __half2 *dst,
+        const __half *w0,
+        const __half *w1,
+        uint64_t in_dim,
+        uint32_t width) {
+    const uint64_t gid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x;
+    const uint64_t count = (uint64_t)width * in_dim;
+    if (gid >= count) return;
+    const uint64_t row = gid / in_dim;
+    const uint64_t k = gid - row * in_dim;
+    const uint64_t chunk = in_dim / 32u;
+    const uint64_t lane = k / chunk;
+    const uint64_t iter = k - lane * chunk;
+    const uint64_t dst_idx = (row * chunk + iter) * 32u + lane;
+    dst[dst_idx] = __halves2half2(w0[gid], w1[gid]);
+}
+
+__global__ static void matmul_f16_pair_compressor_store_chunk32_kernel(
+        float *out_kv,
+        float *out_score,
+        float *state_kv,
+        float *state_score,
+        const __half2 *w_pair,
+        const float *x,
+        const void *ape,
+        uint32_t ape_type,
+        uint64_t in_dim,
+        uint32_t width,
+        uint32_t ratio,
+        uint32_t pos) {
+    const uint32_t row = blockIdx.x;
+    if (row >= width) return;
+
+    __shared__ float partial_kv[32];
+    __shared__ float partial_score[32];
+    const uint32_t tid = threadIdx.x;
+    float sum_kv = 0.0f;
+    float sum_score = 0.0f;
+    const uint64_t chunk = in_dim / 32u;
+    const uint64_t k0 = (uint64_t)tid * chunk;
+    const __half2 *wr = w_pair + (uint64_t)row * in_dim;
+    for (uint64_t j = 0; j < chunk; j++) {
+        const float xv = x[k0 + j];
+        const __half2 wp = wr[j * 32u + tid];
+        sum_kv += __half2float(__low2half(wp)) * xv;
+        sum_score += __half2float(__high2half(wp)) * xv;
+    }
+    partial_kv[tid] = sum_kv;
+    partial_score[tid] = sum_score;
+    __syncthreads();
+    if (tid == 0u) {
+        float total_kv = 0.0f;
+        float total_score = 0.0f;
+        for (uint32_t i = 0; i < 32u; i++) {
+            total_kv += partial_kv[i];
+            total_score += partial_score[i];
+        }
+        out_kv[row] = total_kv;
+        out_score[row] = total_score;
+        const uint32_t pos_mod = pos % ratio;
+        const uint32_t dst_row = ratio == 4u ? ratio + pos_mod : pos_mod;
+        const uint64_t ape_index = (uint64_t)pos_mod * width + row;
+        const float ape_value = ape_type == 1u
+            ? __half2float(((const __half *)ape)[ape_index])
+            : ((const float *)ape)[ape_index];
+        state_kv[(uint64_t)dst_row * width + row] = total_kv;
+        state_score[(uint64_t)dst_row * width + row] = total_score + ape_value;
+    }
+}
+
+__global__ static void matmul_f16_pair_compressor_store_chunk32_prefetch8_kernel(
+        float *out_kv,
+        float *out_score,
+        float *state_kv,
+        float *state_score,
+        const __half2 *w_pair,
+        const float *x,
+        const void *ape,
+        uint32_t ape_type,
+        uint64_t in_dim,
+        uint32_t width,
+        uint32_t ratio,
+        uint32_t pos) {
+    const uint32_t row = blockIdx.x;
+    if (row >= width) return;
+
+    __shared__ float partial_kv[32];
+    __shared__ float partial_score[32];
+    const uint32_t tid = threadIdx.x;
+    float sum_kv = 0.0f;
+    float sum_score = 0.0f;
+    const uint64_t chunk = in_dim / 32u;
+    const uint64_t k0 = (uint64_t)tid * chunk;
+    const __half2 *wr = w_pair + (uint64_t)row * in_dim;
+    for (uint64_t j = 0; j < chunk; j += 8u) {
+        const float xv0 = x[k0 + j + 0u];
+        const float xv1 = x[k0 + j + 1u];
+        const float xv2 = x[k0 + j + 2u];
+        const float xv3 = x[k0 + j + 3u];
+        const float xv4 = x[k0 + j + 4u];
+        const float xv5 = x[k0 + j + 5u];
+        const float xv6 = x[k0 + j + 6u];
+        const float xv7 = x[k0 + j + 7u];
+        const __half2 wp0 = wr[(j + 0u) * 32u + tid];
+        const __half2 wp1 = wr[(j + 1u) * 32u + tid];
+        const __half2 wp2 = wr[(j + 2u) * 32u + tid];
+        const __half2 wp3 = wr[(j + 3u) * 32u + tid];
+        const __half2 wp4 = wr[(j + 4u) * 32u + tid];
+        const __half2 wp5 = wr[(j + 5u) * 32u + tid];
+        const __half2 wp6 = wr[(j + 6u) * 32u + tid];
+        const __half2 wp7 = wr[(j + 7u) * 32u + tid];
+        sum_kv += __half2float(__low2half(wp0)) * xv0;
+        sum_score += __half2float(__high2half(wp0)) * xv0;
+        sum_kv += __half2float(__low2half(wp1)) * xv1;
+        sum_score += __half2float(__high2half(wp1)) * xv1;
+        sum_kv += __half2float(__low2half(wp2)) * xv2;
+        sum_score += __half2float(__high2half(wp2)) * xv2;
+        sum_kv += __half2float(__low2half(wp3)) * xv3;
+        sum_score += __half2float(__high2half(wp3)) * xv3;
+        sum_kv += __half2float(__low2half(wp4)) * xv4;
+        sum_score += __half2float(__high2half(wp4)) * xv4;
+        sum_kv += __half2float(__low2half(wp5)) * xv5;
+        sum_score += __half2float(__high2half(wp5)) * xv5;
+        sum_kv += __half2float(__low2half(wp6)) * xv6;
+        sum_score += __half2float(__high2half(wp6)) * xv6;
+        sum_kv += __half2float(__low2half(wp7)) * xv7;
+        sum_score += __half2float(__high2half(wp7)) * xv7;
+    }
+    partial_kv[tid] = sum_kv;
+    partial_score[tid] = sum_score;
+    __syncthreads();
+    if (tid == 0u) {
+        float total_kv = 0.0f;
+        float total_score = 0.0f;
+        for (uint32_t i = 0; i < 32u; i++) {
+            total_kv += partial_kv[i];
+            total_score += partial_score[i];
+        }
+        out_kv[row] = total_kv;
+        out_score[row] = total_score;
+        const uint32_t pos_mod = pos % ratio;
+        const uint32_t dst_row = ratio == 4u ? ratio + pos_mod : pos_mod;
+        const uint64_t ape_index = (uint64_t)pos_mod * width + row;
+        const float ape_value = ape_type == 1u
+            ? __half2float(((const __half *)ape)[ape_index])
+            : ((const float *)ape)[ape_index];
+        state_kv[(uint64_t)dst_row * width + row] = total_kv;
+        state_score[(uint64_t)dst_row * width + row] = total_score + ape_value;
+    }
+}
+
+static const __half2 *cuda_f16_pair_chunk32_get(
+        const void *model_map,
+        uint64_t weight0_offset,
+        uint64_t weight1_offset,
+        const __half *w0,
+        const __half *w1,
+        uint64_t in_dim,
+        uint32_t width,
+        int logical_tier) {
+    const int device_id = (logical_tier >= 0 && logical_tier < g_n_gpus)
+        ? g_gpu[logical_tier].device_id : logical_tier;
+    for (const cuda_f16_pair_chunk32_range &r : g_f16_pair_chunk32_ranges) {
+        if (r.host_base == model_map &&
+            r.weight0_offset == weight0_offset &&
+            r.weight1_offset == weight1_offset &&
+            r.in_dim == in_dim && r.width == width &&
+            r.device_id == device_id) {
+            return r.device_ptr;
+        }
+    }
+    if (g_f16_pair_chunk32_disabled_after_oom) return NULL;
+    /* Lazy cudaMalloc is forbidden while the decode island capture is active.
+     * Querying the legacy stream would miss capture on g_decode_graph_stream. */
+    if (g_decode_graph_capturing) return NULL;
+
+    cudaStreamCaptureStatus capture_status = cudaStreamCaptureStatusNone;
+    if (cudaStreamIsCapturing(0, &capture_status) != cudaSuccess ||
+        capture_status != cudaStreamCaptureStatusNone) {
+        (void)cudaGetLastError();
+        return NULL;
+    }
+    if (in_dim == 0u || (in_dim % 32u) != 0u ||
+        (uint64_t)width > UINT64_MAX / in_dim) return NULL;
+    const uint64_t count = (uint64_t)width * in_dim;
+    if (count > SIZE_MAX / sizeof(__half2)) return NULL;
+
+    int previous_device = -1;
+    if (cudaGetDevice(&previous_device) != cudaSuccess ||
+        cudaSetDevice(device_id) != cudaSuccess) {
+        (void)cudaGetLastError();
+        return NULL;
+    }
+    __half2 *device_ptr = NULL;
+    cudaError_t err = cudaMalloc(&device_ptr, (size_t)count * sizeof(__half2));
+    if (err != cudaSuccess) {
+        fprintf(stderr,
+                "ds4: CUDA exact F16 compressor transpose disabled after "
+                "allocation failure (%.2f MiB): %s\n",
+                (double)(count * sizeof(__half2)) / 1048576.0,
+                cudaGetErrorString(err));
+        (void)cudaGetLastError();
+        g_f16_pair_chunk32_disabled_after_oom = 1;
+        if (previous_device >= 0) (void)cudaSetDevice(previous_device);
+        return NULL;
+    }
+    const uint64_t blocks = (count + 255u) / 256u;
+    f16_pair_chunk32_repack_kernel<<<(unsigned)blocks, 256>>>(
+        device_ptr, w0, w1, in_dim, width);
+    err = cudaGetLastError();
+    if (err != cudaSuccess) {
+        (void)cudaFree(device_ptr);
+        if (previous_device >= 0) (void)cudaSetDevice(previous_device);
+        return NULL;
+    }
+    g_f16_pair_chunk32_ranges.push_back({
+        model_map, weight0_offset, weight1_offset, in_dim, width,
+        device_ptr, device_id});
+    if (previous_device >= 0) (void)cudaSetDevice(previous_device);
+    return device_ptr;
+}
+
 __global__ static void matmul_f32_kernel(
         float *out,
         const float *w,
@@ -4991,47 +10700,19 @@ __device__ __forceinline__ static int32_t dot_i8_block(const int8_t *a, const in
     return dot;
 }
 
-__global__ static DS4_CUDA_UNUSED void matmul_q8_0_kernel(
-        float *out,
-        const unsigned char *w,
-        const float *x,
-        uint64_t in_dim,
-        uint64_t out_dim,
-        uint64_t n_tok) {
-    uint64_t row = (uint64_t)blockIdx.x;
-    uint64_t tok = (uint64_t)blockIdx.y;
-    if (row >= out_dim || tok >= n_tok) return;
-    const uint64_t blocks = (in_dim + 31) / 32;
-    const unsigned char *wr = w + row * blocks * 34;
-    const float *xr = x + tok * in_dim;
-    float acc = 0.0f;
-
-    for (uint64_t b = threadIdx.x; b < blocks; b += blockDim.x) {
-        uint64_t i0 = b * 32;
-        uint64_t bn = in_dim - i0 < 32 ? in_dim - i0 : 32;
-        float amax = 0.0f;
-        for (uint64_t i = 0; i < bn; i++) amax = fmaxf(amax, fabsf(xr[i0 + i]));
-        float d = amax / 127.0f;
-        float id = d != 0.0f ? 1.0f / d : 0.0f;
-        const __half *scale_h = (const __half *)(wr + b * 34);
-        const int8_t *qs = (const int8_t *)(wr + b * 34 + 2);
-        int dot = 0;
-        for (uint64_t i = 0; i < bn; i++) {
-            int q = (int)lrintf(xr[i0 + i] * id);
-            q = q > 127 ? 127 : (q < -128 ? -128 : q);
-            dot += (int)qs[i] * q;
-        }
-        acc += __half2float(*scale_h) * d * (float)dot;
-    }
-
-    __shared__ float partial[256];
-    partial[threadIdx.x] = acc;
-    __syncthreads();
-    for (uint32_t stride = blockDim.x >> 1; stride > 0; stride >>= 1) {
-        if (threadIdx.x < stride) partial[threadIdx.x] += partial[threadIdx.x + stride];
-        __syncthreads();
-    }
-    if (threadIdx.x == 0) out[tok * out_dim + row] = partial[0];
+__device__ __forceinline__ static int32_t dot_i8x32_aligned_int4(
+        const int4 &a0, const int4 &a1, const int8_t *b) {
+    const int32_t *bi = (const int32_t *)b;
+    int32_t dot = 0;
+    dot = __dp4a(a0.x, bi[0], dot);
+    dot = __dp4a(a0.y, bi[1], dot);
+    dot = __dp4a(a0.z, bi[2], dot);
+    dot = __dp4a(a0.w, bi[3], dot);
+    dot = __dp4a(a1.x, bi[4], dot);
+    dot = __dp4a(a1.y, bi[5], dot);
+    dot = __dp4a(a1.z, bi[6], dot);
+    dot = __dp4a(a1.w, bi[7], dot);
+    return dot;
 }
 
 __global__ static void quantize_q8_0_f32_kernel(
@@ -5339,6 +11020,51 @@ __global__ static void matmul_q8_0_pair_preq_warp8_kernel(
     }
 }
 
+__global__ static void matmul_q8_0_pair_aligned_preq_warp8_kernel(
+        float *out0,
+        float *out1,
+        const int4 *w0_qs,
+        const __half *w0_dq,
+        const int4 *w1_qs,
+        const __half *w1_dq,
+        const int8_t *xq,
+        const float *xscale,
+        uint64_t out0_dim,
+        uint64_t out1_dim,
+        uint64_t blocks) {
+    const uint64_t row = (uint64_t)blockIdx.x * 8u + (threadIdx.x >> 5u);
+    const uint64_t tok = (uint64_t)blockIdx.y;
+    const uint32_t lane = threadIdx.x & 31u;
+    if (row >= out0_dim && row >= out1_dim) return;
+    float acc0 = 0.0f;
+    float acc1 = 0.0f;
+    const int8_t *xqr = xq + tok * blocks * 32u;
+    const float *xsr = xscale + tok * blocks;
+    const uint64_t rbase = row * blocks;
+    for (uint64_t b = lane; b < blocks; b += 32u) {
+        const int8_t *xqb = xqr + b * 32u;
+        const float xs = xsr[b];
+        if (row < out0_dim) {
+            const int4 q0 = w0_qs[(rbase + b) * 2u];
+            const int4 q1 = w0_qs[(rbase + b) * 2u + 1u];
+            const int32_t dot = dot_i8x32_aligned_int4(q0, q1, xqb);
+            acc0 += __half2float(w0_dq[rbase + b]) * xs * (float)dot;
+        }
+        if (row < out1_dim) {
+            const int4 q0 = w1_qs[(rbase + b) * 2u];
+            const int4 q1 = w1_qs[(rbase + b) * 2u + 1u];
+            const int32_t dot = dot_i8x32_aligned_int4(q0, q1, xqb);
+            acc1 += __half2float(w1_dq[rbase + b]) * xs * (float)dot;
+        }
+    }
+    acc0 = warp_sum_f32(acc0);
+    acc1 = warp_sum_f32(acc1);
+    if (lane == 0u) {
+        if (row < out0_dim) out0[tok * out0_dim + row] = acc0;
+        if (row < out1_dim) out1[tok * out1_dim + row] = acc1;
+    }
+}
+
 __global__ static void shared_mid_q8_0_preq_warp8_exact_kernel(
         float *mid,
         const unsigned char *gate_w,
@@ -5638,6 +11364,103 @@ __global__ static void matmul_q8_0_hc_expand_preq_warp8_kernel(
     }
 }
 
+__global__ static void matmul_q8_0_hc_expand_aligned_preq_warp8_kernel(
+        float *out_hc,
+        float *block_out,
+        const float *block_add,
+        const float *block_add2,
+        const float *owned_home_slots,
+        const float *owned_peer_packed,
+        const int32_t *owned_selected,
+        const float *residual_hc,
+        const float *split,
+        const int4 *w_qs,
+        const __half *w_dq,
+        const int8_t *xq,
+        const float *xscale,
+        uint64_t out_dim,
+        uint32_t n_embd,
+        uint32_t n_hc,
+        uint64_t blocks,
+        int has_add,
+        int has_add2,
+        int has_owned_slots,
+        uint32_t owned_expert_split) {
+    const uint64_t row = (uint64_t)blockIdx.x * 8u + (threadIdx.x >> 5u);
+    const uint32_t lane = threadIdx.x & 31u;
+    if (row >= out_dim) return;
+    const uint64_t rbase = row * blocks;
+    float acc = 0.0f;
+    for (uint64_t b = lane; b < blocks; b += 32u) {
+        const int4 w0 = w_qs[(rbase + b) * 2u];
+        const int4 w1 = w_qs[(rbase + b) * 2u + 1u];
+        const int32_t dot = dot_i8x32_aligned_int4(w0, w1, xq + b * 32u);
+        acc += __half2float(w_dq[rbase + b]) * xscale[b] * (float)dot;
+    }
+    acc = warp_sum_f32(acc);
+    if (lane == 0) {
+        const uint32_t d = (uint32_t)row;
+        block_out[d] = acc;
+        float block_v = acc;
+        if (has_owned_slots) {
+            const float routed = moe_owned_packed_combine_row(
+                    owned_home_slots,
+                    owned_peer_packed,
+                    owned_selected,
+                    d,
+                    (uint32_t)out_dim,
+                    owned_expert_split);
+            block_v = __fadd_rn(block_v, routed);
+        } else if (has_add) {
+            float add_v = block_add[d];
+            if (has_add2) add_v += block_add2[d];
+            block_v += add_v;
+        }
+        const float *post = split + n_hc;
+        const float *comb = split + 2u * n_hc;
+        for (uint32_t dst_hc = 0; dst_hc < n_hc; dst_hc++) {
+            float hc_acc = block_v * post[dst_hc];
+            for (uint32_t src_hc = 0; src_hc < n_hc; src_hc++) {
+                const float comb_v = comb[dst_hc + (uint64_t)src_hc * n_hc];
+                const float res_v = residual_hc[(uint64_t)src_hc * n_embd + d];
+                hc_acc += comb_v * res_v;
+            }
+            out_hc[(uint64_t)dst_hc * n_embd + d] = hc_acc;
+        }
+    }
+}
+
+/* Split A/B matmul leg for the aligned artifact.  Unlike the general MMQ
+ * aligned consumer, this intentionally keeps the fused baseline's canonical
+ * Q8_0 activation codes and FP32 scale.  Lane/block assignment, multiply
+ * order and warp reduction are identical to the fused aligned kernel above;
+ * only the HC epilogue moves to its own launch. */
+__global__ static void matmul_q8_0_aligned_preq_warp8_kernel(
+        float *out,
+        const int4 *w_qs,
+        const __half *w_dq,
+        const int8_t *xq,
+        const float *xscale,
+        uint64_t out_dim,
+        uint64_t blocks) {
+    const uint64_t row =
+        (uint64_t)blockIdx.x * 8u + (threadIdx.x >> 5u);
+    const uint32_t lane = threadIdx.x & 31u;
+    if (row >= out_dim) return;
+    const uint64_t rbase = row * blocks;
+    float acc = 0.0f;
+    for (uint64_t b = lane; b < blocks; b += 32u) {
+        const int4 w0 = w_qs[(rbase + b) * 2u];
+        const int4 w1 = w_qs[(rbase + b) * 2u + 1u];
+        const int32_t dot =
+            dot_i8x32_aligned_int4(w0, w1, xq + b * 32u);
+        acc += __half2float(w_dq[rbase + b]) *
+               xscale[b] * (float)dot;
+    }
+    acc = warp_sum_f32(acc);
+    if (lane == 0u) out[row] = acc;
+}
+
 __global__ static void matmul_q8_0_kslice_hc_expand_add_preq_warp8_kernel(
         float *out_hc,
         float *block_out,
@@ -6372,6 +12195,39 @@ __global__ static void grouped_q8_0_a_preq_warp8_kernel(
     if (lane == 0) low[tok * low_dim + row] = acc;
 }
 
+__global__ static void grouped_q8_0_a_aligned_preq_warp8_kernel(
+        float *low,
+        const int4 *w_qs,
+        const __half *w_dq,
+        const int8_t *xq,
+        const float *xscale,
+        uint64_t rank,
+        uint32_t n_groups,
+        uint32_t n_tokens,
+        uint64_t blocks) {
+    const uint64_t row = (uint64_t)blockIdx.x * 8u + (threadIdx.x >> 5u);
+    const uint64_t tok = (uint64_t)blockIdx.y;
+    const uint32_t lane = threadIdx.x & 31u;
+    const uint64_t low_dim = (uint64_t)n_groups * rank;
+    if (row >= low_dim || tok >= n_tokens) return;
+
+    const uint64_t group = row / rank;
+    const uint64_t xrow = tok * (uint64_t)n_groups + group;
+    const int8_t *xqr = xq + xrow * blocks * 32u;
+    const float *xsr = xscale + xrow * blocks;
+    const uint64_t rbase = row * blocks;
+    float acc = 0.0f;
+    for (uint64_t b = lane; b < blocks; b += 32u) {
+        const int4 q0 = w_qs[(rbase + b) * 2u];
+        const int4 q1 = w_qs[(rbase + b) * 2u + 1u];
+        const int32_t dot = dot_i8x32_aligned_int4(
+            q0, q1, xqr + b * 32u);
+        acc += __half2float(w_dq[rbase + b]) * xsr[b] * (float)dot;
+    }
+    acc = warp_sum_f32(acc);
+    if (lane == 0u) low[tok * low_dim + row] = acc;
+}
+
 __global__ static void grouped_q8_0_a_preq_warp8_tok2_kernel(
         float *low,
         const unsigned char *w,
@@ -6821,6 +12677,96 @@ __global__ static void head_rms_norm_rope_tail_kernel(
     }
 }
 
+/* Fused epilogue for the resident Q4 attn_q_b GEMM.  cuBLAS writes the
+ * projection once as F16; this kernel converts directly to the canonical
+ * F32 graph tensor while applying the same per-head RMS normalization and
+ * RoPE tail as head_rms_norm_rope_tail_kernel. */
+__global__ static void head_rms_norm_rope_tail_from_half_kernel(
+        float *out,
+        const __half *x,
+        uint32_t n_tok,
+        uint32_t n_head,
+        uint32_t head_dim,
+        uint32_t n_rot,
+        uint32_t pos0,
+        uint32_t n_ctx_orig,
+        int inverse,
+        float freq_base,
+        float freq_scale,
+        float ext_factor,
+        float attn_factor,
+        float beta_fast,
+        float beta_slow,
+        float eps) {
+    const uint32_t row = blockIdx.x;
+    if (row >= n_tok * n_head) return;
+    const uint32_t t = row / n_head;
+    const __half *xr = x + (uint64_t)row * head_dim;
+    float *orow = out + (uint64_t)row * head_dim;
+    float sum = 0.0f;
+    for (uint32_t i = threadIdx.x; i < head_dim; i += blockDim.x) {
+        const float v = __half2float(xr[i]);
+        sum += v * v;
+    }
+    __shared__ float partial[256];
+    partial[threadIdx.x] = sum;
+    __syncthreads();
+    for (uint32_t stride = blockDim.x >> 1; stride > 0; stride >>= 1) {
+        if (threadIdx.x < stride) {
+            partial[threadIdx.x] += partial[threadIdx.x + stride];
+        }
+        __syncthreads();
+    }
+
+    const float scale = rsqrtf(partial[0] / (float)head_dim + eps);
+    const uint32_t n_nope = head_dim - n_rot;
+    for (uint32_t i = threadIdx.x; i < n_nope; i += blockDim.x) {
+        orow[i] = __half2float(xr[i]) * scale;
+    }
+
+    float corr0 = 0.0f, corr1 = 0.0f;
+    if (ext_factor != 0.0f) {
+        const float denom = 2.0f * logf(freq_base);
+        corr0 = floorf((float)n_rot *
+                       logf((float)n_ctx_orig /
+                            (beta_fast * 2.0f * (float)M_PI)) /
+                       denom);
+        corr1 = ceilf((float)n_rot *
+                      logf((float)n_ctx_orig /
+                           (beta_slow * 2.0f * (float)M_PI)) /
+                      denom);
+        corr0 = fmaxf(0.0f, corr0);
+        corr1 = fminf((float)(n_rot - 1u), corr1);
+    }
+    const __half *tail = xr + n_nope;
+    float *out_tail = orow + n_nope;
+    for (uint32_t pair = threadIdx.x;
+         pair < n_rot / 2u;
+         pair += blockDim.x) {
+        const uint32_t i = pair * 2u;
+        const float theta_extrap =
+            (float)(pos0 + t) *
+            powf(freq_base, -((float)i) / (float)n_rot);
+        const float theta_interp = freq_scale * theta_extrap;
+        float theta = theta_interp;
+        float mscale = attn_factor;
+        if (ext_factor != 0.0f) {
+            const float ramp_mix =
+                rope_yarn_ramp_dev(corr0, corr1, (int)i) * ext_factor;
+            theta = theta_interp * (1.0f - ramp_mix) +
+                    theta_extrap * ramp_mix;
+            mscale *= 1.0f + 0.1f * logf(1.0f / freq_scale);
+        }
+        const float c = cosf(theta) * mscale;
+        float s = sinf(theta) * mscale;
+        if (inverse) s = -s;
+        const float x0 = __half2float(tail[i]) * scale;
+        const float x1 = __half2float(tail[i + 1u]) * scale;
+        out_tail[i] = x0 * c - x1 * s;
+        out_tail[i + 1u] = x0 * s + x1 * c;
+    }
+}
+
 __device__ static float rope_yarn_ramp_dev(float low, float high, int i0) {
     float y = ((float)(i0 / 2) - low) / fmaxf(0.001f, high - low);
     return 1.0f - fminf(1.0f, fmaxf(0.0f, y));
@@ -7000,38 +12946,6 @@ __device__ static float model_scalar_dev(const void *base, uint64_t offset, uint
     return ((const float *)p)[idx];
 }
 
-__device__ static float rope_yarn_ramp_cpu_equiv_dev(float low, float high, int i0) {
-    float y = ((float)(i0 / 2) - low) / fmaxf(0.001f, high - low);
-    return 1.0f - fminf(1.0f, fmaxf(0.0f, y));
-}
-
-__device__ static DS4_CUDA_UNUSED void rope_tail_one_dev(float *x, uint32_t head_dim, uint32_t n_rot, uint32_t pos, uint32_t n_ctx_orig, float freq_base, float freq_scale, float ext_factor, float attn_factor, float beta_fast, float beta_slow) {
-    uint32_t n_nope = head_dim - n_rot;
-    float corr0 = 0.0f, corr1 = 0.0f;
-    if (ext_factor != 0.0f) {
-        float denom = 2.0f * logf(freq_base);
-        corr0 = fmaxf(0.0f, floorf((float)n_rot * logf((float)n_ctx_orig / (beta_fast * 2.0f * (float)M_PI)) / denom));
-        corr1 = fminf((float)(n_rot - 1), ceilf((float)n_rot * logf((float)n_ctx_orig / (beta_slow * 2.0f * (float)M_PI)) / denom));
-    }
-    for (uint32_t i = 0; i < n_rot; i += 2) {
-        float theta_extrap = (float)pos * powf(freq_base, -((float)i) / (float)n_rot);
-        float theta_interp = freq_scale * theta_extrap;
-        float theta = theta_interp;
-        float mscale = attn_factor;
-        if (ext_factor != 0.0f) {
-            float mix = rope_yarn_ramp_cpu_equiv_dev(corr0, corr1, (int)i) * ext_factor;
-            theta = theta_interp * (1.0f - mix) + theta_extrap * mix;
-            mscale *= 1.0f + 0.1f * logf(1.0f / freq_scale);
-        }
-        float c = cosf(theta) * mscale;
-        float s = sinf(theta) * mscale;
-        float x0 = x[n_nope + i];
-        float x1 = x[n_nope + i + 1];
-        x[n_nope + i] = x0 * c - x1 * s;
-        x[n_nope + i + 1] = x0 * s + x1 * c;
-    }
-}
-
 __device__ static void fp8_kv_quantize_row(
         float    *xr,
         uint32_t  head_dim,
@@ -10539,18 +16453,6 @@ __device__ __forceinline__ void tt_ldmatrix_x2_trans_addr(uint32_t (&r)[2], unsi
 #endif
 }
 
-__device__ __forceinline__ void tt_ldmatrix_x4(uint32_t (&r)[4], const void *p) {
-    tt_ldmatrix_x4_addr(r, tt_smem_addr(p));
-}
-
-__device__ __forceinline__ void tt_ldmatrix_x2(uint32_t (&r)[2], const void *p) {
-    tt_ldmatrix_x2_addr(r, tt_smem_addr(p));
-}
-
-__device__ __forceinline__ void tt_ldmatrix_x2_trans(uint32_t (&r)[2], const void *p) {
-    tt_ldmatrix_x2_trans_addr(r, tt_smem_addr(p));
-}
-
 __device__ __forceinline__ void tt_mma_m16n8k16_f16_f32(
         float *d,
         const uint32_t (&a)[4],
@@ -11775,6 +17677,26 @@ __global__ static void hc_split_weighted_sum_fused_kernel(
     }
 }
 
+/* Emit one canonical block_q8_1 from a warp holding 32 consecutive normalized
+ * values.  This is intentionally the same XOR butterfly, scale, rounding and
+ * half2 conversion as cuda/mmq/quantize.cu::quantize_q8_1. */
+__device__ static __forceinline__ void cuda_q8_fold_store_warp(
+        cuda_block_q8_1 *q81, uint64_t element, float value) {
+    const uint32_t lane = threadIdx.x & 31u;
+    float amax = fabsf(value);
+    float sum = value;
+#pragma unroll
+    for (uint32_t offset = 16u; offset > 0u; offset >>= 1u) {
+        amax = fmaxf(amax, __shfl_xor_sync(0xffffffffu, amax, offset, 32));
+        sum += __shfl_xor_sync(0xffffffffu, sum, offset, 32);
+    }
+    const float d = amax / 127.0f;
+    const int8_t q = amax == 0.0f ? (int8_t)0 : (int8_t)roundf(value / d);
+    cuda_block_q8_1 *block = q81 + element / 32u;
+    block->qs[lane] = q;
+    if (lane == 0u) block->ds = __floats2half2_rn(d, sum);
+}
+
 __global__ static void hc_split_weighted_sum_norm_fused_kernel(
         float *out,
         float *norm_out,
@@ -11789,7 +17711,8 @@ __global__ static void hc_split_weighted_sum_norm_fused_kernel(
         uint32_t n_rows,
         uint32_t sinkhorn_iters,
         float epsv,
-        float norm_eps) {
+        float norm_eps,
+        cuda_block_q8_1 *q81) {
     const uint32_t t = blockIdx.x;
     const uint32_t d = threadIdx.x;
     if (t >= n_rows || n_hc != 4) return;
@@ -11818,7 +17741,97 @@ __global__ static void hc_split_weighted_sum_norm_fused_kernel(
     const float norm_scale = rsqrtf(partial[0] / (float)n_embd + norm_eps);
     for (uint32_t col = d; col < n_embd; col += blockDim.x) {
         const float v = out[(uint64_t)t * n_embd + col];
-        norm_out[(uint64_t)t * n_embd + col] = v * norm_scale * norm_w[col];
+        const uint64_t element = (uint64_t)t * n_embd + col;
+        const float normalized = v * norm_scale * norm_w[col];
+        norm_out[element] = normalized;
+        if (q81) cuda_q8_fold_store_warp(q81, element, normalized);
+    }
+}
+
+/* Split form of the exact single-row HC weighted-sum + RMS kernel.  The
+ * reference kernel uses one 256-thread CTA; at n_embd=4096 that leaves one CTA
+ * to read 64 KiB and serialize both phases.  Sixteen partial CTAs compute the
+ * same per-column weighted sums.  A small reduction CTA then replays the
+ * original per-thread ascending-column FMA chain and shared-memory tree from
+ * those exact outputs, and a sixteen-CTA store writes the normalized values.
+ * The output tensors and floating-point reduction DAG are bit-identical. */
+__global__ static void hc_split_weighted_sum_norm_fused_partial4096_kernel(
+        float *out,
+        float *split,
+        const float *mix,
+        const float *residual_hc,
+        const float *scale,
+        const float *base,
+        uint32_t sinkhorn_iters,
+        float epsv) {
+    constexpr uint32_t n_embd = 4096u;
+    constexpr uint32_t mix_hc = 24u;
+    constexpr uint32_t tile = 256u;
+    const uint32_t d = threadIdx.x;
+    const uint32_t tile0 = blockIdx.x * tile;
+
+    __shared__ float sp[mix_hc];
+    if (d == 0u) {
+        hc4_split_one(sp, mix, scale, base, sinkhorn_iters, epsv);
+    }
+    __syncthreads();
+    if (blockIdx.x == 0u && d < mix_hc) {
+        split[d] = sp[d];
+    }
+
+#pragma unroll
+    for (uint32_t j = 0; j < tile / 256u; j++) {
+        const uint32_t col = tile0 + j * 256u + d;
+        float acc = 0.0f;
+#pragma unroll
+        for (uint32_t h = 0; h < 4u; h++) {
+            acc += residual_hc[(uint64_t)h * n_embd + col] * sp[h];
+        }
+        out[col] = acc;
+    }
+}
+
+__global__ static void hc_split_weighted_sum_norm_fused_reduce4096_kernel(
+        const float *out,
+        float *norm_scale,
+        float norm_eps) {
+    constexpr uint32_t n_embd = 4096u;
+    const uint32_t d = threadIdx.x;
+
+    __shared__ float partial[256];
+    float sum = 0.0f;
+#pragma unroll
+    for (uint32_t j = 0; j < n_embd / 256u; j++) {
+        const uint32_t col = d + j * 256u;
+        sum += out[col] * out[col];
+    }
+    partial[d] = sum;
+    __syncthreads();
+    for (uint32_t stride = blockDim.x >> 1; stride > 0; stride >>= 1) {
+        if (d < stride) partial[d] += partial[d + stride];
+        __syncthreads();
+    }
+    if (d == 0u) {
+        norm_scale[0] = rsqrtf(partial[0] / (float)n_embd + norm_eps);
+    }
+}
+
+__global__ static void hc_split_weighted_sum_norm_fused_store4096_kernel(
+        const float *out,
+        float *norm_out,
+        const float *norm_w,
+        const float *norm_scale,
+        cuda_block_q8_1 *q81) {
+    constexpr uint32_t tile = 256u;
+    const uint32_t d = threadIdx.x;
+    const uint32_t tile0 = blockIdx.x * tile;
+    const float scale = norm_scale[0];
+#pragma unroll
+    for (uint32_t j = 0; j < tile / 256u; j++) {
+        const uint32_t col = tile0 + j * 256u + d;
+        const float normalized = out[col] * scale * norm_w[col];
+        norm_out[col] = normalized;
+        if (q81) cuda_q8_fold_store_warp(q81, col, normalized);
     }
 }
 
@@ -13218,6 +19231,7 @@ static void *indexer_mxf4_scratch_alloc(uint64_t bytes) {
     if (g_indexer_mxf4_scratch_bytes >= bytes) {
         return g_indexer_mxf4_scratch;
     }
+    cuda_q8_fold_invalidate_all();
 
     int device = -1;
     if (!cuda_ok(cudaGetDevice(&device),
@@ -13365,18 +19379,32 @@ __device__ __forceinline__ static void top2_insert_candidate(
 }
 
 /* DSpark markov chain step: out = argmax_i(logits[i] + dot(w2[i], w1[prev]))
- * over the vocab, entirely on-device (logits row stays resident; the chain
- * loop only reads back 4 bytes per draft). w1/w2 are q8_0 with 272-byte rows
- * (8 blocks of 32). Single block; ~35 MB w2 read per step. */
+ * over the vocab, entirely on-device.  w1/w2 are q8_0 with 272-byte rows
+ * (8 blocks of 32); the legacy wrapper reads one key after each step, while
+ * the batched proposer below feeds the selected token directly to the next
+ * launch and performs one result readback for the whole chain. */
 __global__ static void dspark_markov_argmax_kernel(
         unsigned long long *out_key,
         const float *logits,
-        const unsigned char *w1_row,
+        const unsigned char *w1,
         const unsigned char *w2,
         uint32_t vocab,
-        uint32_t rank_blocks) {
+        uint32_t rank_blocks,
+        const uint32_t *prev_token_device,
+        const uint32_t *active_device) {
     __shared__ float state[256];
+    __shared__ uint32_t prev_token;
+    __shared__ uint32_t active;
     const uint32_t tid = threadIdx.x;
+    if (tid == 0u) {
+        if (prev_token_device) prev_token = *prev_token_device;
+        active = active_device ? *active_device : 1u;
+    }
+    __syncthreads();
+    if (active == 0u) return;
+    const unsigned char *w1_row = prev_token_device
+        ? w1 + (uint64_t)prev_token * rank_blocks * 34u
+        : w1;
     if (tid < rank_blocks * 32u) {
         const uint32_t b = tid >> 5, k = tid & 31u;
         const unsigned char *blk = w1_row + (uint64_t)b * 34u;
@@ -13433,6 +19461,168 @@ __global__ static void dspark_markov_argmax_kernel(
     }
 }
 
+/* Device-resident DSpark proposer tail.  ds4_cuda.cu intentionally does not
+ * include ds4_gpu.h, so keep this result declaration byte-for-byte identical
+ * to ds4_gpu_dspark_device_proposal there. */
+#define DS4_GPU_DSPARK_MAX_DRAFTS 6u
+#define DS4_GPU_DSPARK_DEVICE_PROPOSAL_BYTES 2048u
+typedef struct {
+    int32_t  tokens[DS4_GPU_DSPARK_MAX_DRAFTS];
+    float    confidence_logits[DS4_GPU_DSPARK_MAX_DRAFTS];
+    uint32_t proposal_len;
+    uint32_t confidence_len;
+    uint32_t status;
+    uint32_t reserved;
+} ds4_gpu_dspark_device_proposal;
+static_assert(sizeof(ds4_gpu_dspark_device_proposal) == 64u,
+              "DSpark device proposal ABI must match ds4_gpu.h");
+static_assert(offsetof(ds4_gpu_dspark_device_proposal, tokens) == 0u &&
+              offsetof(ds4_gpu_dspark_device_proposal,
+                       confidence_logits) == 24u &&
+              offsetof(ds4_gpu_dspark_device_proposal, proposal_len) == 48u &&
+              offsetof(ds4_gpu_dspark_device_proposal, confidence_len) == 52u &&
+              offsetof(ds4_gpu_dspark_device_proposal, status) == 56u &&
+              offsetof(ds4_gpu_dspark_device_proposal, reserved) == 60u,
+              "DSpark device proposal field offsets must match ds4_gpu.h");
+
+typedef struct {
+    unsigned long long best_key[DS4_GPU_DSPARK_MAX_DRAFTS];
+    uint32_t prev_token;
+    uint32_t status;
+    uint32_t active;
+    uint32_t proposal_len;
+    uint32_t confidence_len;
+    int32_t  tokens[DS4_GPU_DSPARK_MAX_DRAFTS];
+    float    confidence_logits[DS4_GPU_DSPARK_MAX_DRAFTS];
+} cuda_dspark_device_proposal_state;
+static_assert(sizeof(ds4_gpu_dspark_device_proposal) +
+                  sizeof(cuda_dspark_device_proposal_state) <=
+              DS4_GPU_DSPARK_DEVICE_PROPOSAL_BYTES,
+              "DSpark device proposal storage is too small");
+
+__global__ static void dspark_device_proposal_init_kernel(
+        cuda_dspark_device_proposal_state *state,
+        uint32_t first_prev_token) {
+    if (blockIdx.x != 0u || threadIdx.x != 0u) return;
+    state->prev_token = first_prev_token;
+    state->status = 1u;
+    state->active = 1u;
+    state->proposal_len = 0u;
+    state->confidence_len = 0u;
+    for (uint32_t i = 0; i < DS4_GPU_DSPARK_MAX_DRAFTS; i++) {
+        state->best_key[i] = 0ull;
+        state->tokens[i] = -1;
+        state->confidence_logits[i] = 0.0f;
+    }
+}
+
+__device__ __forceinline__ static float dspark_q8_value(
+        const unsigned char *row,
+        uint32_t index) {
+    const unsigned char *blk = row + (uint64_t)(index >> 5u) * 34u;
+    const float d = __half2float(*(const __half *)blk);
+    return d * (float)((const int8_t *)(blk + 2))[index & 31u];
+}
+
+/* The confidence projection has one Q8_0 output row.  A single thread keeps
+ * activation quantization and block accumulation in a stable, explicit
+ * order; this tail is tiny beside the vocab-wide Markov scan and avoids
+ * materializing or reading back the [hidden || w1(prev)] feature vector. */
+__global__ static void dspark_device_confidence_q8_kernel(
+        cuda_dspark_device_proposal_state *state,
+        const float *hidden_rows,
+        const unsigned char *w1,
+        const unsigned char *confidence,
+        uint32_t rank_blocks,
+        uint32_t hidden_dim,
+        uint32_t draft,
+        float confidence_threshold,
+        int reuse_confidence0,
+        float confidence0) {
+    if (blockIdx.x != 0u || threadIdx.x != 0u || state->status == 0u) return;
+    if (state->active == 0u) return;
+    float acc = 0.0f;
+    if (draft == 0u && reuse_confidence0) {
+        acc = confidence0;
+    } else {
+        const uint32_t hidden_blocks = hidden_dim / 32u;
+        const uint32_t feature_blocks = hidden_blocks + rank_blocks;
+        const uint64_t w1_row_bytes = (uint64_t)rank_blocks * 34u;
+        const unsigned char *w1_row =
+            w1 + (uint64_t)state->prev_token * w1_row_bytes;
+        const float *hidden = hidden_rows + (uint64_t)draft * hidden_dim;
+
+        for (uint32_t b = 0; b < feature_blocks; b++) {
+            float values[32];
+            float amax = 0.0f;
+            for (uint32_t k = 0; k < 32u; k++) {
+                const uint32_t feature = b * 32u + k;
+                const float v = feature < hidden_dim
+                    ? hidden[feature]
+                    : dspark_q8_value(w1_row, feature - hidden_dim);
+                values[k] = v;
+                const float av = fabsf(v);
+                if (av > amax) amax = av;
+            }
+            const float xscale = amax / 127.0f;
+            const float inv = xscale != 0.0f ? 1.0f / xscale : 0.0f;
+            const unsigned char *wblk = confidence + (uint64_t)b * 34u;
+            const float wscale = __half2float(*(const __half *)wblk);
+            const int8_t *wq = (const int8_t *)(wblk + 2);
+            int isum = 0;
+            for (uint32_t k = 0; k < 32u; k++) {
+                int q = __float2int_rn(values[k] * inv);
+                if (q > 127) q = 127;
+                if (q < -128) q = -128;
+                isum += (int)wq[k] * q;
+            }
+            /* Explicit rounded operations prevent contraction from silently
+             * changing the documented block-by-block accumulation order. */
+            const float scaled = __fmul_rn(__fmul_rn(wscale, xscale),
+                                           (float)isum);
+            acc = __fadd_rn(acc, scaled);
+        }
+    }
+    state->confidence_logits[draft] = acc;
+    state->confidence_len = draft + 1u;
+    const float e = expf(acc >= 0.0f ? -acc : acc);
+    const float confidence_value = acc >= 0.0f
+        ? 1.0f / (1.0f + e)
+        : e / (1.0f + e);
+    if (confidence_value < confidence_threshold) state->active = 0u;
+}
+
+__global__ static void dspark_device_proposal_advance_kernel(
+        cuda_dspark_device_proposal_state *state,
+        uint32_t vocab,
+        uint32_t draft) {
+    if (blockIdx.x != 0u || threadIdx.x != 0u || state->status == 0u ||
+        state->active == 0u) return;
+    const unsigned long long key = state->best_key[draft];
+    const uint32_t token = ~(uint32_t)(key & 0xffffffffu);
+    if (key == 0ull || token >= vocab) {
+        state->status = 0u;
+        return;
+    }
+    state->tokens[draft] = (int32_t)token;
+    state->prev_token = token;
+    state->proposal_len = draft + 1u;
+}
+
+__global__ static void dspark_device_proposal_export_kernel(
+        ds4_gpu_dspark_device_proposal *out,
+        const cuda_dspark_device_proposal_state *state) {
+    if (blockIdx.x != 0u || threadIdx.x != 0u) return;
+    for (uint32_t i = 0; i < DS4_GPU_DSPARK_MAX_DRAFTS; i++) {
+        out->tokens[i] = state->tokens[i];
+        out->confidence_logits[i] = state->confidence_logits[i];
+    }
+    out->proposal_len = state->proposal_len;
+    out->confidence_len = state->confidence_len;
+    out->status = state->status;
+    out->reserved = 0u;
+}
+
 __global__ static void indexer_top1_kernel(
         uint32_t *selected,
         const float *scores,
@@ -14342,13 +20532,171 @@ extern "C" int ds4_gpu_dspark_markov_argmax_tensor(
         dspark_markov_argmax_kernel<<<128, 256>>>(
                 (unsigned long long *)out_idx->ptr,
                 (const float *)logits_row->ptr,
-                w1_row, w2, vocab, rank_blocks);
+                w1_row, w2, vocab, rank_blocks, NULL, NULL);
         rc = cuda_ok(cudaGetLastError(), "dspark markov argmax launch");
     }
     if (logical_tier != dev_save) (void)cudaSetDevice(dev_save);
     return rc;
 }
 
+static int dspark_device_proposer_env_enabled(void) {
+    const char *enable = getenv("DS4_CUDA_DSPARK_DEVICE_PROPOSER");
+    if (!enable || !enable[0] || strcmp(enable, "0") == 0 ||
+        strcmp(enable, "off") == 0 || strcmp(enable, "false") == 0) {
+        return 0;
+    }
+    return getenv("DS4_CUDA_DSPARK_NO_DEVICE_PROPOSER") == NULL;
+}
+
+extern "C" int ds4_gpu_dspark_markov_confidence_q8_tensor(
+        ds4_gpu_tensor       *out_result,
+        const ds4_gpu_tensor *logits_rows,
+        const ds4_gpu_tensor *hidden_rows,
+        const void           *model_map,
+        uint64_t              model_size,
+        uint64_t              w1_offset,
+        uint64_t              w2_offset,
+        uint64_t              confidence_offset,
+        uint32_t              first_prev_token,
+        uint32_t              vocab,
+        uint32_t              rank,
+        uint32_t              hidden_dim,
+        uint32_t              n_drafts,
+        float                 confidence_threshold,
+        int                   reuse_confidence0,
+        float                 confidence0) {
+    if (!dspark_device_proposer_env_enabled() ||
+        !out_result || !logits_rows || !hidden_rows || !model_map ||
+        vocab == 0u || first_prev_token >= vocab ||
+        rank == 0u || rank > 256u || (rank & 31u) != 0u ||
+        hidden_dim == 0u || (hidden_dim & 31u) != 0u ||
+        n_drafts == 0u || n_drafts > DS4_GPU_DSPARK_MAX_DRAFTS ||
+        !(confidence_threshold > 0.0f && confidence_threshold <= 1.0f) ||
+        g_decode_graph_capturing ||
+        out_result->bytes < DS4_GPU_DSPARK_DEVICE_PROPOSAL_BYTES) {
+        return 0;
+    }
+    if ((uint64_t)n_drafts > UINT64_MAX / vocab / sizeof(float) ||
+        logits_rows->bytes <
+            (uint64_t)n_drafts * vocab * sizeof(float) ||
+        (uint64_t)n_drafts > UINT64_MAX / hidden_dim / sizeof(float) ||
+        hidden_rows->bytes <
+            (uint64_t)n_drafts * hidden_dim * sizeof(float)) {
+        return 0;
+    }
+
+    const int logical_tier = ds4_tensor_device_idx(logits_rows);
+    if (logical_tier < 0 || logical_tier >= g_n_gpus ||
+        ds4_tensor_device_idx(hidden_rows) != logical_tier ||
+        ds4_tensor_device_idx(out_result) != logical_tier) {
+        return 0;
+    }
+    const uint32_t rank_blocks = rank / 32u;
+    const uint64_t markov_row_bytes = (uint64_t)rank_blocks * 34u;
+    const uint64_t markov_bytes = (uint64_t)vocab * markov_row_bytes;
+    const uint64_t feature_dim = (uint64_t)hidden_dim + rank;
+    if (feature_dim > UINT32_MAX || (feature_dim & 31u) != 0u) return 0;
+    const uint64_t confidence_bytes = (feature_dim / 32u) * 34u;
+    if (w1_offset > model_size || markov_bytes > model_size - w1_offset ||
+        w2_offset > model_size || markov_bytes > model_size - w2_offset ||
+        confidence_offset > model_size ||
+        confidence_bytes > model_size - confidence_offset) {
+        return 0;
+    }
+
+    int saved_device = -1;
+    if (cudaGetDevice(&saved_device) != cudaSuccess) return 0;
+    const int target_device = g_gpu[logical_tier].device_id;
+    if (target_device != saved_device &&
+        cudaSetDevice(target_device) != cudaSuccess) {
+        return 0;
+    }
+
+    const unsigned char *w1 =
+        (const unsigned char *)cuda_resolve_weight_ptr(
+            model_map, w1_offset, markov_bytes, logical_tier,
+            "dspark device proposer w1");
+    const unsigned char *w2 =
+        (const unsigned char *)cuda_resolve_weight_ptr(
+            model_map, w2_offset, markov_bytes, logical_tier,
+            "dspark device proposer w2");
+    const unsigned char *confidence =
+        (const unsigned char *)cuda_resolve_weight_ptr(
+            model_map, confidence_offset, confidence_bytes, logical_tier,
+            "dspark device proposer confidence");
+    if (!w1 || !w2 || !confidence) {
+        if (target_device != saved_device) (void)cudaSetDevice(saved_device);
+        return 0;
+    }
+
+    /* Session-owned storage prevents concurrent proposer calls from aliasing
+     * the backend's global temporary slab.  The compact public result occupies
+     * the first 64 bytes; the private state starts at an 8-byte-aligned offset
+     * immediately after it and lives through the caller's synchronous read. */
+    cuda_dspark_device_proposal_state *state =
+        (cuda_dspark_device_proposal_state *)(
+            (unsigned char *)out_result->ptr +
+            sizeof(ds4_gpu_dspark_device_proposal));
+
+    int rc = 0;
+    const cudaStream_t stream = cuda_decode_stream();
+    dspark_device_proposal_init_kernel<<<1, 1, 0, stream>>>(
+        state, first_prev_token);
+    rc = cuda_ok(cudaGetLastError(), "dspark device proposer init launch");
+    for (uint32_t draft = 0; rc && draft < n_drafts; draft++) {
+        dspark_device_confidence_q8_kernel<<<1, 1, 0, stream>>>(
+            state,
+            (const float *)hidden_rows->ptr,
+            w1,
+            confidence,
+            rank_blocks,
+            hidden_dim,
+            draft,
+            confidence_threshold,
+            reuse_confidence0,
+            confidence0);
+        rc = cuda_ok(cudaGetLastError(),
+                     "dspark device confidence launch");
+        if (!rc) break;
+        dspark_markov_argmax_kernel<<<128, 256, 0, stream>>>(
+            &state->best_key[draft],
+            (const float *)logits_rows->ptr + (uint64_t)draft * vocab,
+            w1,
+            w2,
+            vocab,
+            rank_blocks,
+            &state->prev_token,
+            &state->active);
+        rc = cuda_ok(cudaGetLastError(),
+                     "dspark device markov launch");
+        if (!rc) break;
+        dspark_device_proposal_advance_kernel<<<1, 1, 0, stream>>>(
+            state, vocab, draft);
+        rc = cuda_ok(cudaGetLastError(),
+                     "dspark device proposer advance launch");
+    }
+    if (rc) {
+        dspark_device_proposal_export_kernel<<<1, 1, 0, stream>>>(
+            (ds4_gpu_dspark_device_proposal *)out_result->ptr,
+            state);
+        rc = cuda_ok(cudaGetLastError(),
+                     "dspark device proposer export launch");
+    }
+    if (target_device != saved_device) (void)cudaSetDevice(saved_device);
+
+    if (rc) {
+        static int logged = 0;
+        if (!logged) {
+            logged = 1;
+            fprintf(stderr,
+                    "ds4: CUDA DSpark device proposer enabled "
+                    "(Q8 confidence/Markov, max_drafts=%u)\n",
+                    DS4_GPU_DSPARK_MAX_DRAFTS);
+        }
+    }
+    return rc;
+}
+
 extern "C" int ds4_gpu_indexer_topk_tensor(
         ds4_gpu_tensor       *selected,
         const ds4_gpu_tensor *scores,
@@ -15356,7 +21704,8 @@ extern "C" int ds4_gpu_matmul_q4_K_pair_decode_tensor(
     return ds4_mmq_q4_K_dense_pair_vec(
         w0, w1, (const float *)x->ptr,
         (float *)out0->ptr, (float *)out1->ptr,
-        (int)out_dim, (int)in_dim, cuda_decode_stream()) == 0;
+        (int)out_dim, (int)out_dim, /*N=*/1, (int)in_dim,
+        cuda_decode_stream()) == 0;
 }
 
 extern "C" int ds4_gpu_matmul_q8_0_pair_tensor(
@@ -15398,6 +21747,32 @@ extern "C" int ds4_gpu_matmul_q8_0_pair_tensor(
     const char *w0 = cuda_resolve_weight_ptr(model_map, weight0_offset, weight0_bytes, logical_tier, "q8_0_pair0");
     const char *w1 = cuda_resolve_weight_ptr(model_map, weight1_offset, weight1_bytes, logical_tier, "q8_0_pair1");
     if (!w0 || !w1) return 0;
+    const bool fused_aligned_candidate =
+        g_n_gpus == 1 &&
+        logical_tier >= 0 && logical_tier < DS4_MAX_GPUS &&
+        g_cuda_is_gb10[logical_tier] &&
+        getenv("DS4_CUDA_NO_Q8_FUSED_ALIGNED") == NULL &&
+        n_tok == 1u && in_dim <= INT_MAX &&
+        out0_dim <= INT_MAX && out1_dim <= INT_MAX &&
+        (in_dim % 1024u) == 0u &&
+        (out0_dim % 128u) == 0u && (out1_dim % 128u) == 0u &&
+        cuda_aligned_q8_enabled() && cuda_q8_use_dp4a();
+    const uint64_t aligned0_bytes = fused_aligned_candidate
+        ? ds4_mmq_q8_0_aligned_bytes((int)out0_dim, (int)in_dim) : 0u;
+    const uint64_t aligned1_bytes = fused_aligned_candidate
+        ? ds4_mmq_q8_0_aligned_bytes((int)out1_dim, (int)in_dim) : 0u;
+    const char *aligned0 = aligned0_bytes
+        ? cuda_derived_weight_ptr(
+              model_map, weight0_offset, weight0_bytes,
+              CUDA_DERIVED_Q8_0_ALIGNED_DENSE,
+              in_dim, out0_dim, 1u, aligned0_bytes)
+        : NULL;
+    const char *aligned1 = aligned1_bytes
+        ? cuda_derived_weight_ptr(
+              model_map, weight1_offset, weight1_bytes,
+              CUDA_DERIVED_Q8_0_ALIGNED_DENSE,
+              in_dim, out1_dim, 1u, aligned1_bytes)
+        : NULL;
 
     const bool force_decode_warp =
         n_tok == 2u && g_glm_mtp_verify_mode;
@@ -15593,21 +21968,57 @@ extern "C" int ds4_gpu_matmul_q8_0_pair_tensor(
         return cuda_ok(cudaGetLastError(), "matmul_q8_0 pair1 batch launch");
     }
     const uint64_t max_out = out0_dim > out1_dim ? out0_dim : out1_dim;
-    matmul_q8_0_pair_preq_warp8_kernel<<<((unsigned)max_out + 7u) / 8u, 256, 0, cuda_decode_stream()>>>(
-            (float *)out0->ptr,
-            (float *)out1->ptr,
-            reinterpret_cast<const unsigned char *>(w0),
-            reinterpret_cast<const unsigned char *>(w1),
-            xq,
-            xscale,
-            in_dim,
-            out0_dim,
-            out1_dim,
-            blocks,
-            use_dp4a);
+    const dim3 decode_grid(((unsigned)max_out + 7u) / 8u, 1u, 1u);
+    if (aligned0 && aligned1) {
+        const uint64_t nblk0 = out0_dim * blocks;
+        const uint64_t nblk1 = out1_dim * blocks;
+        const uint64_t dq0_bytes =
+            (nblk0 * sizeof(__half) + 63u) & ~63ull;
+        const uint64_t dq1_bytes =
+            (nblk1 * sizeof(__half) + 63u) & ~63ull;
+        matmul_q8_0_pair_aligned_preq_warp8_kernel<<<
+            decode_grid, 256, 0, cuda_decode_stream()>>>(
+                (float *)out0->ptr,
+                (float *)out1->ptr,
+                (const int4 *)(aligned0 + dq0_bytes),
+                (const __half *)aligned0,
+                (const int4 *)(aligned1 + dq1_bytes),
+                (const __half *)aligned1,
+                xq, xscale, out0_dim, out1_dim, blocks);
+    } else {
+        matmul_q8_0_pair_preq_warp8_kernel<<<
+            decode_grid, 256, 0, cuda_decode_stream()>>>(
+                (float *)out0->ptr,
+                (float *)out1->ptr,
+                reinterpret_cast<const unsigned char *>(w0),
+                reinterpret_cast<const unsigned char *>(w1),
+                xq, xscale, in_dim, out0_dim, out1_dim, blocks, use_dp4a);
+    }
     return cuda_ok(cudaGetLastError(), "matmul_q8_0 pair warp launch");
 }
 
+static int cuda_matmul_q4_K_pair_tensor_impl(
+        ds4_gpu_tensor *out0, ds4_gpu_tensor *out1,
+        const void *model_map, uint64_t model_size,
+        uint64_t weight0_offset, uint64_t weight1_offset,
+        uint64_t in_dim, uint64_t out0_dim, uint64_t out1_dim,
+        const ds4_gpu_tensor *x, uint64_t n_tok);
+
+extern "C" int ds4_gpu_matmul_q4_K_pair_tensor(
+        ds4_gpu_tensor *out0, ds4_gpu_tensor *out1,
+        const void *model_map, uint64_t model_size,
+        uint64_t weight0_offset, uint64_t weight1_offset,
+        uint64_t in_dim, uint64_t out0_dim, uint64_t out1_dim,
+        const ds4_gpu_tensor *x, uint64_t n_tok) {
+    /* Cross-CUDA pair dispatch predates the GB10 specialization.  Preserve it
+     * verbatim outside GB10; this switch remains its established rollback. */
+    if (getenv("DS4_CUDA_DISABLE_Q4_DENSE_PAIR") != NULL) return 0;
+    return cuda_matmul_q4_K_pair_tensor_impl(
+        out0, out1, model_map, model_size,
+        weight0_offset, weight1_offset,
+        in_dim, out0_dim, out1_dim, x, n_tok);
+}
+
 extern "C" int ds4_gpu_matmul_q8_0_decode_rows_exact_tensor(
         ds4_gpu_tensor       *out,
         const void             *model_map,
@@ -15635,6 +22046,31 @@ extern "C" int ds4_gpu_matmul_q8_0_decode_rows_exact_tensor(
         ds4_tensor_device_idx(x) != logical_tier) {
         return 0;
     }
+    /* Match the canonical one-row decode path when an aligned Q8 artifact is
+     * present.  Its NC kernel reads each weight row once while preserving the
+     * same per-column block walk and warp reduction, so exact-N can batch the
+     * vocab head without silently switching to the raw warp8 arithmetic. */
+    const uint64_t aligned_bytes =
+        (in_dim % 1024u) == 0u && (out_dim % 128u) == 0u
+            ? ds4_mmq_q8_0_aligned_bytes((int)out_dim, (int)in_dim)
+            : 0u;
+    const char *aligned = aligned_bytes && cuda_aligned_q8_enabled()
+        ? cuda_derived_weight_ptr(
+              model_map, weight_offset, weight_bytes,
+              CUDA_DERIVED_Q8_0_ALIGNED_DENSE,
+              in_dim, out_dim, 1u, aligned_bytes)
+        : NULL;
+    if (aligned && n_rows <= 8u) {
+        const int aligned_rc = ds4_mmq_q8_0_aligned_dense_vec(
+                aligned,
+                (const float *)x->ptr,
+                (float *)out->ptr,
+                (int)out_dim,
+                (int)n_rows,
+                (int)in_dim,
+                cuda_decode_stream());
+        if (aligned_rc == 0) return 1;
+    }
     const char *wptr = cuda_resolve_weight_ptr(
             model_map, weight_offset, weight_bytes, logical_tier,
             "q8_0 decode rows exact");
@@ -15650,14 +22086,14 @@ extern "C" int ds4_gpu_matmul_q8_0_decode_rows_exact_tensor(
     int8_t *xq = (int8_t *)tmp;
     float *xscale = (float *)((char *)tmp + scale_offset);
     dim3 qgrid((unsigned)blocks, n_rows, 1u);
-    quantize_q8_0_f32_kernel<<<qgrid, 32>>>(
+    quantize_q8_0_f32_kernel<<<qgrid, 32, 0, cuda_decode_stream()>>>(
             xq, xscale, (const float *)x->ptr, in_dim, blocks);
     if (!cuda_ok(cudaGetLastError(),
                  "q8_0 decode rows exact quantize launch")) {
         return 0;
     }
     dim3 grid(((unsigned)out_dim + 7u) / 8u, n_rows, 1u);
-    matmul_q8_0_preq_warp8_kernel<<<grid, 256>>>(
+    matmul_q8_0_preq_warp8_kernel<<<grid, 256, 0, cuda_decode_stream()>>>(
             (float *)out->ptr,
             reinterpret_cast<const unsigned char *>(wptr),
             xq, xscale, in_dim, out_dim, blocks, cuda_q8_use_dp4a());
@@ -15739,6 +22175,95 @@ extern "C" int ds4_gpu_matmul_q8_0_pair_decode_rows_exact_tensor(
                    "q8_0 pair decode rows exact warp launch");
 }
 
+typedef struct {
+    const char *raw;
+    const char *aligned;
+    int8_t     *xq;
+    float      *xscale;
+    uint64_t    blocks;
+    int         use_dp4a;
+} cuda_q8_hc_matmul_prepared;
+
+/* One preparation contract feeds both sides of the HC A/B: identical weight
+ * resolution, aligned-artifact selection, scratch layout and Q8_0 activation
+ * quantization.  Keeping this centralized prevents the split experiment from
+ * silently inheriting MMQ's half-scale Q8_1 numerics. */
+static int cuda_q8_hc_matmul_prepare(
+        ds4_gpu_tensor       *block_out,
+        const void             *model_map,
+        uint64_t                model_size,
+        uint64_t                weight_offset,
+        uint64_t                in_dim,
+        uint64_t                out_dim,
+        const ds4_gpu_tensor *x,
+        int                     logical_tier,
+        const char             *label,
+        cuda_q8_hc_matmul_prepared *prepared) {
+    if (!block_out || !model_map || !x || !prepared ||
+        in_dim == 0u || out_dim == 0u) {
+        return 0;
+    }
+    const uint64_t blocks = (in_dim + 31u) / 32u;
+    if (blocks == 0u || blocks > UINT64_MAX / 34u ||
+        out_dim > UINT64_MAX / (blocks * 34u) ||
+        weight_offset > model_size) {
+        return 0;
+    }
+    const uint64_t weight_bytes = out_dim * blocks * 34u;
+    if (weight_bytes > model_size - weight_offset ||
+        in_dim > UINT64_MAX / sizeof(float) ||
+        out_dim > UINT64_MAX / sizeof(float) ||
+        x->bytes < in_dim * sizeof(float) ||
+        block_out->bytes < out_dim * sizeof(float)) {
+        return 0;
+    }
+    const char *raw = cuda_resolve_weight_ptr(
+        model_map, weight_offset, weight_bytes, logical_tier,
+        label ? label : "q8_0_hc_expand");
+    if (!raw) return 0;
+    const uint64_t aligned_bytes =
+        in_dim <= INT_MAX && out_dim <= INT_MAX &&
+        (in_dim % 1024u) == 0u && (out_dim % 128u) == 0u
+            ? ds4_mmq_q8_0_aligned_bytes((int)out_dim, (int)in_dim)
+            : 0u;
+    const int use_dp4a = cuda_q8_use_dp4a();
+    const char *aligned =
+        g_n_gpus == 1 &&
+        logical_tier >= 0 && logical_tier < DS4_MAX_GPUS &&
+        g_cuda_is_gb10[logical_tier] &&
+        getenv("DS4_CUDA_NO_Q8_FUSED_ALIGNED") == NULL &&
+        aligned_bytes != 0u && cuda_aligned_q8_enabled() && use_dp4a
+            ? cuda_derived_weight_ptr(
+                  model_map, weight_offset, weight_bytes,
+                  CUDA_DERIVED_Q8_0_ALIGNED_DENSE,
+                  in_dim, out_dim, 1u, aligned_bytes)
+            : NULL;
+
+    const uint64_t xq_bytes = blocks * 32u;
+    const uint64_t scale_offset = (xq_bytes + 15u) & ~15ull;
+    if (blocks > (UINT64_MAX - scale_offset) / sizeof(float)) return 0;
+    const uint64_t tmp_bytes = scale_offset + blocks * sizeof(float);
+    void *tmp = cuda_tmp_alloc_on(
+        logical_tier, tmp_bytes, "q8_0 hc expand prequant");
+    if (!tmp) return 0;
+    int8_t *xq = (int8_t *)tmp;
+    float *xscale = (float *)((char *)tmp + scale_offset);
+    quantize_q8_0_f32_kernel<<<
+        (unsigned)blocks, 32, 0, cuda_decode_stream()>>>(
+            xq, xscale, (const float *)x->ptr, in_dim, blocks);
+    if (!cuda_ok(cudaGetLastError(),
+                 "matmul_q8_0_hc_expand quantize launch")) {
+        return 0;
+    }
+    prepared->raw = raw;
+    prepared->aligned = aligned;
+    prepared->xq = xq;
+    prepared->xscale = xscale;
+    prepared->blocks = blocks;
+    prepared->use_dp4a = use_dp4a;
+    return 1;
+}
+
 static int cuda_matmul_q8_0_hc_expand_tensor_labeled(
         ds4_gpu_tensor       *out_hc,
         ds4_gpu_tensor       *block_out,
@@ -15764,15 +22289,9 @@ static int cuda_matmul_q8_0_hc_expand_tensor_labeled(
         out_dim != (uint64_t)n_embd) {
         return 0;
     }
-    const uint64_t blocks = (in_dim + 31) / 32;
-    if (weight_offset > model_size || out_dim > UINT64_MAX / (blocks * 34)) return 0;
-    const uint64_t weight_bytes = out_dim * blocks * 34;
     const uint64_t hc_bytes = (uint64_t)n_hc * n_embd * sizeof(float);
     const uint64_t split_bytes = (uint64_t)(2u * n_hc + n_hc * n_hc) * sizeof(float);
-    if (weight_bytes > model_size - weight_offset ||
-        x->bytes < in_dim * sizeof(float) ||
-        block_out->bytes < out_dim * sizeof(float) ||
-        residual_hc->bytes < hc_bytes ||
+    if (residual_hc->bytes < hc_bytes ||
         split->bytes < split_bytes ||
         out_hc->bytes < hc_bytes ||
         (block_add && block_add->bytes < out_dim * sizeof(float)) ||
@@ -15782,24 +22301,45 @@ static int cuda_matmul_q8_0_hc_expand_tensor_labeled(
           owned_expert_split == 0u ||
           owned_home_slots->bytes < 6u * out_dim * sizeof(float) ||
           owned_peer_packed->bytes < 4u * out_dim * sizeof(float) ||
-          owned_selected->bytes < 6u * sizeof(int32_t)))) {
+         owned_selected->bytes < 6u * sizeof(int32_t)))) {
         return 0;
     }
-    const int logical_tier = ds4_tensor_device_idx(out_hc);
-    const char *wptr = cuda_resolve_weight_ptr(model_map, weight_offset, weight_bytes, logical_tier, label ? label : "q8_0_hc_expand");
-    if (!wptr) return 0;
-
-    const uint64_t xq_bytes = blocks * 32u;
-    const uint64_t scale_offset = (xq_bytes + 15u) & ~15ull;
-    const uint64_t tmp_bytes = scale_offset + blocks * sizeof(float);
-    void *tmp = cuda_tmp_alloc_on(logical_tier, tmp_bytes, "q8_0 hc expand prequant");
-    if (!tmp) return 0;
-    int8_t *xq = (int8_t *)tmp;
-    float *xscale = (float *)((char *)tmp + scale_offset);
-    const int use_dp4a = cuda_q8_use_dp4a();
-    quantize_q8_0_f32_kernel<<<(unsigned)blocks, 32, 0, cuda_decode_stream()>>>(xq, xscale, (const float *)x->ptr, in_dim, blocks);
-    if (!cuda_ok(cudaGetLastError(), "matmul_q8_0_hc_expand quantize launch")) return 0;
-    matmul_q8_0_hc_expand_preq_warp8_kernel<<<((unsigned)out_dim + 7u) / 8u, 256, 0, cuda_decode_stream()>>>(
+    cuda_q8_hc_matmul_prepared prepared = {};
+    if (!cuda_q8_hc_matmul_prepare(
+            block_out, model_map, model_size, weight_offset,
+            in_dim, out_dim, x, ds4_tensor_device_idx(out_hc),
+            label, &prepared)) {
+        return 0;
+    }
+    const char *wptr = prepared.raw;
+    const char *aligned = prepared.aligned;
+    int8_t *xq = prepared.xq;
+    float *xscale = prepared.xscale;
+    const uint64_t blocks = prepared.blocks;
+    const int use_dp4a = prepared.use_dp4a;
+    const dim3 grid(((unsigned)out_dim + 7u) / 8u, 1u, 1u);
+    if (aligned) {
+        const uint64_t nblk = out_dim * blocks;
+        const uint64_t dq_bytes = (nblk * sizeof(__half) + 63u) & ~63ull;
+        matmul_q8_0_hc_expand_aligned_preq_warp8_kernel<<<grid, 256, 0, cuda_decode_stream()>>>(
+            (float *)out_hc->ptr,
+            (float *)block_out->ptr,
+            block_add ? (const float *)block_add->ptr : (const float *)block_out->ptr,
+            block_add2 ? (const float *)block_add2->ptr : (const float *)block_out->ptr,
+            owned_home_slots ? (const float *)owned_home_slots->ptr : NULL,
+            owned_peer_packed ? (const float *)owned_peer_packed->ptr : NULL,
+            owned_selected ? (const int32_t *)owned_selected->ptr : NULL,
+            (const float *)residual_hc->ptr,
+            (const float *)split->ptr,
+            (const int4 *)(aligned + dq_bytes),
+            (const __half *)aligned,
+            xq, xscale, out_dim, n_embd, n_hc, blocks,
+            block_add ? 1 : 0,
+            block_add2 ? 1 : 0,
+            owned_home_slots ? 1 : 0,
+            owned_expert_split);
+    } else {
+        matmul_q8_0_hc_expand_preq_warp8_kernel<<<grid, 256, 0, cuda_decode_stream()>>>(
             (float *)out_hc->ptr,
             (float *)block_out->ptr,
             block_add ? (const float *)block_add->ptr : (const float *)block_out->ptr,
@@ -15810,21 +22350,56 @@ static int cuda_matmul_q8_0_hc_expand_tensor_labeled(
             (const float *)residual_hc->ptr,
             (const float *)split->ptr,
             reinterpret_cast<const unsigned char *>(wptr),
-            xq,
-            xscale,
-            in_dim,
-            out_dim,
-            n_embd,
-            n_hc,
-            blocks,
+            xq, xscale, in_dim, out_dim, n_embd, n_hc, blocks,
             block_add ? 1 : 0,
             block_add2 ? 1 : 0,
             owned_home_slots ? 1 : 0,
             owned_expert_split,
             use_dp4a);
+    }
     return cuda_ok(cudaGetLastError(), "matmul_q8_0_hc_expand launch");
 }
 
+static int cuda_matmul_q8_0_hc_split_matmul_tensor_labeled(
+        ds4_gpu_tensor       *block_out,
+        const void             *model_map,
+        uint64_t                model_size,
+        uint64_t                weight_offset,
+        uint64_t                in_dim,
+        uint64_t                out_dim,
+        const ds4_gpu_tensor *x,
+        int                     logical_tier,
+        const char             *label) {
+    cuda_q8_hc_matmul_prepared prepared = {};
+    if (!cuda_q8_hc_matmul_prepare(
+            block_out, model_map, model_size, weight_offset,
+            in_dim, out_dim, x, logical_tier, label, &prepared)) {
+        return 0;
+    }
+    const cudaStream_t stream = cuda_decode_stream();
+    if (prepared.aligned) {
+        const uint64_t nblk = out_dim * prepared.blocks;
+        const uint64_t dq_bytes =
+            (nblk * sizeof(__half) + 63u) & ~63ull;
+        matmul_q8_0_aligned_preq_warp8_kernel<<<
+            ((unsigned)out_dim + 7u) / 8u, 256, 0, stream>>>(
+                (float *)block_out->ptr,
+                (const int4 *)(prepared.aligned + dq_bytes),
+                (const __half *)prepared.aligned,
+                prepared.xq, prepared.xscale,
+                out_dim, prepared.blocks);
+    } else {
+        matmul_q8_0_preq_warp8_kernel<<<
+            ((unsigned)out_dim + 7u) / 8u, 256, 0, stream>>>(
+                (float *)block_out->ptr,
+                reinterpret_cast<const unsigned char *>(prepared.raw),
+                prepared.xq, prepared.xscale,
+                in_dim, out_dim, prepared.blocks, prepared.use_dp4a);
+    }
+    return cuda_ok(cudaGetLastError(),
+                   "matmul_q8_0_hc_split matmul launch");
+}
+
 extern "C" int ds4_gpu_matmul_f16_tensor(ds4_gpu_tensor *out, const void *model_map, uint64_t model_size, uint64_t weight_offset, uint64_t in_dim, uint64_t out_dim, const ds4_gpu_tensor *x, uint64_t n_tok) {
     if (!out || !x || !model_map) return 0;
     if (weight_offset > model_size || out_dim > UINT64_MAX / in_dim) return 0;
@@ -15932,6 +22507,78 @@ extern "C" int ds4_gpu_matmul_f16_tensor(ds4_gpu_tensor *out, const void *model_
     return cuda_ok(cudaGetLastError(), "matmul_f16 launch");
 }
 
+/* Return the activation mode that exactly matches the currently selected
+ * standalone one-token path: 1 keeps normalized activations in F32, 2 rounds
+ * them through F16.  cuBLAS and alternate reduction modes deliberately reject
+ * the fusion, so the graph caller retains its established fallback. */
+static int cuda_hc_rms_norm_mix_f16_mode(void) {
+    const char *enable = getenv("DS4_CUDA_ENABLE_HC_NORM_MIX_FUSE");
+    if (!enable || !enable[0] ||
+        (enable[0] == '0' && enable[1] == '\0') ||
+        getenv("DS4_CUDA_DISABLE_HC_NORM_MIX_FUSE") != NULL ||
+        getenv("DS4_CUDA_SERIAL_F16_MATMUL") != NULL ||
+        getenv("DS4_CUDA_NO_ORDERED_F16_MATMUL") != NULL) {
+        return 0;
+    }
+
+    const int small_out_one_token =
+        !g_quality_mode &&
+        getenv("DS4_CUDA_F16_SMALL_OUT") != NULL &&
+        getenv("DS4_CUDA_NO_F16_SMALL_OUT") == NULL;
+    if (small_out_one_token) return 2;
+
+    const int cublas_one_token =
+        g_cublas_ready &&
+        getenv("DS4_CUDA_NO_F16_CUBLAS_ONE") == NULL &&
+        (!g_quality_mode || getenv("DS4_CUDA_F16_CUBLAS_ONE") != NULL);
+    if (cublas_one_token) return 0;
+    return 1;
+}
+
+extern "C" int ds4_gpu_hc_rms_norm_mix_f16_available(void) {
+    return cuda_hc_rms_norm_mix_f16_mode() != 0;
+}
+
+extern "C" int ds4_gpu_hc_rms_norm_mix_f16_tensor(
+        ds4_gpu_tensor *out,
+        const ds4_gpu_tensor *x,
+        const void *model_map,
+        uint64_t model_size,
+        uint64_t weight_offset,
+        uint32_t n,
+        uint32_t out_dim,
+        float eps) {
+    const int mode = cuda_hc_rms_norm_mix_f16_mode();
+    if (mode == 0 || !out || !x || !model_map ||
+        n != 16384u || out_dim != 24u ||
+        weight_offset > model_size) {
+        return 0;
+    }
+
+    const uint64_t weight_bytes =
+        (uint64_t)n * out_dim * sizeof(uint16_t);
+    if (weight_bytes > model_size - weight_offset ||
+        x->bytes < (uint64_t)n * sizeof(float) ||
+        out->bytes < (uint64_t)out_dim * sizeof(float) ||
+        ds4_tensor_device_idx(x) != ds4_tensor_device_idx(out)) {
+        return 0;
+    }
+
+    const int logical_tier = ds4_tensor_device_idx(out);
+    const char *wptr = cuda_resolve_weight_ptr(
+        model_map, weight_offset, weight_bytes, logical_tier,
+        "hc rms-norm/f16-mix");
+    if (!wptr) return 0;
+
+    hc_rms_norm_mix_f16_kernel<<<12u, 256u, 0, cuda_decode_stream()>>>(
+        (float *)out->ptr,
+        (const float *)x->ptr,
+        (const __half *)wptr,
+        eps,
+        mode == 2 ? 1 : 0);
+    return cuda_ok(cudaGetLastError(), "hc rms-norm/f16-mix launch");
+}
+
 extern "C" int ds4_gpu_matmul_f16_rms_fold_tensor(
         ds4_gpu_tensor *out,
         const void *model_map,
@@ -16221,22 +22868,145 @@ extern "C" int ds4_gpu_matmul_f16_pair_compressor_store_tensor(
         const ds4_gpu_tensor *x,
         uint32_t ratio,
         uint32_t pos) {
-    (void)out_kv;
-    (void)out_score;
-    (void)state_kv;
-    (void)state_score;
-    (void)model_map;
-    (void)model_size;
-    (void)weight_kv_offset;
-    (void)weight_score_offset;
-    (void)ape_offset;
-    (void)ape_type;
-    (void)in_dim;
-    (void)width;
-    (void)x;
-    (void)ratio;
-    (void)pos;
-    return 0;
+    if (!g_cuda_f16_pair_compressor_store ||
+        getenv("DS4_CUDA_NO_F16_PAIR_MATMUL") != NULL ||
+        getenv("DS4_CUDA_SERIAL_F16_MATMUL") != NULL ||
+        getenv("DS4_CUDA_SERIAL_ROUTER") != NULL ||
+        getenv("DS4_CUDA_NO_ORDERED_F16_MATMUL") != NULL) {
+        return 0;
+    }
+    if (!out_kv || !out_score || !state_kv || !state_score || !x ||
+        !model_map || in_dim == 0u || width == 0u || ratio == 0u ||
+        (ape_type != 0u && ape_type != 1u)) {
+        return -1;
+    }
+    /* These are the three resident Flash/Pro compressor projection shapes
+     * validated on GB10. Other layouts retain the ordinary pair+store path. */
+    if (in_dim != 4096u ||
+        !((ratio == 4u && (width == 256u || width == 1024u)) ||
+          (ratio == 128u && width == 512u))) {
+        return 0;
+    }
+    if ((uint64_t)width > UINT64_MAX / in_dim ||
+        weight_kv_offset > model_size ||
+        weight_score_offset > model_size ||
+        ape_offset > model_size) {
+        return -1;
+    }
+    const uint64_t weight_elems = (uint64_t)width * in_dim;
+    if (weight_elems > UINT64_MAX / sizeof(uint16_t) ||
+        in_dim > UINT64_MAX / sizeof(float) ||
+        (uint64_t)width > UINT64_MAX / sizeof(float)) {
+        return -1;
+    }
+    const uint64_t weight_bytes = weight_elems * sizeof(uint16_t);
+    const uint64_t output_bytes = (uint64_t)width * sizeof(float);
+    const uint64_t input_bytes = in_dim * sizeof(float);
+    const uint64_t ape_elem_bytes = ape_type == 1u ? 2u : 4u;
+    if ((uint64_t)width > UINT64_MAX / ratio) return -1;
+    const uint64_t ape_elems = (uint64_t)width * ratio;
+    if (ape_elems > UINT64_MAX / ape_elem_bytes) return -1;
+    const uint64_t ape_bytes = ape_elems * ape_elem_bytes;
+    const uint64_t coff = ratio == 4u ? 2u : 1u;
+    const uint64_t state_rows = coff * ratio;
+    if (state_rows > UINT64_MAX / width) return -1;
+    const uint64_t state_elems = state_rows * width;
+    if (state_elems > UINT64_MAX / sizeof(float)) return -1;
+    const uint64_t state_bytes = state_elems * sizeof(float);
+    if (weight_bytes > model_size - weight_kv_offset ||
+        weight_bytes > model_size - weight_score_offset ||
+        ape_bytes > model_size - ape_offset ||
+        x->bytes < input_bytes ||
+        out_kv->bytes < output_bytes ||
+        out_score->bytes < output_bytes ||
+        state_kv->bytes < state_bytes ||
+        state_score->bytes < state_bytes) {
+        return -1;
+    }
+    const int logical_tier = ds4_tensor_device_idx(out_kv);
+    if (logical_tier < 0 || logical_tier >= DS4_MAX_GPUS) return -1;
+    if (!g_cuda_is_gb10[logical_tier]) return 0;
+    if (ds4_tensor_device_idx(out_score) != logical_tier ||
+        ds4_tensor_device_idx(state_kv) != logical_tier ||
+        ds4_tensor_device_idx(state_score) != logical_tier ||
+        ds4_tensor_device_idx(x) != logical_tier) {
+        return -1;
+    }
+    const int expected_device = logical_tier < g_n_gpus
+        ? g_gpu[logical_tier].device_id : -1;
+    int active_device = -1;
+    if (expected_device < 0 ||
+        cudaGetDevice(&active_device) != cudaSuccess ||
+        active_device != expected_device) {
+        (void)cudaGetLastError();
+        return -1;
+    }
+    const __half *w_kv = (const __half *)cuda_resolve_weight_ptr(
+        model_map, weight_kv_offset, weight_bytes, logical_tier,
+        "f16 compressor kv");
+    const __half *w_score = (const __half *)cuda_resolve_weight_ptr(
+        model_map, weight_score_offset, weight_bytes, logical_tier,
+        "f16 compressor score");
+    const void *ape = cuda_resolve_weight_ptr(
+        model_map, ape_offset, ape_bytes, logical_tier, "compressor ape");
+    if (!w_kv || !w_score || !ape) return -1;
+
+    const __half2 *w_pair_chunk32 = NULL;
+    if (g_n_gpus == 1 &&
+        getenv("DS4_CUDA_NO_F16_PAIR_COMPRESSOR_TRANSPOSE") == NULL) {
+        w_pair_chunk32 = cuda_f16_pair_chunk32_get(
+            model_map, weight_kv_offset, weight_score_offset,
+            w_kv, w_score, in_dim, width, logical_tier);
+    }
+    if (w_pair_chunk32) {
+        if (getenv("DS4_CUDA_NO_F16_PAIR_COMPRESSOR_TRANSPOSE_PREFETCH8") == NULL &&
+            (in_dim % 256u) == 0u) {
+            matmul_f16_pair_compressor_store_chunk32_prefetch8_kernel<<<width, 32, 0, cuda_decode_stream()>>>(
+                (float *)out_kv->ptr,
+                (float *)out_score->ptr,
+                (float *)state_kv->ptr,
+                (float *)state_score->ptr,
+                w_pair_chunk32,
+                (const float *)x->ptr,
+                ape,
+                ape_type,
+                in_dim,
+                width,
+                ratio,
+                pos);
+        } else {
+            matmul_f16_pair_compressor_store_chunk32_kernel<<<width, 32, 0, cuda_decode_stream()>>>(
+                (float *)out_kv->ptr,
+                (float *)out_score->ptr,
+                (float *)state_kv->ptr,
+                (float *)state_score->ptr,
+                w_pair_chunk32,
+                (const float *)x->ptr,
+                ape,
+                ape_type,
+                in_dim,
+                width,
+                ratio,
+                pos);
+        }
+    } else {
+        matmul_f16_pair_compressor_store_ordered_chunks_kernel<<<width, 32, 0, cuda_decode_stream()>>>(
+            (float *)out_kv->ptr,
+            (float *)out_score->ptr,
+            (float *)state_kv->ptr,
+            (float *)state_score->ptr,
+            w_kv,
+            w_score,
+            (const float *)x->ptr,
+            ape,
+            ape_type,
+            in_dim,
+            width,
+            ratio,
+            pos);
+    }
+    return cuda_ok(cudaGetLastError(),
+                   "f16 pair compressor/store launch") ? 1 : -1;
 }
 
 extern "C" int ds4_gpu_matmul_f32_tensor(ds4_gpu_tensor *out, const void *model_map, uint64_t model_size, uint64_t weight_offset, uint64_t in_dim, uint64_t out_dim, const ds4_gpu_tensor *x, uint64_t n_tok) {
@@ -16364,6 +23134,131 @@ __global__ static void attention_noncausal_raw_batch_heads_kernel(
     }
 }
 
+/* Experimental DSpark verifier attention for the native 512-wide head.  Eight
+ * heads share each staged raw row, while every warp maintains its own online
+ * softmax state.  Seeding (max,sum) with the sink gives the sink a zero value
+ * contribution without materializing scores or a sink row.  Keep this behind
+ * an explicit gate: the online recurrence is mathematically equivalent to the
+ * reference kernel, but its floating-point accumulation order is different. */
+template <uint32_t ROWS_PER_STAGE, uint32_t HEADS_PER_GROUP>
+__global__ static void __launch_bounds__(256, 2)
+attention_noncausal_raw_batch_heads_online_kernel(
+        float *heads,
+        const float *sinks,
+        const float *q,
+        const float *raw_kv,
+        uint32_t n_tokens,
+        uint32_t n_raw,
+        uint32_t raw_cap,
+        uint32_t raw_start,
+        uint32_t n_head,
+        uint32_t head_dim) {
+    const uint32_t tok = blockIdx.x;
+    const uint32_t head_group = blockIdx.y;
+    if (tok >= n_tokens || raw_cap == 0u || raw_start >= raw_cap ||
+        n_raw == 0u || n_raw > raw_cap || head_dim != 512u) {
+        return;
+    }
+
+    const uint32_t lane = threadIdx.x & 31u;
+    const uint32_t warp = threadIdx.x >> 5u;
+    const uint32_t h = head_group * HEADS_PER_GROUP + warp;
+    const bool valid_head = warp < HEADS_PER_GROUP && h < n_head;
+
+    /* One 512-float row is 128 float4 values.  Staging it once lets every
+     * verifier head reuse the same raw-ring traffic. */
+    __shared__ float4 kv_shared[ROWS_PER_STAGE * 128u];
+
+    const float4 *q4 = valid_head
+        ? (const float4 *)(q + ((uint64_t)tok * n_head + h) * head_dim)
+        : NULL;
+    float4 q0 = make_float4(0.0f, 0.0f, 0.0f, 0.0f);
+    float4 q1 = q0, q2 = q0, q3 = q0;
+    if (valid_head) {
+        q0 = q4[lane +  0u];
+        q1 = q4[lane + 32u];
+        q2 = q4[lane + 64u];
+        q3 = q4[lane + 96u];
+    }
+
+    const float scale = rsqrtf((float)head_dim);
+    float max_s = valid_head ? sinks[h] : -INFINITY;
+    float sum_s = valid_head ? 1.0f : 0.0f;
+    float4 o0 = make_float4(0.0f, 0.0f, 0.0f, 0.0f);
+    float4 o1 = o0, o2 = o0, o3 = o0;
+
+    for (uint32_t row0 = 0; row0 < n_raw; row0 += ROWS_PER_STAGE) {
+        const uint32_t nr = n_raw - row0 < ROWS_PER_STAGE
+            ? n_raw - row0 : ROWS_PER_STAGE;
+        for (uint32_t off = threadIdx.x;
+             off < nr * 128u;
+             off += blockDim.x) {
+            const uint32_t rr = off >> 7u;
+            const uint32_t c4 = off & 127u;
+            const uint32_t logical_row = row0 + rr;
+            /* n_raw <= raw_cap means the visible interval wraps at most once.
+             * Use 64-bit addition so a future large ring cannot overflow here. */
+            const uint32_t physical_row = (uint32_t)(
+                ((uint64_t)raw_start + logical_row) % raw_cap);
+            const float4 *src = (const float4 *)(
+                raw_kv + (uint64_t)physical_row * head_dim);
+            kv_shared[off] = src[c4];
+        }
+        __syncthreads();
+
+        if (valid_head) {
+            for (uint32_t rr = 0; rr < nr; rr++) {
+                const float4 *kv4 = kv_shared + rr * 128u;
+                const float4 k0 = kv4[lane +  0u];
+                const float4 k1 = kv4[lane + 32u];
+                const float4 k2 = kv4[lane + 64u];
+                const float4 k3 = kv4[lane + 96u];
+                float score = dot4_f32(q0, k0) + dot4_f32(q1, k1) +
+                              dot4_f32(q2, k2) + dot4_f32(q3, k3);
+                score = warp_sum_f32(score) * scale;
+                score = __shfl_sync(0xffffffffu, score, 0);
+
+                const float new_m = fmaxf(max_s, score);
+                const float old_scale = expf(max_s - new_m);
+                const float row_scale = expf(score - new_m);
+                sum_s = sum_s * old_scale + row_scale;
+                o0.x = o0.x * old_scale + k0.x * row_scale;
+                o0.y = o0.y * old_scale + k0.y * row_scale;
+                o0.z = o0.z * old_scale + k0.z * row_scale;
+                o0.w = o0.w * old_scale + k0.w * row_scale;
+                o1.x = o1.x * old_scale + k1.x * row_scale;
+                o1.y = o1.y * old_scale + k1.y * row_scale;
+                o1.z = o1.z * old_scale + k1.z * row_scale;
+                o1.w = o1.w * old_scale + k1.w * row_scale;
+                o2.x = o2.x * old_scale + k2.x * row_scale;
+                o2.y = o2.y * old_scale + k2.y * row_scale;
+                o2.z = o2.z * old_scale + k2.z * row_scale;
+                o2.w = o2.w * old_scale + k2.w * row_scale;
+                o3.x = o3.x * old_scale + k3.x * row_scale;
+                o3.y = o3.y * old_scale + k3.y * row_scale;
+                o3.z = o3.z * old_scale + k3.z * row_scale;
+                o3.w = o3.w * old_scale + k3.w * row_scale;
+                max_s = new_m;
+            }
+        }
+        __syncthreads();
+    }
+
+    if (valid_head) {
+        const float inv_s = sum_s == 0.0f ? 0.0f : 1.0f / sum_s;
+        o0.x *= inv_s; o0.y *= inv_s; o0.z *= inv_s; o0.w *= inv_s;
+        o1.x *= inv_s; o1.y *= inv_s; o1.z *= inv_s; o1.w *= inv_s;
+        o2.x *= inv_s; o2.y *= inv_s; o2.z *= inv_s; o2.w *= inv_s;
+        o3.x *= inv_s; o3.y *= inv_s; o3.z *= inv_s; o3.w *= inv_s;
+        float4 *out4 = (float4 *)(
+            heads + ((uint64_t)tok * n_head + h) * head_dim);
+        out4[lane +  0u] = o0;
+        out4[lane + 32u] = o1;
+        out4[lane + 64u] = o2;
+        out4[lane + 96u] = o3;
+    }
+}
+
 extern "C" int ds4_gpu_attention_noncausal_raw_batch_heads_tensor(
         ds4_gpu_tensor       *heads,
         const void             *model_map,
@@ -16377,14 +23272,27 @@ extern "C" int ds4_gpu_attention_noncausal_raw_batch_heads_tensor(
         uint32_t                raw_start,
         uint32_t                n_head,
         uint32_t                head_dim) {
-    if (!heads || !q || !raw_kv || !model_map ||
-        n_tokens == 0 || n_raw == 0 || raw_cap < n_raw ||
-        raw_start >= raw_cap || n_head == 0 || head_dim == 0 ||
-        sinks_offset > model_size ||
-        (uint64_t)n_head * sizeof(float) > model_size - sinks_offset ||
-        heads->bytes < (uint64_t)n_tokens * n_head * head_dim * sizeof(float) ||
-        q->bytes < (uint64_t)n_tokens * n_head * head_dim * sizeof(float) ||
-        raw_kv->bytes < (uint64_t)raw_cap * head_dim * sizeof(float)) {
+    if (!heads || !q || !raw_kv || !heads->ptr || !q->ptr || !raw_kv->ptr ||
+        !model_map ||
+        n_tokens == 0u || n_raw == 0u || raw_cap < n_raw ||
+        raw_start >= raw_cap || n_head == 0u || head_dim == 0u) {
+        return 0;
+    }
+    const uint64_t sink_bytes = (uint64_t)n_head * sizeof(float);
+    if ((uint64_t)n_tokens > UINT64_MAX / n_head) return 0;
+    const uint64_t head_rows = (uint64_t)n_tokens * n_head;
+    if (head_rows > UINT64_MAX / head_dim) return 0;
+    const uint64_t head_elems = head_rows * head_dim;
+    if (head_elems > UINT64_MAX / sizeof(float) ||
+        (uint64_t)raw_cap > UINT64_MAX / head_dim) {
+        return 0;
+    }
+    const uint64_t raw_elems = (uint64_t)raw_cap * head_dim;
+    if (raw_elems > UINT64_MAX / sizeof(float) ||
+        sinks_offset > model_size || sink_bytes > model_size - sinks_offset ||
+        heads->bytes < head_elems * sizeof(float) ||
+        q->bytes < head_elems * sizeof(float) ||
+        raw_kv->bytes < raw_elems * sizeof(float)) {
         return 0;
     }
     const int logical_tier = ds4_tensor_device_idx(heads);
@@ -16392,16 +23300,56 @@ extern "C" int ds4_gpu_attention_noncausal_raw_batch_heads_tensor(
             model_map, sinks_offset, (uint64_t)n_head * sizeof(float), logical_tier,
             "dspark_attn_sinks");
     if (!sinks) return 0;
-    const size_t shmem = (size_t)n_raw * sizeof(float);
-    if (shmem > 32768) return 0; /* draft blocks are tiny; guard anyway */
-    dim3 grid(n_tokens, n_head, 1);
-    attention_noncausal_raw_batch_heads_kernel<<<grid, 256, shmem>>>(
-            (float *)heads->ptr,
-            sinks,
-            (const float *)q->ptr,
-            (const float *)raw_kv->ptr,
-            n_tokens, n_raw, raw_cap, raw_start, n_head, head_dim);
-    if (!cuda_ok(cudaGetLastError(), "attention noncausal raw batch heads launch")) return 0;
+
+    /* The reference path remains the unconditional fallback.  The online path
+     * is deliberately narrow: DSpark's raw ring is eight rows, its current
+     * attention head is 512 floats, and float4 staging requires aligned device
+     * views.  The disable flag always wins over the enable flag. */
+    const bool online_requested =
+        cuda_env_flag_enabled("DS4_CUDA_ENABLE_DSPARK_NONCAUSAL_ONLINE", 0) &&
+        !cuda_env_flag_enabled("DS4_CUDA_DISABLE_DSPARK_NONCAUSAL_ONLINE", 0);
+    const bool online_shape =
+        n_tokens <= 8u && n_raw <= 8u && head_dim == 512u &&
+        n_head <= 256u &&
+        ds4_tensor_device_idx(q) == logical_tier &&
+        ds4_tensor_device_idx(raw_kv) == logical_tier &&
+        ((((uintptr_t)heads->ptr | (uintptr_t)q->ptr |
+           (uintptr_t)raw_kv->ptr) & 15u) == 0u);
+    if (online_requested && online_shape) {
+        static int logged_online = 0;
+        if (!logged_online) {
+            logged_online = 1;
+            fprintf(stderr,
+                    "ds4: CUDA DSpark noncausal attention using tiled "
+                    "online softmax\n");
+        }
+        dim3 grid(n_tokens, (n_head + 7u) / 8u, 1u);
+        attention_noncausal_raw_batch_heads_online_kernel<4, 8>
+            <<<grid, 256>>>(
+                (float *)heads->ptr,
+                sinks,
+                (const float *)q->ptr,
+                (const float *)raw_kv->ptr,
+                n_tokens, n_raw, raw_cap, raw_start, n_head, head_dim);
+        if (!cuda_ok(cudaGetLastError(),
+                     "attention noncausal raw batch heads online launch")) {
+            return 0;
+        }
+    } else {
+        const size_t shmem = (size_t)n_raw * sizeof(float);
+        if (shmem > 32768u) return 0;
+        dim3 grid(n_tokens, n_head, 1u);
+        attention_noncausal_raw_batch_heads_kernel<<<grid, 256, shmem>>>(
+                (float *)heads->ptr,
+                sinks,
+                (const float *)q->ptr,
+                (const float *)raw_kv->ptr,
+                n_tokens, n_raw, raw_cap, raw_start, n_head, head_dim);
+        if (!cuda_ok(cudaGetLastError(),
+                     "attention noncausal raw batch heads launch")) {
+            return 0;
+        }
+    }
     static int verify_left = -1;
     if (verify_left < 0) {
         verify_left = getenv("DS4_DSPARK_VERIFY_NONCAUSAL") != NULL ? 3 : 0;
@@ -18691,6 +25639,22 @@ extern "C" int ds4_gpu_attention_output_q8_batch_tensor(
     const unsigned char *out_b = reinterpret_cast<const unsigned char *>(
             cuda_resolve_weight_ptr(model_map, out_b_offset, out_b_bytes, logical_tier, "attn_out_b"));
     if (!out_a || !out_b) return 0;
+    const bool out_a_aligned_candidate =
+        g_n_gpus == 1 &&
+        logical_tier >= 0 && logical_tier < DS4_MAX_GPUS &&
+        g_cuda_is_gb10[logical_tier] &&
+        getenv("DS4_CUDA_NO_Q8_FUSED_ALIGNED") == NULL &&
+        n_tokens == 1u && group_dim <= INT_MAX && low_dim <= INT_MAX &&
+        (group_dim % 1024u) == 0u && (low_dim % 128u) == 0u &&
+        cuda_aligned_q8_enabled() && cuda_q8_use_dp4a();
+    const uint64_t out_a_aligned_bytes = out_a_aligned_candidate
+        ? ds4_mmq_q8_0_aligned_bytes((int)low_dim, (int)group_dim) : 0u;
+    const char *out_a_aligned = out_a_aligned_bytes
+        ? cuda_derived_weight_ptr(
+              model_map, out_a_offset, out_a_bytes,
+              CUDA_DERIVED_Q8_0_ALIGNED_DENSE,
+              group_dim, low_dim, 1u, out_a_aligned_bytes)
+        : NULL;
 
     const uint32_t profile = getenv("DS4_CUDA_ATTN_OUTPUT_PROFILE") != NULL;
     cudaEvent_t prof_ev[3] = {NULL, NULL, NULL};
@@ -18820,16 +25784,29 @@ extern "C" int ds4_gpu_attention_output_q8_batch_tensor(
                                                                    use_dp4a);
         } else {
             dim3 grid_a(((unsigned)low_dim + 7u) / 8u, (unsigned)n_tokens, 1);
-            grouped_q8_0_a_preq_warp8_kernel<<<grid_a, 256, 0, cuda_decode_stream()>>>((float *)low->ptr,
-                                                              out_a,
-                                                              xq,
-                                                              xscale,
-                                                              group_dim,
-                                                              rank,
-                                                              n_groups,
-                                                              n_tokens,
-                                                              blocks_a,
-                                                              use_dp4a);
+            if (out_a_aligned) {
+                const uint64_t nblk = low_dim * blocks_a;
+                const uint64_t dq_bytes =
+                    (nblk * sizeof(__half) + 63u) & ~63ull;
+                grouped_q8_0_a_aligned_preq_warp8_kernel<<<
+                    grid_a, 256, 0, cuda_decode_stream()>>>(
+                        (float *)low->ptr,
+                        (const int4 *)(out_a_aligned + dq_bytes),
+                        (const __half *)out_a_aligned,
+                        xq, xscale, rank, n_groups, n_tokens, blocks_a);
+            } else {
+                grouped_q8_0_a_preq_warp8_kernel<<<grid_a, 256, 0, cuda_decode_stream()>>>(
+                        (float *)low->ptr,
+                        out_a,
+                        xq,
+                        xscale,
+                        group_dim,
+                        rank,
+                        n_groups,
+                        n_tokens,
+                        blocks_a,
+                        use_dp4a);
+            }
         }
         if (!cuda_ok(cudaGetLastError(), "attention_output_q8_a preq launch")) return 0;
     }
@@ -18904,6 +25881,23 @@ extern "C" int ds4_gpu_attention_output_low_q8_rows_exact_tensor(
             cuda_resolve_weight_ptr(model_map, a_offset, out_a_bytes,
                                     logical_tier, "attn_out_a_rows"));
     if (!out_a) return 0;
+    const bool aligned_candidate =
+        g_n_gpus == 1 &&
+        logical_tier >= 0 && logical_tier < DS4_MAX_GPUS &&
+        g_cuda_is_gb10[logical_tier] &&
+        getenv("DS4_CUDA_NO_Q8_FUSED_ALIGNED") == NULL &&
+        n_rows == 1u && group0 == 0u && group_cnt == n_groups_total &&
+        group_dim <= INT_MAX && low_dim <= INT_MAX &&
+        (group_dim % 1024u) == 0u && (low_dim % 128u) == 0u &&
+        cuda_aligned_q8_enabled() && cuda_q8_use_dp4a();
+    const uint64_t aligned_bytes = aligned_candidate
+        ? ds4_mmq_q8_0_aligned_bytes((int)low_dim, (int)group_dim) : 0u;
+    const char *aligned = aligned_bytes
+        ? cuda_derived_weight_ptr(
+              model_map, a_offset, out_a_bytes,
+              CUDA_DERIVED_Q8_0_ALIGNED_DENSE,
+              group_dim, low_dim, 1u, aligned_bytes)
+        : NULL;
 
     const uint64_t x_rows = (uint64_t)n_rows * group_cnt;
     const uint64_t xq_bytes = x_rows * blocks_a * 32u;
@@ -18927,16 +25921,29 @@ extern "C" int ds4_gpu_attention_output_low_q8_rows_exact_tensor(
     if (!cuda_ok(cudaGetLastError(),
                  "attention_output_low_q8 rows prequant launch")) return 0;
     dim3 grid_a(((unsigned)low_dim + 7u) / 8u, n_rows, 1u);
-    grouped_q8_0_a_preq_warp8_kernel<<<grid_a, 256, 0, cuda_decode_stream()>>>((float *)low->ptr,
-                                                      out_a,
-                                                      xq,
-                                                      xscale,
-                                                      group_dim,
-                                                      rank,
-                                                      group_cnt,
-                                                      n_rows,
-                                                      blocks_a,
-                                                      use_dp4a);
+    if (aligned) {
+        const uint64_t nblk = low_dim * blocks_a;
+        const uint64_t dq_bytes =
+            (nblk * sizeof(__half) + 63u) & ~63ull;
+        grouped_q8_0_a_aligned_preq_warp8_kernel<<<
+            grid_a, 256, 0, cuda_decode_stream()>>>(
+                (float *)low->ptr,
+                (const int4 *)(aligned + dq_bytes),
+                (const __half *)aligned,
+                xq, xscale, rank, group_cnt, n_rows, blocks_a);
+    } else {
+        grouped_q8_0_a_preq_warp8_kernel<<<grid_a, 256, 0, cuda_decode_stream()>>>(
+                (float *)low->ptr,
+                out_a,
+                xq,
+                xscale,
+                group_dim,
+                rank,
+                group_cnt,
+                n_rows,
+                blocks_a,
+                use_dp4a);
+    }
     return cuda_ok(cudaGetLastError(),
                    "attention_output_low_q8 rows launch");
 }
@@ -19586,45 +26593,6 @@ __device__ static void dev_dot_iq2_xxs_q8_K_block4(
     for (uint32_t p = 0; p < n; p++) acc[p] += 0.125f * xd * ys[p]->d * (float)bsum[p];
 }
 
-__device__ static DS4_CUDA_UNUSED void dev_dot_iq2_xxs_q8_K_block8(
-        const cuda_block_iq2_xxs *x,
-        const cuda_block_q8_K *y0,
-        const cuda_block_q8_K *y1,
-        const cuda_block_q8_K *y2,
-        const cuda_block_q8_K *y3,
-        const cuda_block_q8_K *y4,
-        const cuda_block_q8_K *y5,
-        const cuda_block_q8_K *y6,
-        const cuda_block_q8_K *y7,
-        uint32_t n,
-        float acc[8]) {
-    const float xd = dev_f16_to_f32(x->d);
-    const uint16_t *q2 = x->qs;
-    int32_t bsum[8] = {0, 0, 0, 0, 0, 0, 0, 0};
-    const int8_t *q8[8] = {
-        y0 ? y0->qs : NULL, y1 ? y1->qs : NULL, y2 ? y2->qs : NULL, y3 ? y3->qs : NULL,
-        y4 ? y4->qs : NULL, y5 ? y5->qs : NULL, y6 ? y6->qs : NULL, y7 ? y7->qs : NULL,
-    };
-    for (int ib32 = 0; ib32 < CUDA_QK_K / 32; ib32++) {
-        const uint32_t aux0 = (uint32_t)q2[0] | ((uint32_t)q2[1] << 16);
-        const uint32_t aux1 = (uint32_t)q2[2] | ((uint32_t)q2[3] << 16);
-        q2 += 4;
-        const uint32_t ls = 2u * (aux1 >> 28) + 1u;
-        const uint8_t a0 = (uint8_t)(aux0 & 0xffu);
-        const uint8_t a1 = (uint8_t)((aux0 >> 8) & 0xffu);
-        const uint8_t a2 = (uint8_t)((aux0 >> 16) & 0xffu);
-        const uint8_t a3 = (uint8_t)((aux0 >> 24) & 0xffu);
-        for (uint32_t p = 0; p < n; p++) {
-            int32_t sumi = 0;
-            sumi += dev_dot_iq2_pair_16(a0, (aux1 >> 0) & 127u, a1, (aux1 >> 7) & 127u, q8[p] + ib32 * 32);
-            sumi += dev_dot_iq2_pair_16(a2, (aux1 >> 14) & 127u, a3, (aux1 >> 21) & 127u, q8[p] + ib32 * 32 + 16);
-            bsum[p] += sumi * (int32_t)ls;
-        }
-    }
-    const cuda_block_q8_K *ys[8] = { y0, y1, y2, y3, y4, y5, y6, y7 };
-    for (uint32_t p = 0; p < n; p++) acc[p] += 0.125f * xd * ys[p]->d * (float)bsum[p];
-}
-
 __device__ static void dev_q4_K_get_scale_min(
         uint32_t j,
         const uint8_t *scales,
@@ -19639,6 +26607,48 @@ __device__ static void dev_q4_K_get_scale_min(
     }
 }
 
+/* Expand one contiguous 16-value chunk per thread.  This mirrors the Q4_K
+ * production dequantization algebra and performs exactly one float-to-half
+ * rounding when publishing the resident matrix.  The source pointer may be
+ * device, managed/HMM, or CUDA-mapped host memory. */
+__global__ static void dequant_q4_K_to_f16_kernel(
+        __half *out,
+        const cuda_block_q4_K *w,
+        uint64_t in_dim,
+        uint64_t out_dim,
+        uint64_t blocks) {
+    const uint64_t chunk =
+        (uint64_t)blockIdx.x * blockDim.x + threadIdx.x;
+    const uint64_t chunks_per_row = in_dim / 16u;
+    const uint64_t total_chunks = out_dim * chunks_per_row;
+    if (chunk >= total_chunks) return;
+
+    const uint64_t row = chunk / chunks_per_row;
+    const uint64_t col0 = (chunk - row * chunks_per_row) * 16u;
+    const uint64_t block_in_row = col0 / CUDA_QK_K;
+    const uint32_t within0 = (uint32_t)(col0 % CUDA_QK_K);
+    const cuda_block_q4_K *block = w + row * blocks + block_in_row;
+    const float d = dev_f16_to_f32(block->d);
+    const float dmin = dev_f16_to_f32(block->dmin);
+
+#pragma unroll
+    for (uint32_t k = 0; k < 16u; k++) {
+        const uint32_t within = within0 + k;
+        const uint32_t group = within >> 5u;
+        uint8_t scale_code, min_code;
+        dev_q4_K_get_scale_min(
+            group, block->scales, &scale_code, &min_code);
+        const uint8_t packed =
+            block->qs[(group >> 1u) * 32u + (within & 31u)];
+        const uint8_t q = (group & 1u) ? (packed >> 4u)
+                                       : (packed & 15u);
+        const float value =
+            (d * (float)scale_code) * (float)q -
+            dmin * (float)min_code;
+        out[row * in_dim + col0 + k] = __float2half_rn(value);
+    }
+}
+
 __device__ __forceinline__ static int32_t dev_dot_q4_32(const uint8_t *qs, const int8_t *q8, int shift) {
     int32_t sum = 0;
     #pragma unroll
@@ -20176,162 +27186,6 @@ __global__ static void q8_K_quantize_sidecar_kernel(
     if (tid == 0u) yb->d = 1.0f / iscale_s;
 }
 
-__global__ static DS4_CUDA_UNUSED void moe_gate_up_mid_kernel(
-        float *gate_out,
-        float *up_out,
-        float *mid_out,
-        const char *gate_base,
-        const char *up_base,
-        const cuda_block_q8_K *xq,
-        const int32_t *selected,
-        const float *weights,
-        uint64_t gate_expert_bytes,
-        uint64_t gate_row_bytes,
-        uint32_t xq_blocks,
-        uint32_t expert_mid_dim,
-        uint32_t n_expert,
-        float clamp) {
-    uint32_t row = blockIdx.x;
-    uint32_t pair = blockIdx.y;
-    if (row >= expert_mid_dim) return;
-    uint32_t tok = pair / n_expert;
-    uint32_t slot = pair - tok * n_expert;
-    int32_t expert_i = selected[(uint64_t)tok * n_expert + slot];
-    if (expert_i < 0) expert_i = 0;
-    uint32_t expert = (uint32_t)expert_i;
-    const cuda_block_iq2_xxs *gr = (const cuda_block_iq2_xxs *)(gate_base + (uint64_t)expert * gate_expert_bytes + (uint64_t)row * gate_row_bytes);
-    const cuda_block_iq2_xxs *ur = (const cuda_block_iq2_xxs *)(up_base + (uint64_t)expert * gate_expert_bytes + (uint64_t)row * gate_row_bytes);
-    const cuda_block_q8_K *xqb = xq + (uint64_t)tok * xq_blocks;
-    float gate = 0.0f;
-    float up = 0.0f;
-    for (uint32_t b = threadIdx.x; b < xq_blocks; b += blockDim.x) {
-        gate += dev_dot_iq2_xxs_q8_K_block(gr + b, xqb + b);
-        up += dev_dot_iq2_xxs_q8_K_block(ur + b, xqb + b);
-    }
-    __shared__ float partial_gate[256];
-    __shared__ float partial_up[256];
-    partial_gate[threadIdx.x] = gate;
-    partial_up[threadIdx.x] = up;
-    __syncthreads();
-    for (uint32_t stride = blockDim.x >> 1; stride > 0; stride >>= 1) {
-        if (threadIdx.x < stride) {
-            partial_gate[threadIdx.x] += partial_gate[threadIdx.x + stride];
-            partial_up[threadIdx.x] += partial_up[threadIdx.x + stride];
-        }
-        __syncthreads();
-    }
-    if (threadIdx.x == 0) {
-        gate = partial_gate[0];
-        up = partial_up[0];
-        if (clamp > 1.0e-6f) {
-            if (gate > clamp) gate = clamp;
-            if (up > clamp) up = clamp;
-            if (up < -clamp) up = -clamp;
-        }
-        const uint64_t off = (uint64_t)pair * expert_mid_dim + row;
-        gate_out[off] = gate;
-        up_out[off] = up;
-        mid_out[off] = (gate / (1.0f + expf(-gate))) * up * weights[(uint64_t)tok * n_expert + slot];
-    }
-}
-
-__global__ static DS4_CUDA_UNUSED void moe_gate_up_mid_warp8_kernel(
-        float *gate_out,
-        float *up_out,
-        float *mid_out,
-        const char *gate_base,
-        const char *up_base,
-        const cuda_block_q8_K *xq,
-        const int32_t *selected,
-        const float *weights,
-        uint64_t gate_expert_bytes,
-        uint64_t gate_row_bytes,
-        uint32_t xq_blocks,
-        uint32_t expert_mid_dim,
-        uint32_t n_expert,
-        float clamp) {
-    uint32_t lane = threadIdx.x & 31u;
-    uint32_t warp = threadIdx.x >> 5u;
-    uint32_t row = blockIdx.x * 8u + warp;
-    uint32_t pair = blockIdx.y;
-    if (row >= expert_mid_dim) return;
-    uint32_t tok = pair / n_expert;
-    uint32_t slot = pair - tok * n_expert;
-    int32_t expert_i = selected[(uint64_t)tok * n_expert + slot];
-    if (expert_i < 0) expert_i = 0;
-    uint32_t expert = (uint32_t)expert_i;
-    const cuda_block_iq2_xxs *gr = (const cuda_block_iq2_xxs *)(gate_base + (uint64_t)expert * gate_expert_bytes + (uint64_t)row * gate_row_bytes);
-    const cuda_block_iq2_xxs *ur = (const cuda_block_iq2_xxs *)(up_base + (uint64_t)expert * gate_expert_bytes + (uint64_t)row * gate_row_bytes);
-    const cuda_block_q8_K *xqb = xq + (uint64_t)tok * xq_blocks;
-    float gate = 0.0f;
-    float up = 0.0f;
-    for (uint32_t b = lane; b < xq_blocks; b += 32u) {
-        gate += dev_dot_iq2_xxs_q8_K_block(gr + b, xqb + b);
-        up += dev_dot_iq2_xxs_q8_K_block(ur + b, xqb + b);
-    }
-    gate = warp_sum_f32(gate);
-    up = warp_sum_f32(up);
-    if (lane == 0) {
-        if (clamp > 1.0e-6f) {
-            if (gate > clamp) gate = clamp;
-            if (up > clamp) up = clamp;
-            if (up < -clamp) up = -clamp;
-        }
-        const uint64_t off = (uint64_t)pair * expert_mid_dim + row;
-        gate_out[off] = gate;
-        up_out[off] = up;
-        mid_out[off] = (gate / (1.0f + expf(-gate))) * up * weights[(uint64_t)tok * n_expert + slot];
-    }
-}
-
-__global__ static DS4_CUDA_UNUSED void moe_gate_up_mid_hwarp16_kernel(
-        float *gate_out,
-        float *up_out,
-        float *mid_out,
-        const char *gate_base,
-        const char *up_base,
-        const cuda_block_q8_K *xq,
-        const int32_t *selected,
-        const float *weights,
-        uint64_t gate_expert_bytes,
-        uint64_t gate_row_bytes,
-        uint32_t xq_blocks,
-        uint32_t expert_mid_dim,
-        uint32_t n_expert,
-        float clamp) {
-    uint32_t lane = threadIdx.x & 15u;
-    uint32_t row = blockIdx.x * 16u + (threadIdx.x >> 4u);
-    uint32_t pair = blockIdx.y;
-    if (row >= expert_mid_dim) return;
-    uint32_t tok = pair / n_expert;
-    uint32_t slot = pair - tok * n_expert;
-    int32_t expert_i = selected[(uint64_t)tok * n_expert + slot];
-    if (expert_i < 0) expert_i = 0;
-    uint32_t expert = (uint32_t)expert_i;
-    const cuda_block_iq2_xxs *gr = (const cuda_block_iq2_xxs *)(gate_base + (uint64_t)expert * gate_expert_bytes + (uint64_t)row * gate_row_bytes);
-    const cuda_block_iq2_xxs *ur = (const cuda_block_iq2_xxs *)(up_base + (uint64_t)expert * gate_expert_bytes + (uint64_t)row * gate_row_bytes);
-    const cuda_block_q8_K *xqb = xq + (uint64_t)tok * xq_blocks;
-    float gate = 0.0f;
-    float up = 0.0f;
-    for (uint32_t b = lane; b < xq_blocks; b += 16u) {
-        gate += dev_dot_iq2_xxs_q8_K_block(gr + b, xqb + b);
-        up += dev_dot_iq2_xxs_q8_K_block(ur + b, xqb + b);
-    }
-    gate = half_warp_sum_f32(gate, lane);
-    up = half_warp_sum_f32(up, lane);
-    if (lane == 0) {
-        if (clamp > 1.0e-6f) {
-            if (gate > clamp) gate = clamp;
-            if (up > clamp) up = clamp;
-            if (up < -clamp) up = -clamp;
-        }
-        const uint64_t off = (uint64_t)pair * expert_mid_dim + row;
-        gate_out[off] = gate;
-        up_out[off] = up;
-        mid_out[off] = (gate / (1.0f + expf(-gate))) * up * weights[(uint64_t)tok * n_expert + slot];
-    }
-}
-
 // perf-04: launch-geometry tuning for the routed-MoE gate/up decode kernels
 // (moe_gate_up_mid_qwarp32 / _decode_lut_qwarp32 / _decode_q4K_qwarp32). Each
 // block processes MOE_DECODE_ROW_TILES tiles of 32 rows (row_lane in [0,32)).
@@ -20749,68 +27603,6 @@ __global__ static void moe_gate_up_mid_sorted_qwarp32_kernel(
     }
 }
 
-__global__ static DS4_CUDA_UNUSED void moe_gate_up_mid_expert_tile8_kernel(
-        float *gate_out,
-        float *up_out,
-        float *mid_out,
-        const char *gate_base,
-        const char *up_base,
-        const cuda_block_q8_K *xq,
-        const uint32_t *sorted_pairs,
-        const uint32_t *offsets,
-        const uint32_t *counts,
-        const uint32_t *tile_total,
-        const uint32_t *tile_experts,
-        const uint32_t *tile_starts,
-        const float *weights,
-        uint64_t gate_expert_bytes,
-        uint64_t gate_row_bytes,
-        uint32_t xq_blocks,
-        uint32_t expert_mid_dim,
-        uint32_t n_expert,
-        float clamp) {
-    uint32_t tile = blockIdx.y;
-    if (tile >= *tile_total) return;
-    uint32_t group = threadIdx.x >> 3u;
-    uint32_t lane = threadIdx.x & 7u;
-    uint32_t pair_slot = group & 7u;
-    uint32_t row_lane = group >> 3u;
-    uint32_t expert = tile_experts[tile];
-    uint32_t local_pair = tile_starts[tile] + pair_slot;
-    if (local_pair >= counts[expert]) return;
-    uint32_t sorted_idx = offsets[expert] + local_pair;
-    uint32_t pair = sorted_pairs[sorted_idx];
-    uint32_t tok = pair / n_expert;
-    uint32_t slot = pair - tok * n_expert;
-    const cuda_block_q8_K *xqb = xq + (uint64_t)tok * xq_blocks;
-
-    for (uint32_t rr = 0; rr < 2u; rr++) {
-        uint32_t row = blockIdx.x * 8u + row_lane + rr * 4u;
-        if (row >= expert_mid_dim) continue;
-        const cuda_block_iq2_xxs *gr = (const cuda_block_iq2_xxs *)(gate_base + (uint64_t)expert * gate_expert_bytes + (uint64_t)row * gate_row_bytes);
-        const cuda_block_iq2_xxs *ur = (const cuda_block_iq2_xxs *)(up_base + (uint64_t)expert * gate_expert_bytes + (uint64_t)row * gate_row_bytes);
-        float gate = 0.0f;
-        float up = 0.0f;
-        for (uint32_t b = lane; b < xq_blocks; b += 8u) {
-            gate += dev_dot_iq2_xxs_q8_K_block(gr + b, xqb + b);
-            up += dev_dot_iq2_xxs_q8_K_block(ur + b, xqb + b);
-        }
-        gate = quarter_warp_sum_f32(gate, lane);
-        up = quarter_warp_sum_f32(up, lane);
-        if (lane == 0) {
-            if (clamp > 1.0e-6f) {
-                if (gate > clamp) gate = clamp;
-                if (up > clamp) up = clamp;
-                if (up < -clamp) up = -clamp;
-            }
-            const uint64_t off = (uint64_t)pair * expert_mid_dim + row;
-            gate_out[off] = gate;
-            up_out[off] = up;
-            mid_out[off] = (gate / (1.0f + expf(-gate))) * up * weights[(uint64_t)tok * n_expert + slot];
-        }
-    }
-}
-
 __global__ static void moe_gate_up_mid_expert_tile4_row32_kernel(
         float *gate_out,
         float *up_out,
@@ -21220,90 +28012,6 @@ __global__ static void moe_gate_up_mid_sorted_p2_qwarp32_kernel(
     }
 }
 
-__global__ static DS4_CUDA_UNUSED void moe_down_kernel(
-        float *down_out,
-        const char *down_base,
-        const cuda_block_q8_K *midq,
-        const int32_t *selected,
-        uint64_t down_expert_bytes,
-        uint64_t down_row_bytes,
-        uint32_t midq_blocks,
-        uint32_t out_dim,
-        uint32_t n_expert) {
-    uint32_t row = blockIdx.x;
-    uint32_t pair = blockIdx.y;
-    if (row >= out_dim) return;
-    uint32_t tok = pair / n_expert;
-    uint32_t slot = pair - tok * n_expert;
-    int32_t expert_i = selected[(uint64_t)tok * n_expert + slot];
-    if (expert_i < 0) expert_i = 0;
-    const cuda_block_q2_K *wr = (const cuda_block_q2_K *)(down_base + (uint64_t)(uint32_t)expert_i * down_expert_bytes + (uint64_t)row * down_row_bytes);
-    const cuda_block_q8_K *xq = midq + (uint64_t)pair * midq_blocks;
-    float acc = 0.0f;
-    for (uint32_t b = threadIdx.x; b < midq_blocks; b += blockDim.x) acc += dev_dot_q2_K_q8_K_block(wr + b, xq + b);
-    __shared__ float partial[256];
-    partial[threadIdx.x] = acc;
-    __syncthreads();
-    for (uint32_t stride = blockDim.x >> 1; stride > 0; stride >>= 1) {
-        if (threadIdx.x < stride) partial[threadIdx.x] += partial[threadIdx.x + stride];
-        __syncthreads();
-    }
-    if (threadIdx.x == 0) down_out[(uint64_t)pair * out_dim + row] = partial[0];
-}
-
-__global__ static DS4_CUDA_UNUSED void moe_down_warp8_kernel(
-        float *down_out,
-        const char *down_base,
-        const cuda_block_q8_K *midq,
-        const int32_t *selected,
-        uint64_t down_expert_bytes,
-        uint64_t down_row_bytes,
-        uint32_t midq_blocks,
-        uint32_t out_dim,
-        uint32_t n_expert) {
-    uint32_t lane = threadIdx.x & 31u;
-    uint32_t warp = threadIdx.x >> 5u;
-    uint32_t row = blockIdx.x * 8u + warp;
-    uint32_t pair = blockIdx.y;
-    if (row >= out_dim) return;
-    uint32_t tok = pair / n_expert;
-    uint32_t slot = pair - tok * n_expert;
-    int32_t expert_i = selected[(uint64_t)tok * n_expert + slot];
-    if (expert_i < 0) expert_i = 0;
-    const cuda_block_q2_K *wr = (const cuda_block_q2_K *)(down_base + (uint64_t)(uint32_t)expert_i * down_expert_bytes + (uint64_t)row * down_row_bytes);
-    const cuda_block_q8_K *xq = midq + (uint64_t)pair * midq_blocks;
-    float acc = 0.0f;
-    for (uint32_t b = lane; b < midq_blocks; b += 32u) acc += dev_dot_q2_K_q8_K_block(wr + b, xq + b);
-    acc = warp_sum_f32(acc);
-    if (lane == 0) down_out[(uint64_t)pair * out_dim + row] = acc;
-}
-
-__global__ static DS4_CUDA_UNUSED void moe_down_hwarp16_kernel(
-        float *down_out,
-        const char *down_base,
-        const cuda_block_q8_K *midq,
-        const int32_t *selected,
-        uint64_t down_expert_bytes,
-        uint64_t down_row_bytes,
-        uint32_t midq_blocks,
-        uint32_t out_dim,
-        uint32_t n_expert) {
-    uint32_t lane = threadIdx.x & 15u;
-    uint32_t row = blockIdx.x * 16u + (threadIdx.x >> 4u);
-    uint32_t pair = blockIdx.y;
-    if (row >= out_dim) return;
-    uint32_t tok = pair / n_expert;
-    uint32_t slot = pair - tok * n_expert;
-    int32_t expert_i = selected[(uint64_t)tok * n_expert + slot];
-    if (expert_i < 0) expert_i = 0;
-    const cuda_block_q2_K *wr = (const cuda_block_q2_K *)(down_base + (uint64_t)(uint32_t)expert_i * down_expert_bytes + (uint64_t)row * down_row_bytes);
-    const cuda_block_q8_K *xq = midq + (uint64_t)pair * midq_blocks;
-    float acc = 0.0f;
-    for (uint32_t b = lane; b < midq_blocks; b += 16u) acc += dev_dot_q2_K_q8_K_block(wr + b, xq + b);
-    acc = half_warp_sum_f32(acc, lane);
-    if (lane == 0) down_out[(uint64_t)pair * out_dim + row] = acc;
-}
-
 __global__ static void moe_down_qwarp32_kernel(
         float *down_out,
         const char *down_base,
@@ -23516,45 +30224,6 @@ __global__ static void moe_down_sorted_qwarp32_kernel(
     if (lane == 0) down_out[(uint64_t)pair * out_dim + row] = acc;
 }
 
-__global__ static DS4_CUDA_UNUSED void moe_down_expert_tile8_kernel(
-        float *down_out,
-        const char *down_base,
-        const cuda_block_q8_K *midq,
-        const uint32_t *sorted_pairs,
-        const uint32_t *offsets,
-        const uint32_t *counts,
-        const uint32_t *tile_total,
-        const uint32_t *tile_experts,
-        const uint32_t *tile_starts,
-        uint64_t down_expert_bytes,
-        uint64_t down_row_bytes,
-        uint32_t midq_blocks,
-        uint32_t out_dim,
-        uint32_t n_expert) {
-    uint32_t tile = blockIdx.y;
-    if (tile >= *tile_total) return;
-    uint32_t group = threadIdx.x >> 3u;
-    uint32_t lane = threadIdx.x & 7u;
-    uint32_t pair_slot = group & 7u;
-    uint32_t row_lane = group >> 3u;
-    uint32_t expert = tile_experts[tile];
-    uint32_t local_pair = tile_starts[tile] + pair_slot;
-    if (local_pair >= counts[expert]) return;
-    uint32_t sorted_idx = offsets[expert] + local_pair;
-    uint32_t pair = sorted_pairs[sorted_idx];
-    const cuda_block_q8_K *xq = midq + (uint64_t)pair * midq_blocks;
-
-    for (uint32_t rr = 0; rr < 2u; rr++) {
-        uint32_t row = blockIdx.x * 8u + row_lane + rr * 4u;
-        if (row >= out_dim) continue;
-        const cuda_block_q2_K *wr = (const cuda_block_q2_K *)(down_base + (uint64_t)expert * down_expert_bytes + (uint64_t)row * down_row_bytes);
-        float acc = 0.0f;
-        for (uint32_t b = lane; b < midq_blocks; b += 8u) acc += dev_dot_q2_K_q8_K_block(wr + b, xq + b);
-        acc = quarter_warp_sum_f32(acc, lane);
-        if (lane == 0) down_out[(uint64_t)pair * out_dim + row] = acc;
-    }
-}
-
 __global__ static void moe_down_expert_tile4_row32_kernel(
         float *down_out,
         const char *down_base,
@@ -24112,6 +30781,280 @@ __global__ static void moe_down_f32_kernel(
     if (threadIdx.x == 0) down_out[(uint64_t)pair * out_dim + row] = partial[0];
 }
 
+typedef struct {
+    int valid;
+    int top6_unique;
+    int storage_kind;
+    const char *gate;
+    const char *up;
+    const char *down;
+    const ds4_gpu_tensor *selected;
+    uint32_t slot_base;
+    uint32_t weight_domain;
+    uint64_t generation;
+    uint64_t upload_event_value;
+} cuda_stream_selected_binding;
+
+static int cuda_stream_selected_binding_acquire(
+        cuda_stream_selected_binding *binding,
+        int logical_tier,
+        const void *model_map,
+        uint32_t layer_index,
+        uint32_t n_total_expert,
+        uint64_t gate_offset,
+        uint64_t up_offset,
+        uint64_t down_offset,
+        uint64_t gate_expert_bytes,
+        uint64_t down_expert_bytes,
+        uint64_t required_slot_count) {
+    if (!binding) return 0;
+    memset(binding, 0, sizeof(*binding));
+    if (gate_expert_bytes == 0 || down_expert_bytes == 0 ||
+        required_slot_count > UINT64_MAX / sizeof(int32_t) ||
+        !g_stream_selected_cache.valid ||
+        (g_stream_selected_cache.storage_kind !=
+             CUDA_STREAM_SELECTED_STORAGE_TRANSIENT &&
+         g_stream_selected_cache.storage_kind !=
+             CUDA_STREAM_SELECTED_STORAGE_PERSISTENT) ||
+        g_stream_selected_cache.logical_tier != logical_tier ||
+        g_stream_selected_cache.model_map != model_map ||
+        g_stream_selected_cache.layer != layer_index ||
+        g_stream_selected_cache.n_total_expert != n_total_expert ||
+        g_stream_selected_cache.slot_count != required_slot_count ||
+        g_stream_selected_cache.gate_offset != gate_offset ||
+        g_stream_selected_cache.up_offset != up_offset ||
+        g_stream_selected_cache.down_offset != down_offset ||
+        g_stream_selected_cache.gate_expert_bytes != gate_expert_bytes ||
+        g_stream_selected_cache.down_expert_bytes != down_expert_bytes ||
+        !g_stream_selected_cache.gate_ptr ||
+        !g_stream_selected_cache.up_ptr ||
+        !g_stream_selected_cache.down_ptr ||
+        !g_stream_selected_cache.slot_selected_tensor.ptr ||
+        g_stream_selected_cache.slot_selected_tensor.bytes <
+            required_slot_count * sizeof(int32_t) ||
+        g_stream_selected_cache.generation == 0 ||
+        g_stream_selected_cache.weight_domain == 0 ||
+        g_stream_selected_cache.slot_base > UINT32_MAX -
+            g_stream_selected_cache.weight_domain) {
+        return 0;
+    }
+    const uint64_t end_slot =
+        (uint64_t)g_stream_selected_cache.slot_base +
+        g_stream_selected_cache.weight_domain;
+    if (end_slot > UINT64_MAX / gate_expert_bytes ||
+        end_slot * gate_expert_bytes >
+            g_stream_selected_cache.gate_capacity ||
+        end_slot * gate_expert_bytes >
+            g_stream_selected_cache.up_capacity ||
+        end_slot > UINT64_MAX / down_expert_bytes ||
+        end_slot * down_expert_bytes >
+            g_stream_selected_cache.down_capacity) {
+        return 0;
+    }
+    binding->valid = 1;
+    binding->top6_unique = g_stream_selected_cache.top6_unique;
+    binding->storage_kind = g_stream_selected_cache.storage_kind;
+    binding->gate = g_stream_selected_cache.gate_ptr +
+        (uint64_t)g_stream_selected_cache.slot_base * gate_expert_bytes;
+    binding->up = g_stream_selected_cache.up_ptr +
+        (uint64_t)g_stream_selected_cache.slot_base * gate_expert_bytes;
+    binding->down = g_stream_selected_cache.down_ptr +
+        (uint64_t)g_stream_selected_cache.slot_base * down_expert_bytes;
+    binding->selected = &g_stream_selected_cache.slot_selected_tensor;
+    binding->slot_base = g_stream_selected_cache.slot_base;
+    binding->weight_domain = g_stream_selected_cache.weight_domain;
+    binding->generation = g_stream_selected_cache.generation;
+    binding->upload_event_value =
+        g_stream_selected_cache.upload_event_value;
+    if (binding->storage_kind ==
+            CUDA_STREAM_SELECTED_STORAGE_PERSISTENT) {
+        g_stream_expert_persistent_dispatches.fetch_add(
+            1, std::memory_order_relaxed);
+    } else {
+        g_stream_expert_transient_dispatches.fetch_add(
+            1, std::memory_order_relaxed);
+    }
+    return 1;
+}
+
+static int cuda_iq2_ssd_grouped_raw_layout(
+        uint32_t gate_type, uint32_t down_type,
+        uint64_t gate_expert_bytes, uint64_t gate_row_bytes,
+        uint64_t down_expert_bytes, uint64_t down_row_bytes,
+        uint32_t expert_in_dim, uint32_t expert_mid_dim,
+        uint32_t out_dim) {
+    if (gate_type != 16u || down_type != 10u ||
+        expert_in_dim == 0u || expert_mid_dim == 0u || out_dim == 0u ||
+        (expert_in_dim % CUDA_QK_K) != 0u ||
+        (expert_mid_dim % CUDA_QK_K) != 0u) {
+        return 0;
+    }
+    const uint64_t iq2_blocks = expert_in_dim / CUDA_QK_K;
+    const uint64_t q2_blocks = expert_mid_dim / CUDA_QK_K;
+    if (iq2_blocks > UINT64_MAX / sizeof(cuda_block_iq2_xxs) ||
+        q2_blocks > UINT64_MAX / sizeof(cuda_block_q2_K)) {
+        return 0;
+    }
+    const uint64_t canonical_gate_row =
+        iq2_blocks * sizeof(cuda_block_iq2_xxs);
+    const uint64_t canonical_down_row =
+        q2_blocks * sizeof(cuda_block_q2_K);
+    if ((uint64_t)expert_mid_dim >
+            UINT64_MAX / canonical_gate_row ||
+        (uint64_t)out_dim > UINT64_MAX / canonical_down_row) {
+        return 0;
+    }
+    return gate_row_bytes == canonical_gate_row &&
+           gate_expert_bytes ==
+               (uint64_t)expert_mid_dim * canonical_gate_row &&
+           down_row_bytes == canonical_down_row &&
+           down_expert_bytes ==
+               (uint64_t)out_dim * canonical_down_row;
+}
+
+extern "C" int ds4_cuda_test_iq2_ssd_grouped_raw_layout(
+        uint32_t gate_type, uint32_t down_type,
+        uint64_t gate_expert_bytes, uint64_t gate_row_bytes,
+        uint64_t down_expert_bytes, uint64_t down_row_bytes,
+        uint32_t expert_in_dim, uint32_t expert_mid_dim,
+        uint32_t out_dim) {
+    return cuda_iq2_ssd_grouped_raw_layout(
+        gate_type, down_type, gate_expert_bytes, gate_row_bytes,
+        down_expert_bytes, down_row_bytes,
+        expert_in_dim, expert_mid_dim, out_dim);
+}
+
+enum { CUDA_MOE_FAST_PROFILE_MAX_EVENTS = 5 };
+
+static std::atomic<uint64_t> g_cuda_moe_fast_profile_reports{0};
+
+extern "C" uint64_t ds4_cuda_test_moe_fast_profile_report_count(void) {
+    return g_cuda_moe_fast_profile_reports.load(std::memory_order_relaxed);
+}
+
+typedef struct {
+    cudaEvent_t events[CUDA_MOE_FAST_PROFILE_MAX_EVENTS];
+    cudaStream_t stream;
+    uint32_t event_count;
+    int active;
+} cuda_moe_fast_profile;
+
+/* The legacy routed-MoE profiler is below the IQ2 MMQ early returns.  Keep a
+ * small event-only recorder for those paths so profiling never adds a stream
+ * synchronization (or even a CUDA call) unless DS4_CUDA_MOE_PROFILE is set.
+ * Destroying a recorded event is non-blocking; this makes failed MMQ entries
+ * safe to abandon before the established fallback is entered. */
+static void cuda_moe_fast_profile_destroy(cuda_moe_fast_profile *profile) {
+    if (!profile || (!profile->active && profile->event_count == 0u)) return;
+    for (uint32_t i = 0; i < CUDA_MOE_FAST_PROFILE_MAX_EVENTS; i++) {
+        if (profile->events[i]) {
+            (void)cudaEventDestroy(profile->events[i]);
+        }
+    }
+    memset(profile, 0, sizeof(*profile));
+}
+
+static void cuda_moe_fast_profile_begin(
+        cuda_moe_fast_profile *profile,
+        cudaStream_t stream,
+        uint32_t event_count) {
+    if (!profile) return;
+    memset(profile, 0, sizeof(*profile));
+    if (getenv("DS4_CUDA_MOE_PROFILE") == NULL) return;
+    if (event_count < 2u ||
+        event_count > CUDA_MOE_FAST_PROFILE_MAX_EVENTS) {
+        return;
+    }
+    cudaStreamCaptureStatus capture = cudaStreamCaptureStatusNone;
+    const cudaError_t capture_err =
+        cudaStreamIsCapturing(stream, &capture);
+    if (capture_err != cudaSuccess ||
+        capture != cudaStreamCaptureStatusNone) {
+        if (capture_err != cudaSuccess) (void)cudaGetLastError();
+        return;
+    }
+    profile->stream = stream;
+    profile->event_count = event_count;
+    for (uint32_t i = 0; i < event_count; i++) {
+        cudaEvent_t event = NULL;
+        if (cudaEventCreate(&event) != cudaSuccess) {
+            cuda_moe_fast_profile_destroy(profile);
+            return;
+        }
+        profile->events[i] = event;
+    }
+    if (cudaEventRecord(profile->events[0], stream) != cudaSuccess) {
+        cuda_moe_fast_profile_destroy(profile);
+        return;
+    }
+    profile->active = 1;
+}
+
+static void cuda_moe_fast_profile_mark(
+        cuda_moe_fast_profile *profile,
+        uint32_t event_index) {
+    if (!profile || !profile->active ||
+        event_index >= profile->event_count ||
+        cudaEventRecord(profile->events[event_index], profile->stream) !=
+            cudaSuccess) {
+        if (profile && profile->active) {
+            cuda_moe_fast_profile_destroy(profile);
+        }
+    }
+}
+
+static void cuda_moe_fast_profile_report(
+        cuda_moe_fast_profile *profile,
+        const char *path,
+        uint32_t n_tokens,
+        uint64_t assignments,
+        const char *const *stage_names) {
+    if (!profile || !profile->active) return;
+    const uint32_t event_count = profile->event_count;
+    float stage_ms[CUDA_MOE_FAST_PROFILE_MAX_EVENTS - 1] = {};
+    float total_ms = 0.0f;
+    int ok = cudaEventSynchronize(profile->events[event_count - 1u]) ==
+        cudaSuccess;
+    for (uint32_t i = 1; ok && i < event_count; i++) {
+        ok = cudaEventElapsedTime(
+                 &stage_ms[i - 1u], profile->events[i - 1u],
+                 profile->events[i]) == cudaSuccess;
+    }
+    if (ok) {
+        ok = cudaEventElapsedTime(
+                 &total_ms, profile->events[0],
+                 profile->events[event_count - 1u]) == cudaSuccess;
+    }
+    if (ok) {
+        char line[512] = {};
+        int written = snprintf(
+            line, sizeof(line),
+            "ds4: CUDA MoE profile path=%s tokens=%u assignments=%llu",
+            path, n_tokens, (unsigned long long)assignments);
+        size_t used = written > 0 ? (size_t)written : 0u;
+        if (used >= sizeof(line)) used = sizeof(line) - 1u;
+        for (uint32_t i = 1; i < event_count && used < sizeof(line); i++) {
+            written = snprintf(
+                line + used, sizeof(line) - used, " %s=%.3f",
+                stage_names[i - 1u], stage_ms[i - 1u]);
+            if (written < 0) break;
+            const size_t appended = (size_t)written;
+            used += appended < sizeof(line) - used
+                ? appended : sizeof(line) - used - 1u;
+        }
+        if (used < sizeof(line)) {
+            (void)snprintf(
+                line + used, sizeof(line) - used,
+                " total=%.3f ms (cudaEvent)", total_ms);
+        }
+        if (fprintf(stderr, "%s\n", line) >= 0) {
+            g_cuda_moe_fast_profile_reports.fetch_add(
+                1u, std::memory_order_relaxed);
+        }
+    }
+    cuda_moe_fast_profile_destroy(profile);
+}
+
 static int routed_moe_launch(
         ds4_gpu_tensor *out,
         ds4_gpu_tensor *gate,
@@ -24190,8 +31133,22 @@ static int routed_moe_launch(
         if (gate_aligned && up_aligned && down_aligned) {
             const cudaStream_t aligned_stream =
                 n_tokens == 1u ? cuda_decode_stream() : (cudaStream_t)0;
-            int rc;
-            if (n_tokens == 1u) {
+            cuda_moe_fast_profile aligned_profile;
+            if (n_tokens > 1u) {
+                memset(&aligned_profile, 0, sizeof(aligned_profile));
+            }
+            const char *aligned_profile_path = NULL;
+            const int dspark_tiny_aligned_vec =
+                n_tokens >= 2u && n_tokens <= 5u &&
+                cuda_env_flag_enabled(
+                    "DS4_CUDA_DSPARK_TINY_ALIGNED_VEC", 0);
+            int rc = -1;
+            if (n_tokens == 1u || dspark_tiny_aligned_vec) {
+                if (dspark_tiny_aligned_vec) {
+                    aligned_profile_path = "iq2_aligned_tiny_vec";
+                    cuda_moe_fast_profile_begin(
+                        &aligned_profile, aligned_stream, 4u);
+                }
                 rc = ds4_mmq_iq2_xxs_aligned_moe_gate_up_mid_vec(
                     gate_aligned, up_aligned,
                     (const float *)x->ptr,
@@ -24202,6 +31159,9 @@ static int routed_moe_launch(
                     (int)n_tokens, (int)n_total_expert, (int)n_expert,
                     clamp, aligned_stream);
                 if (rc == 0) {
+                    if (dspark_tiny_aligned_vec) {
+                        cuda_moe_fast_profile_mark(&aligned_profile, 1u);
+                    }
                     const uint32_t assignments = n_tokens * n_expert;
                     rc = ds4_mmq_q2_K_aligned_moe_vec(
                         down_aligned, (const float *)mid->ptr,
@@ -24211,12 +31171,52 @@ static int routed_moe_launch(
                         (int)assignments, (int)n_total_expert,
                         /*n_expert_used=*/1,
                         aligned_stream);
+                    if (rc == 0 && dspark_tiny_aligned_vec) {
+                        cuda_moe_fast_profile_mark(&aligned_profile, 2u);
+                    }
                 }
-            } else {
-                rc = 1;
+                if (rc == 0 && dspark_tiny_aligned_vec) {
+                    static int logged_dspark_tiny_aligned_vec = 0;
+                    if (!logged_dspark_tiny_aligned_vec) {
+                        logged_dspark_tiny_aligned_vec = 1;
+                        fprintf(stderr,
+                                "ds4: CUDA DSpark tiny batches using "
+                                "aligned vector MoE\n");
+                    }
+                }
+            }
+            /* A successful tiny-vector run is final. For every remaining
+             * multi-token shape, keep the exact direct-prefill scratch sizing.
+             * Large prefill stays device-generic; GB10 also uses the direct
+             * producer for smaller speculative batches. */
+            if (n_tokens > 1u &&
+                (!dspark_tiny_aligned_vec || rc != 0)) {
+                rc = DS4_MMQ_NOT_APPLICABLE;
                 const uint64_t assignments =
                     (uint64_t)n_tokens * n_expert;
-                if (assignments >= 1024u) {
+                const int direct_tier = ds4_tensor_device_idx(out);
+                const int direct_gb10 =
+                    direct_tier >= 0 && direct_tier < DS4_MAX_GPUS &&
+                    g_cuda_is_gb10[direct_tier];
+                const int direct_shape_fits =
+                    n_tokens <= INT_MAX && n_total_expert <= INT_MAX &&
+                    n_expert <= INT_MAX && expert_in_dim <= INT_MAX &&
+                    expert_mid_dim <= INT_MAX && out_dim <= INT_MAX;
+                const int direct_applicable =
+                    g_cuda_direct_q2_prefill && direct_shape_fits &&
+                    (assignments >= 1024u || direct_gb10);
+                /* A failed tiny-vector experiment may enter this path. Its
+                 * partial profile must not leak into the independent fused
+                 * attempt or the legacy fallback. */
+                if (aligned_profile.event_count > 0u) {
+                    cuda_moe_fast_profile_destroy(&aligned_profile);
+                }
+                aligned_profile_path = direct_applicable
+                    ? "iq2_aligned_direct_d2r"
+                    : "iq2_aligned_soa";
+                cuda_moe_fast_profile_begin(
+                    &aligned_profile, aligned_stream, 3u);
+                if (direct_applicable) {
                     size_t input_q8_bytes = 0;
                     size_t down_q8_bytes = 0;
                     size_t work_bytes = 0;
@@ -24252,7 +31252,17 @@ static int routed_moe_launch(
                         }
                     }
                 }
-                if (rc != 0) {
+                /* Only a pre-enqueue NOT_APPLICABLE result may retry the
+                 * materialized SoA path. Other failures may follow enqueue. */
+                if (rc == DS4_MMQ_NOT_APPLICABLE) {
+                    aligned_profile_path = "iq2_aligned_soa";
+                    if (direct_applicable) {
+                        if (aligned_profile.event_count > 0u) {
+                            cuda_moe_fast_profile_destroy(&aligned_profile);
+                        }
+                        cuda_moe_fast_profile_begin(
+                            &aligned_profile, aligned_stream, 3u);
+                    }
                     rc = ds4_mmq_iq2_xxs_q2_K_moe_fused_soa(
                         gate_aligned, up_aligned, down_aligned,
                         (const float *)x->ptr,
@@ -24265,6 +31275,9 @@ static int routed_moe_launch(
                         (int)n_total_expert, (int)n_expert,
                         clamp, aligned_stream);
                 }
+                if (rc == 0) {
+                    cuda_moe_fast_profile_mark(&aligned_profile, 1u);
+                }
             }
             if (rc == 0) {
                 const uint64_t n = (uint64_t)n_tokens * out_dim;
@@ -24274,6 +31287,24 @@ static int routed_moe_launch(
                     NULL, out_dim, n_expert, n_tokens,
                     /*guard_nonfinite=*/1);
                 if (cuda_ok(cudaGetLastError(), "aligned moe sum launch")) {
+                    if (n_tokens > 1u &&
+                        aligned_profile.event_count > 0u) {
+                        static const char *const vector_stage_names[] = {
+                            "iq2_gateup_swiglu", "q2_down", "sum"
+                        };
+                        static const char *const fused_stage_names[] = {
+                            "fused_iq2_gateup_swiglu_q2_down", "sum"
+                        };
+                        const uint32_t aligned_profile_events =
+                            aligned_profile.event_count;
+                        cuda_moe_fast_profile_mark(
+                            &aligned_profile, aligned_profile_events - 1u);
+                        cuda_moe_fast_profile_report(
+                            &aligned_profile, aligned_profile_path,
+                            n_tokens, (uint64_t)n_tokens * n_expert,
+                            aligned_profile_events == 4u
+                                ? vector_stage_names : fused_stage_names);
+                    }
                     static int logged = 0;
                     if (!logged) {
                         logged = 1;
@@ -24284,6 +31315,9 @@ static int routed_moe_launch(
                 }
                 rc = -1;
             }
+            if (n_tokens > 1u && aligned_profile.event_count > 0u) {
+                cuda_moe_fast_profile_destroy(&aligned_profile);
+            }
             fprintf(stderr,
                     "ds4: aligned routed-MoE returned %d "
                     "(layer=%u n_tokens=%u)\n",
@@ -24292,6 +31326,62 @@ static int routed_moe_launch(
         }
     }
 
+    /* Resolve one immutable raw-weight binding before choosing MXFP4, the
+     * IQ2 grouped tier, or the established scratch pipeline.  SSD loads
+     * publish compact weights plus an ID remap; resident models retain their
+     * full domain.  The public routed wrapper holds the matching reader lease
+     * for every return below. */
+    if (gate_expert_bytes == 0 || down_expert_bytes == 0 ||
+        (uint64_t)n_total_expert >
+            UINT64_MAX / gate_expert_bytes ||
+        (uint64_t)n_total_expert >
+            UINT64_MAX / down_expert_bytes) {
+        return 0;
+    }
+    const uint64_t gate_bytes =
+        (uint64_t)n_total_expert * gate_expert_bytes;
+    const uint64_t down_bytes =
+        (uint64_t)n_total_expert * down_expert_bytes;
+    if (gate_bytes > model_size - gate_offset ||
+        gate_bytes > model_size - up_offset ||
+        down_bytes > model_size - down_offset) {
+        return 0;
+    }
+    const uint64_t required_slot_count = (uint64_t)n_tokens * n_expert;
+    const int logical_tier = ds4_tensor_device_idx(out);
+    cuda_stream_selected_binding stream_binding = {};
+    const int use_stream_selected_cache =
+        allow_streaming && g_ssd_streaming_mode &&
+        cuda_stream_selected_binding_acquire(
+            &stream_binding, logical_tier, model_map, layer_index,
+            n_total_expert, gate_offset, up_offset, down_offset,
+            gate_expert_bytes, down_expert_bytes, required_slot_count);
+    if (g_ssd_streaming_mode && allow_streaming &&
+        !use_stream_selected_cache) {
+        fprintf(stderr,
+                "ds4: CUDA streaming selected experts are unavailable for "
+                "layer %u\n",
+                layer_index);
+        return 0;
+    }
+    if (use_stream_selected_cache) selected = stream_binding.selected;
+    const char *gate_w = use_stream_selected_cache
+        ? stream_binding.gate
+        : cuda_resolve_weight_ptr(model_map, gate_offset, gate_bytes,
+                                  logical_tier, "moe_gate");
+    const char *up_w = use_stream_selected_cache
+        ? stream_binding.up
+        : cuda_resolve_weight_ptr(model_map, up_offset, gate_bytes,
+                                  logical_tier, "moe_up");
+    const char *down_w = use_stream_selected_cache
+        ? stream_binding.down
+        : cuda_resolve_weight_ptr(model_map, down_offset, down_bytes,
+                                  logical_tier, "moe_down");
+    if (!gate_w || !up_w || !down_w) return 0;
+    const uint32_t raw_weight_domain = use_stream_selected_cache ?
+        stream_binding.weight_domain : n_total_expert;
+    if (raw_weight_domain == 0u || raw_weight_domain > INT_MAX) return 0;
+
     /* Native MXFP4 routed experts use the vendored MMVQ decode kernels and
      * MMQ matrix kernels. On Blackwell the latter dispatch to FP4 MMA; older
      * CUDA devices use the mathematically equivalent DP4A implementation.
@@ -24302,63 +31392,16 @@ static int routed_moe_launch(
             fprintf(stderr, "ds4: CUDA MXFP4 requires the MMQ backend\n");
             return 0;
         }
-        const uint64_t gate_total =
-            (uint64_t)n_total_expert * gate_expert_bytes;
-        const uint64_t down_total =
-            (uint64_t)n_total_expert * down_expert_bytes;
-        if (gate_total > model_size - gate_offset ||
-            gate_total > model_size - up_offset ||
-            down_total > model_size - down_offset) {
-            return 0;
-        }
-
-        const uint64_t slot_count = (uint64_t)n_tokens * n_expert;
-        const int logical_tier = ds4_tensor_device_idx(out);
-        const int use_stream_selected_cache =
-            allow_streaming &&
-            g_ssd_streaming_mode &&
-            g_stream_selected_cache.valid &&
-            g_stream_selected_cache.logical_tier == logical_tier &&
-            g_stream_selected_cache.model_map == model_map &&
-            g_stream_selected_cache.layer == layer_index &&
-            g_stream_selected_cache.n_total_expert == n_total_expert &&
-            g_stream_selected_cache.slot_count >= slot_count &&
-            g_stream_selected_cache.gate_offset == gate_offset &&
-            g_stream_selected_cache.up_offset == up_offset &&
-            g_stream_selected_cache.down_offset == down_offset &&
-            g_stream_selected_cache.gate_expert_bytes == gate_expert_bytes &&
-            g_stream_selected_cache.down_expert_bytes == down_expert_bytes &&
-            g_stream_selected_cache.gate_ptr &&
-            g_stream_selected_cache.up_ptr &&
-            g_stream_selected_cache.down_ptr &&
-            g_stream_selected_cache.slot_selected_tensor.ptr &&
-            g_stream_selected_cache.slot_selected_tensor.bytes >=
-                slot_count * sizeof(int32_t);
-        if (g_ssd_streaming_mode && allow_streaming &&
-            !use_stream_selected_cache) {
-            fprintf(stderr,
-                    "ds4: CUDA streaming MXFP4 experts are unavailable for layer %u\n",
-                    layer_index);
+        /* MMQ lazy initialization may leave this host thread current on a
+         * different physical device.  Do not trust the logical-device cache
+         * after it; select the owner explicitly before any launch. */
+        if (logical_tier < 0 || logical_tier >= g_n_gpus ||
+            cudaSetDevice(g_gpu[logical_tier].device_id) != cudaSuccess) {
+            (void)cudaGetLastError();
             return 0;
         }
-
-        const ds4_gpu_tensor *mx_selected = use_stream_selected_cache ?
-            &g_stream_selected_cache.slot_selected_tensor : selected;
-        const uint32_t weight_experts = use_stream_selected_cache ?
-            g_stream_selected_cache.compact_count : n_total_expert;
-        const char *gate_w = use_stream_selected_cache ?
-            g_stream_selected_cache.gate_ptr :
-            cuda_resolve_weight_ptr(model_map, gate_offset, gate_total,
-                                    logical_tier, "mxfp4 moe gate");
-        const char *up_w = use_stream_selected_cache ?
-            g_stream_selected_cache.up_ptr :
-            cuda_resolve_weight_ptr(model_map, up_offset, gate_total,
-                                    logical_tier, "mxfp4 moe up");
-        const char *down_w = use_stream_selected_cache ?
-            g_stream_selected_cache.down_ptr :
-            cuda_resolve_weight_ptr(model_map, down_offset, down_total,
-                                    logical_tier, "mxfp4 moe down");
-        if (!gate_w || !up_w || !down_w || weight_experts == 0u) return 0;
+        const ds4_gpu_tensor *mx_selected = selected;
+        const uint32_t weight_experts = raw_weight_domain;
 
         const cudaStream_t stream =
             n_tokens == 1u ? cuda_decode_stream() : (cudaStream_t)0;
@@ -24389,7 +31432,7 @@ static int routed_moe_launch(
                 stream);
             if (rc == 0) {
                 const uint64_t mid_floats =
-                    slot_count * expert_mid_dim;
+                    required_slot_count * expert_mid_dim;
                 moe_mmq_swiglu_weighted_clamp_kernel<<<
                     (uint32_t)((mid_floats + 255u) / 256u), 256, 0, stream>>>(
                     (float *)mid->ptr,
@@ -24405,7 +31448,7 @@ static int routed_moe_launch(
                     (const int32_t *)mx_selected->ptr,
                     (float *)down->ptr,
                     (int)out_dim, (int)expert_mid_dim,
-                    (int)slot_count, (int)weight_experts,
+                    (int)required_slot_count, (int)weight_experts,
                     /*n_expert_used=*/1, stream);
             }
             if (rc == 0) {
@@ -24428,6 +31471,202 @@ static int routed_moe_launch(
                 rc, layer_index, n_tokens);
         return 0;
     }
+
+    /* Opt-in GB10 grouped SSD prefill.  The raw fused entry builds routing
+     * once for gate/up/down over the compact weight domain.  Only its explicit
+     * pre-enqueue NOT_APPLICABLE result may enter the legacy scratch path;
+     * every other error is fenced and fails closed. */
+    const int grouped_enabled = cuda_iq2_ssd_grouped_enabled();
+    const int grouped_required = cuda_iq2_ssd_grouped_required();
+    const int grouped_raw_layout = cuda_iq2_ssd_grouped_raw_layout(
+        gate_type, down_type,
+        gate_expert_bytes, gate_row_bytes,
+        down_expert_bytes, down_row_bytes,
+        expert_in_dim, expert_mid_dim, out_dim);
+    const int grouped_candidate_domain =
+        cuda_iq2_ssd_grouped_candidate_values(
+            iq2_path, g_ssd_streaming_mode, allow_streaming,
+            owned_filtered, n_tokens, n_expert,
+            stream_binding.top6_unique, grouped_raw_layout,
+            use_stream_selected_cache);
+    if (grouped_candidate_domain &&
+        (grouped_enabled || grouped_required)) {
+        g_iq2_ssd_grouped_candidates.fetch_add(
+            1, std::memory_order_relaxed);
+        const cudaStream_t grouped_stream = (cudaStream_t)0;
+        cudaStreamCaptureStatus capture = cudaStreamCaptureStatusNone;
+        const cudaError_t capture_err =
+            cudaStreamIsCapturing(grouped_stream, &capture);
+        const int capture_active = capture_err != cudaSuccess ||
+            capture != cudaStreamCaptureStatusNone;
+        if (capture_err != cudaSuccess) (void)cudaGetLastError();
+        const int tensor_devices_ok = logical_tier == 0 &&
+            ds4_tensor_device_idx(gate) == logical_tier &&
+            ds4_tensor_device_idx(up) == logical_tier &&
+            ds4_tensor_device_idx(mid) == logical_tier &&
+            ds4_tensor_device_idx(down) == logical_tier &&
+            ds4_tensor_device_idx(x) == logical_tier &&
+            ds4_tensor_device_idx(weights) == logical_tier &&
+            stream_binding.selected &&
+            ds4_tensor_device_idx(stream_binding.selected) == logical_tier;
+        /* ds4_mmq_init() may change the thread's physical CUDA device.  It is
+         * part of preflight, and the owner is selected again before waiting
+         * on the upload token or enqueueing the fused pipeline. */
+        const int mmq_ready = cuda_use_mmq();
+        const int eligible = cuda_iq2_ssd_grouped_eligible_values(
+            grouped_enabled,
+            g_ssd_streaming_mode,
+            g_n_gpus == 1 && logical_tier == 0,
+            logical_tier >= 0 && logical_tier < DS4_MAX_GPUS &&
+                g_cuda_is_gb10[logical_tier],
+            g_quality_mode,
+            owned_filtered,
+            capture_active,
+            mmq_ready,
+            n_tokens,
+            n_expert,
+            stream_binding.top6_unique,
+            grouped_raw_layout,
+            use_stream_selected_cache && tensor_devices_ok &&
+                stream_binding.weight_domain > 0u &&
+                stream_binding.weight_domain <= INT_MAX &&
+                n_tokens <= INT_MAX && n_expert <= INT_MAX &&
+                expert_in_dim <= INT_MAX && expert_mid_dim <= INT_MAX &&
+                out_dim <= INT_MAX &&
+                ((uint64_t)n_tokens * out_dim + 255u) / 256u <=
+                    UINT32_MAX);
+        if (!eligible) {
+            if (grouped_required) {
+                g_iq2_ssd_grouped_failures.fetch_add(
+                    1, std::memory_order_relaxed);
+                g_iq2_ssd_grouped_required_failures.fetch_add(
+                    1, std::memory_order_relaxed);
+                fprintf(stderr,
+                        "ds4: required CUDA IQ2 SSD grouped MMQ is not "
+                        "eligible at layer %u (n_tokens=%u)\n",
+                        layer_index, n_tokens);
+                return 0;
+            }
+            g_iq2_ssd_grouped_safe_fallbacks.fetch_add(
+                1, std::memory_order_relaxed);
+        } else {
+            g_iq2_ssd_grouped_eligible.fetch_add(
+                1, std::memory_order_relaxed);
+            if (cudaSetDevice(g_gpu[0].device_id) != cudaSuccess) {
+                (void)cudaGetLastError();
+                g_iq2_ssd_grouped_failures.fetch_add(
+                    1, std::memory_order_relaxed);
+                if (grouped_required) {
+                    g_iq2_ssd_grouped_required_failures.fetch_add(
+                        1, std::memory_order_relaxed);
+                }
+                return 0;
+            }
+            if (stream_binding.upload_event_value != 0) {
+                if (!cuda_stream_selected_wait_upload_on(
+                        stream_binding.upload_event_value,
+                        grouped_stream,
+                        "IQ2 SSD grouped-MMQ upload wait")) {
+                    g_iq2_ssd_grouped_failures.fetch_add(
+                        1, std::memory_order_relaxed);
+                    if (grouped_required) {
+                        g_iq2_ssd_grouped_required_failures.fetch_add(
+                            1, std::memory_order_relaxed);
+                    }
+                    return 0;
+                }
+                g_iq2_ssd_grouped_upload_waits.fetch_add(
+                    1, std::memory_order_relaxed);
+            }
+            if (!g_stream_selected_cache.valid ||
+                g_stream_selected_cache.generation !=
+                    stream_binding.generation ||
+                g_stream_selected_cache.upload_event_value !=
+                    stream_binding.upload_event_value) {
+                g_iq2_ssd_grouped_failures.fetch_add(
+                    1, std::memory_order_relaxed);
+                if (grouped_required) {
+                    g_iq2_ssd_grouped_required_failures.fetch_add(
+                        1, std::memory_order_relaxed);
+                }
+                return 0;
+            }
+            g_iq2_ssd_grouped_attempts.fetch_add(
+                1, std::memory_order_relaxed);
+            cuda_moe_fast_profile grouped_profile = {};
+            cuda_moe_fast_profile_begin(
+                &grouped_profile, grouped_stream, 3u);
+            int rc = ds4_mmq_iq2_xxs_q2_K_moe_fused_raw(
+                gate_w, up_w, down_w,
+                (const float *)x->ptr,
+                (const int32_t *)selected->ptr,
+                (const float *)weights->ptr,
+                (float *)gate->ptr, (float *)up->ptr,
+                (float *)mid->ptr, (float *)down->ptr,
+                (int)expert_mid_dim, (int)expert_in_dim, (int)out_dim,
+                (int)n_tokens, (int)stream_binding.weight_domain,
+                (int)n_expert, clamp, grouped_stream);
+            if (rc == DS4_MMQ_NOT_APPLICABLE) {
+                cuda_moe_fast_profile_destroy(&grouped_profile);
+                g_iq2_ssd_grouped_not_applicable.fetch_add(
+                    1, std::memory_order_relaxed);
+                if (grouped_required) {
+                    g_iq2_ssd_grouped_failures.fetch_add(
+                        1, std::memory_order_relaxed);
+                    g_iq2_ssd_grouped_required_failures.fetch_add(
+                        1, std::memory_order_relaxed);
+                    fprintf(stderr,
+                            "ds4: required CUDA IQ2 SSD grouped MMQ was "
+                            "not applicable at layer %u\n",
+                            layer_index);
+                    return 0;
+                }
+                g_iq2_ssd_grouped_safe_fallbacks.fetch_add(
+                    1, std::memory_order_relaxed);
+            } else {
+                if (rc == 0) {
+                    cuda_moe_fast_profile_mark(&grouped_profile, 1u);
+                    const uint64_t n = (uint64_t)n_tokens * out_dim;
+                    moe_mmq_sum_kernel<<<
+                        (uint32_t)((n + 255u) / 256u), 256, 0,
+                        grouped_stream>>>(
+                        (float *)out->ptr, (const float *)down->ptr,
+                        NULL, out_dim, n_expert, n_tokens,
+                        /*guard_nonfinite=*/1);
+                    rc = cuda_ok(cudaGetLastError(),
+                                 "IQ2 SSD grouped moe sum launch") ? 0 : -1;
+                    if (rc == 0) {
+                        static const char *const grouped_stage_names[] = {
+                            "fused_iq2_gateup_swiglu_q2_down", "sum"
+                        };
+                        cuda_moe_fast_profile_mark(&grouped_profile, 2u);
+                        cuda_moe_fast_profile_report(
+                            &grouped_profile, "iq2_ssd_grouped_raw",
+                            n_tokens, (uint64_t)n_tokens * n_expert,
+                            grouped_stage_names);
+                    }
+                }
+                if (rc != 0) {
+                    cuda_moe_fast_profile_destroy(&grouped_profile);
+                    g_iq2_ssd_grouped_failures.fetch_add(
+                        1, std::memory_order_relaxed);
+                    if (grouped_required) {
+                        g_iq2_ssd_grouped_required_failures.fetch_add(
+                            1, std::memory_order_relaxed);
+                    }
+                    fprintf(stderr,
+                            "ds4: CUDA IQ2 SSD grouped MMQ failed closed "
+                            "with rc=%d at layer %u\n",
+                            rc, layer_index);
+                    return 0;
+                }
+                g_iq2_ssd_grouped_completed.fetch_add(
+                    1, std::memory_order_relaxed);
+                return 1;
+            }
+        }
+    }
+
     /* mmq routed-MoE prefill tier (ported from the Entrpi/ds4 fork).
      * IQ2_XXS gate/up pair (one shared activation quantize + routing
      * pass) -> SwiGLU + clamp + router weight -> Q2_K down, treating
@@ -24436,15 +31675,18 @@ static int routed_moe_launch(
      * [n_tokens, n_expert, *] by the validation above.  Any entry
      * failure falls through to the legacy sorted-pairs path (the
      * buffers are scratch there too). */
-    if (iq2_path && n_tokens > 1u && !owned_filtered && cuda_use_mmq()) {
-        const uint64_t gate_total = (uint64_t)n_total_expert * gate_expert_bytes;
-        const uint64_t down_total = (uint64_t)n_total_expert * down_expert_bytes;
-        const int mmq_tier = ds4_tensor_device_idx(out);
-        const char *gate_w = cuda_resolve_weight_ptr(model_map, gate_offset, gate_total, mmq_tier, "moe gate mmq");
-        const char *up_w = gate_w ? cuda_resolve_weight_ptr(model_map, up_offset, gate_total, mmq_tier, "moe up mmq") : NULL;
-        const char *down_w = up_w ? cuda_resolve_weight_ptr(model_map, down_offset, down_total, mmq_tier, "moe down mmq") : NULL;
+    if (iq2_path && n_tokens > 1u && !owned_filtered && cuda_use_mmq() &&
+        !(g_ssd_streaming_mode && allow_streaming)) {
+        if (logical_tier < 0 || logical_tier >= g_n_gpus ||
+            cudaSetDevice(g_gpu[logical_tier].device_id) != cudaSuccess) {
+            (void)cudaGetLastError();
+            return 0;
+        }
         if (down_w) {
             const uint64_t n_assignments = (uint64_t)n_tokens * n_expert;
+            cuda_moe_fast_profile resident_profile = {};
+            cuda_moe_fast_profile_begin(
+                &resident_profile, (cudaStream_t)0, 5u);
             int rc = ds4_mmq_iq2_xxs_moe_pair(
                     gate_w, up_w, (const float *)x->ptr,
                     (const int32_t *)selected->ptr,
@@ -24453,6 +31695,7 @@ static int routed_moe_launch(
                     (int)n_tokens, (int)n_total_expert, (int)n_expert,
                     (cudaStream_t)0);
             if (rc == 0) {
+                cuda_moe_fast_profile_mark(&resident_profile, 1u);
                 const uint64_t mid_floats = n_assignments * expert_mid_dim;
                 moe_mmq_swiglu_weighted_clamp_kernel<<<(uint32_t)((mid_floats + 255) / 256), 256>>>(
                         (float *)mid->ptr,
@@ -24460,6 +31703,9 @@ static int routed_moe_launch(
                         (const float *)weights->ptr,
                         expert_mid_dim, n_tokens, n_expert, clamp);
                 rc = cuda_ok(cudaGetLastError(), "mmq moe swiglu launch") ? 0 : -1;
+                if (rc == 0) {
+                    cuda_moe_fast_profile_mark(&resident_profile, 2u);
+                }
             }
             if (rc == 0) {
                 rc = ds4_mmq_q2_K_moe(
@@ -24470,6 +31716,9 @@ static int routed_moe_launch(
                         (int)n_assignments, (int)n_total_expert,
                         /*n_expert_used=*/1,
                         (cudaStream_t)0);
+                if (rc == 0) {
+                    cuda_moe_fast_profile_mark(&resident_profile, 3u);
+                }
             }
             if (rc == 0) {
                 const uint64_t n = (uint64_t)n_tokens * out_dim;
@@ -24477,9 +31726,19 @@ static int routed_moe_launch(
                         (float *)out->ptr, (const float *)down->ptr,
                         NULL, out_dim, n_expert, n_tokens,
                         /*guard_nonfinite=*/1);
-                if (cuda_ok(cudaGetLastError(), "mmq moe sum launch")) return 1;
+                if (cuda_ok(cudaGetLastError(), "mmq moe sum launch")) {
+                    static const char *const resident_stage_names[] = {
+                        "iq2_pair", "swiglu", "q2_down", "sum"
+                    };
+                    cuda_moe_fast_profile_mark(&resident_profile, 4u);
+                    cuda_moe_fast_profile_report(
+                        &resident_profile, "iq2_mmq_resident",
+                        n_tokens, n_assignments, resident_stage_names);
+                    return 1;
+                }
                 rc = -1;
             }
+            cuda_moe_fast_profile_destroy(&resident_profile);
             fprintf(stderr, "ds4: mmq routed-MoE tier rc=%d (layer=%u n_tokens=%u); falling back\n",
                     rc, layer_index, n_tokens);
         }
@@ -24499,59 +31758,6 @@ static int routed_moe_launch(
      *                  pairs by expert and uses Q4_K tile8 gate/up + down kernels
      *                  (`DS4_CUDA_MOE_NO_Q4_SORTED=1` restores the older
      *                  token-indexed decode-style prefill kernels). */
-    const uint64_t gate_bytes = (uint64_t)n_total_expert * gate_expert_bytes;
-    const uint64_t down_bytes = (uint64_t)n_total_expert * down_expert_bytes;
-    if (gate_bytes > model_size - gate_offset ||
-        gate_bytes > model_size - up_offset ||
-        down_bytes > model_size - down_offset) {
-        return 0;
-    }
-    const uint64_t required_slot_count = (uint64_t)n_tokens * n_expert;
-    const int logical_tier = ds4_tensor_device_idx(out);
-    const int use_stream_selected_cache =
-        allow_streaming &&
-        g_ssd_streaming_mode &&
-        g_stream_selected_cache.valid &&
-        g_stream_selected_cache.logical_tier == logical_tier &&
-        g_stream_selected_cache.model_map == model_map &&
-        g_stream_selected_cache.layer == layer_index &&
-        g_stream_selected_cache.n_total_expert == n_total_expert &&
-        g_stream_selected_cache.slot_count >= required_slot_count &&
-        g_stream_selected_cache.gate_offset == gate_offset &&
-        g_stream_selected_cache.up_offset == up_offset &&
-        g_stream_selected_cache.down_offset == down_offset &&
-        g_stream_selected_cache.gate_expert_bytes == gate_expert_bytes &&
-        g_stream_selected_cache.down_expert_bytes == down_expert_bytes &&
-        g_stream_selected_cache.gate_ptr &&
-        g_stream_selected_cache.up_ptr &&
-        g_stream_selected_cache.down_ptr &&
-        g_stream_selected_cache.slot_selected_tensor.ptr &&
-        g_stream_selected_cache.slot_selected_tensor.bytes >=
-            required_slot_count * sizeof(int32_t);
-    if (g_ssd_streaming_mode && allow_streaming &&
-        !use_stream_selected_cache) {
-        fprintf(stderr,
-                "ds4: CUDA streaming selected experts are unavailable for layer %u\n",
-                layer_index);
-        return 0;
-    }
-    if (use_stream_selected_cache) {
-        selected = &g_stream_selected_cache.slot_selected_tensor;
-    }
-    const char *gate_w = use_stream_selected_cache ?
-        g_stream_selected_cache.gate_ptr :
-        cuda_resolve_weight_ptr(model_map, gate_offset, gate_bytes,
-                                logical_tier, "moe_gate");
-    const char *up_w = use_stream_selected_cache ?
-        g_stream_selected_cache.up_ptr :
-        cuda_resolve_weight_ptr(model_map, up_offset, gate_bytes,
-                                logical_tier, "moe_up");
-    const char *down_w = use_stream_selected_cache ?
-        g_stream_selected_cache.down_ptr :
-        cuda_resolve_weight_ptr(model_map, down_offset, down_bytes,
-                                logical_tier, "moe_down");
-    if (!gate_w || !up_w || !down_w) return 0;
-
     int ok = 1;
     const uint32_t xq_blocks = expert_in_dim / CUDA_QK_K;
     const uint32_t midq_blocks = expert_mid_dim / CUDA_QK_K;
@@ -24622,7 +31828,7 @@ static int routed_moe_launch(
             n_tokens >= 128u && getenv("DS4_CUDA_MOE_NO_DOWN_TILE16") == NULL;
         const uint32_t use_small_sorted_prep =
             owned_filtered && q4k_path && n_tokens <= 16u && pair_count <= 96u &&
-            n_total_expert <= 128u && use_sorted_pairs && use_expert_tiles &&
+            raw_weight_domain <= 128u && use_sorted_pairs && use_expert_tiles &&
             getenv("DS4_CUDA_MOE_NO_SMALL_SORTED_PREP") == NULL;
         const uint32_t force_q4_down_rowspan =
             getenv("DS4_CUDA_MOE_DOWN_ROW512") != NULL ||
@@ -24744,17 +31950,17 @@ static int routed_moe_launch(
         ok = cuda_ok(cudaGetLastError(), "routed_moe x quantize launch");
         if (prof_ev[1]) (void)cudaEventRecord(prof_ev[1], 0);
         if (ok && use_sorted_pairs) {
-            const uint64_t counts_bytes = (uint64_t)n_total_expert * sizeof(uint32_t);
-            const uint64_t offsets_bytes = ((uint64_t)n_total_expert + 1ull) * sizeof(uint32_t);
-            const uint64_t cursors_bytes = (uint64_t)n_total_expert * sizeof(uint32_t);
+            const uint64_t counts_bytes = (uint64_t)raw_weight_domain * sizeof(uint32_t);
+            const uint64_t offsets_bytes = ((uint64_t)raw_weight_domain + 1ull) * sizeof(uint32_t);
+            const uint64_t cursors_bytes = (uint64_t)raw_weight_domain * sizeof(uint32_t);
             const uint64_t sorted_bytes = (uint64_t)pair_count * sizeof(uint32_t);
-            tile_capacity = (pair_count + expert_tile_m - 1u) / expert_tile_m + n_total_expert;
-            tile16_capacity = (use_down_tile16 || use_q4_mma_tiles16) ? ((pair_count + 15u) / 16u + n_total_expert) : 0u;
-            const uint64_t tile_offsets_bytes = ((uint64_t)n_total_expert + 1ull) * sizeof(uint32_t);
+            tile_capacity = (pair_count + expert_tile_m - 1u) / expert_tile_m + raw_weight_domain;
+            tile16_capacity = (use_down_tile16 || use_q4_mma_tiles16) ? ((pair_count + 15u) / 16u + raw_weight_domain) : 0u;
+            const uint64_t tile_offsets_bytes = ((uint64_t)raw_weight_domain + 1ull) * sizeof(uint32_t);
             const uint64_t tile_total_bytes = sizeof(uint32_t);
             const uint64_t tile_experts_bytes = (uint64_t)tile_capacity * sizeof(uint32_t);
             const uint64_t tile_starts_bytes = (uint64_t)tile_capacity * sizeof(uint32_t);
-            const uint64_t tile16_offsets_bytes = (use_down_tile16 || use_q4_mma_tiles16) ? (((uint64_t)n_total_expert + 1ull) * sizeof(uint32_t)) : 0u;
+            const uint64_t tile16_offsets_bytes = (use_down_tile16 || use_q4_mma_tiles16) ? (((uint64_t)raw_weight_domain + 1ull) * sizeof(uint32_t)) : 0u;
             const uint64_t tile16_total_bytes = (use_down_tile16 || use_q4_mma_tiles16) ? sizeof(uint32_t) : 0u;
             const uint64_t tile16_experts_bytes = (uint64_t)tile16_capacity * sizeof(uint32_t);
             const uint64_t tile16_starts_bytes = (uint64_t)tile16_capacity * sizeof(uint32_t);
@@ -24791,7 +31997,7 @@ static int routed_moe_launch(
                         counts, offsets, cursors, sorted_pairs,
                         tile_offsets, tile_total, tile_experts, tile_starts,
                         tile16_offsets, tile16_total, tile16_experts, tile16_starts,
-                        (const int32_t *)selected->ptr, pair_count, n_total_expert,
+                        (const int32_t *)selected->ptr, pair_count, raw_weight_domain,
                         expert_tile_m, use_down_tile16 || use_q4_mma_tiles16);
                     ok = cuda_ok(cudaGetLastError(),
                                  "routed_moe small sorted setup launch");
@@ -24804,11 +32010,11 @@ static int routed_moe_launch(
                         counts,
                         (const int32_t *)selected->ptr,
                         pair_count,
-                        n_total_expert);
+                        raw_weight_domain);
                     ok = cuda_ok(cudaGetLastError(), "routed_moe sorted count launch");
                 }
                 if (ok && !use_small_sorted_prep) {
-                    moe_prefix_sorted_pairs_kernel<<<1, 1, 0, cuda_decode_stream()>>>(offsets, cursors, counts, n_total_expert);
+                    moe_prefix_sorted_pairs_kernel<<<1, 1, 0, cuda_decode_stream()>>>(offsets, cursors, counts, raw_weight_domain);
                     ok = cuda_ok(cudaGetLastError(), "routed_moe sorted prefix launch");
                 }
                 if (ok && !use_small_sorted_prep) {
@@ -24817,27 +32023,27 @@ static int routed_moe_launch(
                         cursors,
                         (const int32_t *)selected->ptr,
                         pair_count,
-                        n_total_expert);
+                        raw_weight_domain);
                     ok = cuda_ok(cudaGetLastError(), "routed_moe sorted scatter launch");
                 }
                 if (ok && use_expert_tiles && !use_small_sorted_prep) {
-                    moe_build_expert_tile_offsets_kernel<<<1, 1, 0, cuda_decode_stream()>>>(tile_offsets, tile_total, counts, expert_tile_m, n_total_expert);
+                    moe_build_expert_tile_offsets_kernel<<<1, 1, 0, cuda_decode_stream()>>>(tile_offsets, tile_total, counts, expert_tile_m, raw_weight_domain);
                     ok = cuda_ok(cudaGetLastError(), "routed_moe expert tile offsets launch");
                 }
                 if (ok && use_expert_tiles && !use_small_sorted_prep) {
-                    moe_build_expert_tiles_kernel<<<(n_total_expert + 255u) / 256u, 256, 0, cuda_decode_stream()>>>(
-                        tile_experts, tile_starts, tile_offsets, counts, expert_tile_m, n_total_expert);
+                    moe_build_expert_tiles_kernel<<<(raw_weight_domain + 255u) / 256u, 256, 0, cuda_decode_stream()>>>(
+                        tile_experts, tile_starts, tile_offsets, counts, expert_tile_m, raw_weight_domain);
                     ok = cuda_ok(cudaGetLastError(), "routed_moe expert tiles launch");
                 }
                 if (ok && use_expert_tiles && !use_small_sorted_prep &&
                     (use_down_tile16 || use_q4_mma_tiles16)) {
-                    moe_build_expert_tile_offsets_kernel<<<1, 1, 0, cuda_decode_stream()>>>(tile16_offsets, tile16_total, counts, 16u, n_total_expert);
+                    moe_build_expert_tile_offsets_kernel<<<1, 1, 0, cuda_decode_stream()>>>(tile16_offsets, tile16_total, counts, 16u, raw_weight_domain);
                     ok = cuda_ok(cudaGetLastError(), "routed_moe expert tile16 offsets launch");
                 }
                 if (ok && use_expert_tiles && !use_small_sorted_prep &&
                     (use_down_tile16 || use_q4_mma_tiles16)) {
-                    moe_build_expert_tiles_kernel<<<(n_total_expert + 255u) / 256u, 256, 0, cuda_decode_stream()>>>(
-                        tile16_experts, tile16_starts, tile16_offsets, counts, 16u, n_total_expert);
+                    moe_build_expert_tiles_kernel<<<(raw_weight_domain + 255u) / 256u, 256, 0, cuda_decode_stream()>>>(
+                        tile16_experts, tile16_starts, tile16_offsets, counts, 16u, raw_weight_domain);
                     ok = cuda_ok(cudaGetLastError(), "routed_moe expert tile16 launch");
                 }
             }
@@ -24862,7 +32068,7 @@ static int routed_moe_launch(
                         tile16_total && tile16_experts && tile16_starts &&
                         xq_blocks == 16u && cuda_q4_mma_tile16_shmem_ok(0);
                     if (use_q4_mma_t16 && use_gate_row2048) {
-                        const unsigned t16cap = (unsigned)((pair_count + 15u) / 16u + n_total_expert);
+                        const unsigned t16cap = (unsigned)((pair_count + 15u) / 16u + raw_weight_domain);
                         const size_t t16sh = 16u * 16u * sizeof(cuda_block_q8_K);
                         if (gate_row_span == 512u) {
                             dim3 tgrid((expert_mid_dim + 511u) / 512u, t16cap, 1);
@@ -25232,7 +32438,7 @@ static int routed_moe_launch(
                         expert_mid_dim,
                         n_tokens * n_expert,
                         0u,
-                        n_total_expert);
+                        raw_weight_domain);
                 ok = cuda_ok(cudaGetLastError(),
                              "owned routed_moe active mid quantize launch");
             } else {
@@ -25337,7 +32543,7 @@ static int routed_moe_launch(
                         tile16_total && tile16_experts && tile16_starts &&
                         midq_blocks <= 16u && cuda_q4_mma_tile16_shmem_ok(1);
                     if (use_q4_down_t16 && use_q4_down_rowspan) {
-                        const unsigned t16cap = (unsigned)((pair_count + 15u) / 16u + n_total_expert);
+                        const unsigned t16cap = (unsigned)((pair_count + 15u) / 16u + raw_weight_domain);
                         const size_t dt16sh = 16u * (size_t)midq_blocks * sizeof(cuda_block_q8_K);
                         if (down_row_span == 512u) {
                             dim3 tgrid((out_dim + 511u) / 512u, t16cap, 1);
@@ -25997,6 +33203,85 @@ extern "C" int ds4_gpu_routed_moe_owned_packed_combine_tensor(
                    "owned routed_moe packed combine launch");
 }
 
+/* Central lifetime guard for every transient SSD binding consumer.  It is
+ * deliberately outside routed_moe_launch(): all of that function's early
+ * success and error returns then pass through one consume-event publication,
+ * including Q4, MXFP4, grouped IQ2 and the legacy IQ2 fallback. */
+static int cuda_stream_selected_consumer_begin(
+        int allow_streaming, cudaStream_t stream,
+        uint64_t *generation_out) {
+    if (generation_out) *generation_out = 0;
+    if (!allow_streaming || !g_ssd_streaming_mode) {
+        return 1;
+    }
+    if (!generation_out || g_n_gpus != 1 ||
+        cudaSetDevice(g_gpu[0].device_id) != cudaSuccess ||
+        !cuda_stream_selected_consume_prepare()) {
+        (void)cudaGetLastError();
+        return 0;
+    }
+    uint64_t generation = 0;
+    uint64_t upload_event = 0;
+    {
+        std::unique_lock<std::mutex> lock(g_stream_selected_consume_mutex);
+        /* A single reusable event represents the whole consume frontier.
+         * Serialize host acquisitions, then chain this consumer behind the
+         * previous record before the event is reused on its stream. */
+        g_stream_selected_consume_cv.wait(lock, [] {
+            return !g_stream_selected_writer_active &&
+                   g_stream_selected_consume_host_readers == 0u;
+        });
+        if (!g_stream_selected_cache.valid ||
+            g_stream_selected_cache.generation == 0) {
+            return 0;
+        }
+        if (g_stream_selected_consume_pending) {
+            if (!g_stream_selected_consume_done_event ||
+                g_stream_selected_consume_owner_device !=
+                    g_gpu[0].device_id) {
+                return 0;
+            }
+            const cudaError_t consume_wait = cudaStreamWaitEvent(
+                stream, g_stream_selected_consume_done_event, 0);
+            if (consume_wait != cudaSuccess) {
+                fprintf(stderr,
+                        "ds4: CUDA compact-cache consumer-chain wait "
+                        "failed: %s\n",
+                        cudaGetErrorString(consume_wait));
+                (void)cudaGetLastError();
+                return 0;
+            }
+        }
+        generation = g_stream_selected_cache.generation;
+        upload_event = g_stream_selected_cache.upload_event_value;
+        g_stream_selected_consume_host_readers++;
+    }
+    if (!cuda_stream_selected_wait_upload_on(
+            upload_event, stream, "selected-cache consumer upload wait")) {
+        std::lock_guard<std::mutex> lock(g_stream_selected_consume_mutex);
+        g_stream_selected_consume_host_readers--;
+        g_stream_selected_consume_cv.notify_all();
+        return 0;
+    }
+    *generation_out = generation;
+    return 1;
+}
+
+static int cuda_stream_selected_consumer_end(
+        uint64_t generation, cudaStream_t stream) {
+    if (generation == 0) return 1;
+    const int recorded =
+        cuda_stream_selected_consume_record(generation, stream);
+    {
+        std::lock_guard<std::mutex> lock(g_stream_selected_consume_mutex);
+        if (g_stream_selected_consume_host_readers != 0u) {
+            g_stream_selected_consume_host_readers--;
+        }
+        g_stream_selected_consume_cv.notify_all();
+    }
+    return recorded;
+}
+
 extern "C" int ds4_gpu_routed_moe_one_tensor(ds4_gpu_tensor *out, ds4_gpu_tensor *gate, ds4_gpu_tensor *up, ds4_gpu_tensor *mid, ds4_gpu_tensor *down, const void *model_map, uint64_t model_size, uint64_t gate_offset, uint64_t up_offset, uint64_t down_offset, uint32_t gate_type, uint32_t down_type, uint64_t gate_expert_bytes, uint64_t gate_row_bytes, uint64_t down_expert_bytes, uint64_t down_row_bytes, uint32_t expert_in_dim, uint32_t expert_mid_dim, uint32_t out_dim, const ds4_gpu_tensor *selected, const ds4_gpu_tensor *weights, uint32_t n_total_expert, uint32_t n_expert, float clamp, const ds4_gpu_tensor *x,
         const ds4_gpu_tensor *add_in,
         uint32_t layer_index,
@@ -26005,26 +33290,46 @@ extern "C" int ds4_gpu_routed_moe_one_tensor(ds4_gpu_tensor *out, ds4_gpu_tensor
         if (!ds4_gpu_add_tensor(out, out, add_in,
                                 (uint32_t)(out->bytes / sizeof(float)))) return 0;
     }
-    return routed_moe_launch(out, gate, up, mid, down, model_map, model_size,
-                             gate_offset, up_offset, down_offset,
-                             gate_type, down_type,
-                             gate_expert_bytes, gate_row_bytes,
-                             down_expert_bytes, down_row_bytes,
-                             expert_in_dim, expert_mid_dim, out_dim,
-                             selected, weights, n_total_expert, n_expert, clamp, x,
-                             layer_index, 1, force_resident ? 0 : 1, 0);
+    const int allow_streaming = force_resident ? 0 : 1;
+    const cudaStream_t stream = cuda_decode_stream();
+    uint64_t consume_generation = 0;
+    if (!cuda_stream_selected_consumer_begin(
+            allow_streaming, stream, &consume_generation)) return 0;
+    const int rc = routed_moe_launch(
+        out, gate, up, mid, down, model_map, model_size,
+        gate_offset, up_offset, down_offset,
+        gate_type, down_type,
+        gate_expert_bytes, gate_row_bytes,
+        down_expert_bytes, down_row_bytes,
+        expert_in_dim, expert_mid_dim, out_dim,
+        selected, weights, n_total_expert, n_expert, clamp, x,
+        layer_index, 1, allow_streaming, 0);
+    if (!cuda_stream_selected_consumer_end(consume_generation, stream)) {
+        return 0;
+    }
+    return rc;
 }
 extern "C" int ds4_gpu_routed_moe_batch_tensor(ds4_gpu_tensor *out, ds4_gpu_tensor *gate, ds4_gpu_tensor *up, ds4_gpu_tensor *mid, ds4_gpu_tensor *down, const void *model_map, uint64_t model_size, uint64_t gate_offset, uint64_t up_offset, uint64_t down_offset, uint32_t gate_type, uint32_t down_type, uint64_t gate_expert_bytes, uint64_t gate_row_bytes, uint64_t down_expert_bytes, uint64_t down_row_bytes, uint32_t expert_in_dim, uint32_t expert_mid_dim, uint32_t out_dim, const ds4_gpu_tensor *selected, const ds4_gpu_tensor *weights, uint32_t n_total_expert, uint32_t n_expert, float clamp, const ds4_gpu_tensor *x, uint32_t layer_index, uint32_t n_tokens, bool *mid_is_f16, bool force_resident) {
-    (void)force_resident;
     if (mid_is_f16) *mid_is_f16 = false;
-    return routed_moe_launch(out, gate, up, mid, down, model_map, model_size,
-                             gate_offset, up_offset, down_offset,
-                             gate_type, down_type,
-                             gate_expert_bytes, gate_row_bytes,
-                             down_expert_bytes, down_row_bytes,
-                             expert_in_dim, expert_mid_dim, out_dim,
-                             selected, weights, n_total_expert, n_expert, clamp, x,
-                             layer_index, n_tokens, 1, 0);
+    const int allow_streaming = force_resident ? 0 : 1;
+    const cudaStream_t stream = n_tokens == 1u
+        ? cuda_decode_stream() : (cudaStream_t)0;
+    uint64_t consume_generation = 0;
+    if (!cuda_stream_selected_consumer_begin(
+            allow_streaming, stream, &consume_generation)) return 0;
+    const int rc = routed_moe_launch(
+        out, gate, up, mid, down, model_map, model_size,
+        gate_offset, up_offset, down_offset,
+        gate_type, down_type,
+        gate_expert_bytes, gate_row_bytes,
+        down_expert_bytes, down_row_bytes,
+        expert_in_dim, expert_mid_dim, out_dim,
+        selected, weights, n_total_expert, n_expert, clamp, x,
+        layer_index, n_tokens, allow_streaming, 0);
+    if (!cuda_stream_selected_consumer_end(consume_generation, stream)) {
+        return 0;
+    }
+    return rc;
 }
 
 extern "C" int ds4_gpu_routed_moe_batch_owned_tensor(
@@ -26220,37 +33525,111 @@ extern "C" int ds4_gpu_hc_split_weighted_sum_norm_tensor(
             (uint64_t)n_embd * sizeof(float) > model_size - norm_weight_offset) {
             return 0;
         }
-        uint64_t n_rows = out->bytes / out_row_bytes;
-        if (n_rows == 1) {
-            if (mix->bytes < n_rows * mix_bytes ||
-                split->bytes < n_rows * mix_bytes ||
-                residual_hc->bytes < n_rows * residual_row_bytes) {
-                return 0;
-            }
-            const int logical_tier = ds4_tensor_device_idx(out);
-            const float *scale = (const float *)cuda_resolve_weight_ptr(model_map, scale_offset,
-                    3ull * sizeof(float), logical_tier, "hc_scale");
-            const float *base = (const float *)cuda_resolve_weight_ptr(model_map, base_offset,
-                    mix_bytes, logical_tier, "hc_base");
-            const float *norm_w = (const float *)cuda_resolve_weight_ptr(model_map, norm_weight_offset,
-                    (uint64_t)n_embd * sizeof(float), logical_tier, "hc_norm_weight");
-            if (!scale || !base || !norm_w) return 0;
-            hc_split_weighted_sum_norm_fused_kernel<<<(uint32_t)n_rows, 256, 0, cuda_decode_stream()>>>(
+        const uint64_t n_rows = out->bytes / out_row_bytes;
+        if (n_rows > (uint64_t)INT_MAX ||
+            n_rows > UINT64_MAX / mix_bytes ||
+            n_rows > UINT64_MAX / residual_row_bytes) {
+            return 0;
+        }
+        const uint64_t mix_total_bytes = n_rows * mix_bytes;
+        const uint64_t residual_total_bytes = n_rows * residual_row_bytes;
+        if (mix->bytes < mix_total_bytes ||
+            split->bytes < mix_total_bytes ||
+            residual_hc->bytes < residual_total_bytes) {
+            return 0;
+        }
+        const int logical_tier = ds4_tensor_device_idx(out);
+        const float *scale = (const float *)cuda_resolve_weight_ptr(model_map, scale_offset,
+                3ull * sizeof(float), logical_tier, "hc_scale");
+        const float *base = (const float *)cuda_resolve_weight_ptr(model_map, base_offset,
+                mix_bytes, logical_tier, "hc_base");
+        const float *norm_w = (const float *)cuda_resolve_weight_ptr(model_map, norm_weight_offset,
+                (uint64_t)n_embd * sizeof(float), logical_tier, "hc_norm_weight");
+        if (!scale || !base || !norm_w) return 0;
+        const cudaStream_t stream = cuda_decode_stream();
+        if (n_embd == 4096u && n_rows == 1u &&
+            out->ptr != norm_out->ptr &&
+            getenv("DS4_CUDA_NO_HC_SPLIT_NORM_SPLIT4096") == NULL) {
+            bool split_ok = true;
+            hc_split_weighted_sum_norm_fused_partial4096_kernel<<<16u, 256, 0, stream>>>(
                     (float *)out->ptr,
-                    (float *)norm_out->ptr,
                     (float *)split->ptr,
                     (const float *)mix->ptr,
                     (const float *)residual_hc->ptr,
                     scale,
                     base,
-                    norm_w,
-                    n_embd, n_hc, (uint32_t)n_rows, sinkhorn_iters, eps, norm_eps);
-            return cuda_ok(cudaGetLastError(), "hc split weighted sum norm launch");
+                    sinkhorn_iters,
+                    eps);
+            split_ok = cudaGetLastError() == cudaSuccess;
+            float *norm_scale = NULL;
+            if (split_ok) {
+                norm_scale = (float *)cuda_tmp_alloc_on(
+                        logical_tier, sizeof(float), "hc split norm scale");
+                split_ok = norm_scale != NULL;
+            }
+            if (split_ok) {
+                hc_split_weighted_sum_norm_fused_reduce4096_kernel<<<1u, 256, 0, stream>>>(
+                        (const float *)out->ptr,
+                        norm_scale,
+                        norm_eps);
+                split_ok = cudaGetLastError() == cudaSuccess;
+            }
+            cuda_block_q8_1 *fold_q81 = NULL;
+            if (split_ok) {
+                fold_q81 = cuda_q8_fold_prepare(
+                        norm_out->ptr, n_embd, model_map, stream);
+                hc_split_weighted_sum_norm_fused_store4096_kernel<<<16u, 256, 0, stream>>>(
+                        (const float *)out->ptr,
+                        (float *)norm_out->ptr,
+                        norm_w,
+                        norm_scale,
+                        fold_q81);
+                split_ok = cudaGetLastError() == cudaSuccess;
+            }
+            if (split_ok) {
+                cuda_q8_fold_publish(norm_out->ptr, n_embd, model_map,
+                                     stream, fold_q81);
+                return 1;
+            }
+            if (fold_q81) cuda_q8_fold_invalidate_all();
+            /* The split launches are adjacent, so any pre-enqueue failure
+             * leaves a valid fused-reference retry: it rewrites out, split,
+             * and norm_out without consuming partial state. */
         }
+        cuda_block_q8_1 *fold_q81 = n_rows == 1u
+            ? cuda_q8_fold_prepare(norm_out->ptr, n_embd, model_map, stream)
+            : NULL;
+        hc_split_weighted_sum_norm_fused_kernel<<<(uint32_t)n_rows, 256, 0, stream>>>(
+                (float *)out->ptr,
+                (float *)norm_out->ptr,
+                (float *)split->ptr,
+                (const float *)mix->ptr,
+                (const float *)residual_hc->ptr,
+                scale,
+                base,
+                norm_w,
+                n_embd, n_hc, (uint32_t)n_rows, sinkhorn_iters, eps,
+                norm_eps, fold_q81);
+        const cudaError_t launch_err = cudaGetLastError();
+        if (launch_err == cudaSuccess) {
+            cuda_q8_fold_publish(norm_out->ptr, n_embd, model_map,
+                                 stream, fold_q81);
+        } else if (fold_q81) {
+            cuda_q8_fold_invalidate_all();
+        }
+        return cuda_ok(launch_err, "hc split weighted sum norm launch");
     }
     /* Multi-row fallback: norm EVERY row (rms_norm_weight_tensor is the
      * single-row entry and would leave rows 1..n-1 of norm_out untouched). */
     if (!out || n_embd == 0) return 0;
+    const uint64_t fallback_row_bytes = (uint64_t)n_embd * sizeof(float);
+    if (out->bytes < fallback_row_bytes ||
+        out->bytes % fallback_row_bytes != 0 ||
+        out->bytes / fallback_row_bytes > (uint64_t)INT_MAX) {
+        return 0;
+    }
+    const uint32_t fallback_rows =
+        (uint32_t)(out->bytes / fallback_row_bytes);
     return ds4_gpu_hc_split_weighted_sum_tensor(out, split, mix, residual_hc,
                                                   model_map, model_size,
                                                   scale_offset, base_offset,
@@ -26259,8 +33638,7 @@ extern "C" int ds4_gpu_hc_split_weighted_sum_norm_tensor(
            ds4_gpu_rms_norm_weight_rows_tensor(
                    norm_out, out, model_map, model_size,
                    norm_weight_offset, n_embd,
-                   (uint32_t)(out->bytes /
-                              ((uint64_t)n_embd * sizeof(float))),
+                   fallback_rows,
                    norm_eps);
 }
 extern "C" int ds4_gpu_output_hc_weights_tensor(
@@ -26333,7 +33711,7 @@ extern "C" int ds4_gpu_hc_expand_split_tensor(ds4_gpu_tensor *out_hc, const ds4_
     uint32_t mix_hc = 2u * n_hc + n_hc * n_hc;
     uint64_t n_elem = (uint64_t)n_tokens * n_hc * n_embd;
     const float *base = (const float *)split->ptr;
-    hc_expand_kernel<<<(n_elem + 255) / 256, 256>>>((float *)out_hc->ptr,
+    hc_expand_kernel<<<(n_elem + 255) / 256, 256, 0, cuda_decode_stream()>>>((float *)out_hc->ptr,
                                                     (const float *)block_out->ptr,
                                                     (const float *)block_out->ptr,
                                                     (const float *)block_out->ptr,
@@ -26369,7 +33747,7 @@ extern "C" int ds4_gpu_hc_expand_add2_split_tensor(ds4_gpu_tensor *out_hc, const
     uint32_t mix_hc = 2u * n_hc + n_hc * n_hc;
     uint64_t n_elem = (uint64_t)n_tokens * n_hc * n_embd;
     const float *base = (const float *)split->ptr;
-    hc_expand_kernel<<<(n_elem + 255) / 256, 256>>>((float *)out_hc->ptr,
+    hc_expand_kernel<<<(n_elem + 255) / 256, 256, 0, cuda_decode_stream()>>>((float *)out_hc->ptr,
                                                     (const float *)block_out->ptr,
                                                     (const float *)block_add->ptr,
                                                     (const float *)block_add2->ptr,
@@ -26395,25 +33773,22 @@ extern "C" int ds4_gpu_shared_down_hc_expand_q8_0_tensor(
         const ds4_gpu_tensor *split,
         uint32_t                n_embd,
         uint32_t                n_hc) {
-    if (getenv("DS4_CUDA_DISABLE_Q8_HC_EXPAND_FUSED") == NULL) {
-        return cuda_matmul_q8_0_hc_expand_tensor_labeled(out_hc, shared_out,
-                                                        model_map, model_size,
-                                                        weight_offset,
-                                                        in_dim, out_dim,
-                                                        shared_mid,
-                                                        routed_out,
-                                                        NULL,
-                                                        NULL, NULL, NULL, 0,
-                                                        residual_hc,
-                                                        split,
-                                                        n_embd, n_hc,
-                                                        "shared_down_hc_expand");
-    }
-    return ds4_gpu_matmul_q8_0_tensor(shared_out, model_map, model_size,
-                                        weight_offset, in_dim, out_dim,
-                                        shared_mid, 1) &&
-           ds4_gpu_hc_expand_add_split_tensor(out_hc, shared_out, routed_out,
-	                                                residual_hc, split, n_embd, n_hc);
+    const int path = cuda_q8_hc_expand_policy_path(/*owned=*/0);
+    const int result = path == CUDA_Q8_HC_EXPAND_FUSED
+        ? cuda_matmul_q8_0_hc_expand_tensor_labeled(
+              out_hc, shared_out, model_map, model_size, weight_offset,
+              in_dim, out_dim, shared_mid, routed_out, NULL,
+              NULL, NULL, NULL, 0, residual_hc, split, n_embd, n_hc,
+              "shared_down_hc_expand")
+        : (cuda_matmul_q8_0_hc_split_matmul_tensor_labeled(
+               shared_out, model_map, model_size, weight_offset,
+               in_dim, out_dim, shared_mid,
+               ds4_tensor_device_idx(out_hc),
+               "shared_down_hc_expand_split") &&
+           ds4_gpu_hc_expand_add_split_tensor(
+               out_hc, shared_out, routed_out, residual_hc, split,
+               n_embd, n_hc));
+    return cuda_q8_hc_expand_policy_complete(path, result);
 }
 
 extern "C" int ds4_gpu_shared_down_hc_expand_add_q8_0_tensor(
@@ -26431,26 +33806,22 @@ extern "C" int ds4_gpu_shared_down_hc_expand_add_q8_0_tensor(
         const ds4_gpu_tensor *split,
         uint32_t                n_embd,
         uint32_t                n_hc) {
-    if (getenv("DS4_CUDA_DISABLE_Q8_HC_EXPAND_FUSED") == NULL) {
-        return cuda_matmul_q8_0_hc_expand_tensor_labeled(out_hc, shared_out,
-                                                        model_map, model_size,
-                                                        weight_offset,
-                                                        in_dim, out_dim,
-                                                        shared_mid,
-                                                        routed_out,
-                                                        routed_add,
-                                                        NULL, NULL, NULL, 0,
-                                                        residual_hc,
-                                                        split,
-                                                        n_embd, n_hc,
-                                                        "shared_down_hc_expand_add");
-    }
-    return ds4_gpu_matmul_q8_0_tensor(shared_out, model_map, model_size,
-                                        weight_offset, in_dim, out_dim,
-                                        shared_mid, 1) &&
-           ds4_gpu_hc_expand_add2_split_tensor(out_hc, shared_out, routed_out,
-                                                routed_add, residual_hc, split,
-                                                n_embd, n_hc);
+    const int path = cuda_q8_hc_expand_policy_path(/*owned=*/0);
+    const int result = path == CUDA_Q8_HC_EXPAND_FUSED
+        ? cuda_matmul_q8_0_hc_expand_tensor_labeled(
+              out_hc, shared_out, model_map, model_size, weight_offset,
+              in_dim, out_dim, shared_mid, routed_out, routed_add,
+              NULL, NULL, NULL, 0, residual_hc, split, n_embd, n_hc,
+              "shared_down_hc_expand_add")
+        : (cuda_matmul_q8_0_hc_split_matmul_tensor_labeled(
+               shared_out, model_map, model_size, weight_offset,
+               in_dim, out_dim, shared_mid,
+               ds4_tensor_device_idx(out_hc),
+               "shared_down_hc_expand_add_split") &&
+           ds4_gpu_hc_expand_add2_split_tensor(
+               out_hc, shared_out, routed_out, routed_add, residual_hc,
+               split, n_embd, n_hc));
+    return cuda_q8_hc_expand_policy_complete(path, result);
 }
 
 extern "C" int ds4_gpu_shared_down_hc_expand_owned_q8_0_tensor(
@@ -26470,8 +33841,8 @@ extern "C" int ds4_gpu_shared_down_hc_expand_owned_q8_0_tensor(
         const ds4_gpu_tensor *split,
         uint32_t n_embd,
         uint32_t n_hc) {
-    if (getenv("DS4_CUDA_DISABLE_Q8_HC_EXPAND_FUSED") != NULL) return 0;
-    return cuda_matmul_q8_0_hc_expand_tensor_labeled(
+    const int path = cuda_q8_hc_expand_policy_path(/*owned=*/1);
+    const int result = cuda_matmul_q8_0_hc_expand_tensor_labeled(
             out_hc,
             shared_out,
             model_map,
@@ -26491,6 +33862,7 @@ extern "C" int ds4_gpu_shared_down_hc_expand_owned_q8_0_tensor(
             n_embd,
             n_hc,
             "shared_down_hc_expand_owned");
+    return cuda_q8_hc_expand_policy_complete(path, result);
 }
 
 extern "C" int ds4_gpu_matmul_q8_0_hc_expand_tensor(
@@ -26506,24 +33878,391 @@ extern "C" int ds4_gpu_matmul_q8_0_hc_expand_tensor(
         const ds4_gpu_tensor *split,
         uint32_t                n_embd,
         uint32_t                n_hc) {
-    if (getenv("DS4_CUDA_DISABLE_Q8_HC_EXPAND_FUSED") == NULL) {
-        return cuda_matmul_q8_0_hc_expand_tensor_labeled(out_hc, block_out,
-                                                        model_map, model_size,
-                                                        weight_offset,
-                                                        in_dim, out_dim,
-                                                        x,
-                                                        NULL,
-                                                        NULL,
-                                                        NULL, NULL, NULL, 0,
-                                                        residual_hc,
-                                                        split,
-                                                        n_embd, n_hc,
-                                                        "q8_hc_expand");
-    }
-    return ds4_gpu_matmul_q8_0_tensor(block_out, model_map, model_size,
-                                        weight_offset, in_dim, out_dim, x, 1) &&
-           ds4_gpu_hc_expand_split_tensor(out_hc, block_out, residual_hc,
-                                            split, n_embd, n_hc);
+    const int path = cuda_q8_hc_expand_policy_path(/*owned=*/0);
+    const int result = path == CUDA_Q8_HC_EXPAND_FUSED
+        ? cuda_matmul_q8_0_hc_expand_tensor_labeled(
+              out_hc, block_out, model_map, model_size, weight_offset,
+              in_dim, out_dim, x, NULL, NULL, NULL, NULL, NULL, 0,
+              residual_hc, split, n_embd, n_hc, "q8_hc_expand")
+        : (cuda_matmul_q8_0_hc_split_matmul_tensor_labeled(
+               block_out, model_map, model_size, weight_offset,
+               in_dim, out_dim, x, ds4_tensor_device_idx(out_hc),
+               "q8_hc_expand_split") &&
+           ds4_gpu_hc_expand_split_tensor(
+               out_hc, block_out, residual_hc, split, n_embd, n_hc));
+    return cuda_q8_hc_expand_policy_complete(path, result);
+}
+
+/* Device oracle for the Q8 shared-down/HC A/B boundary.  Its 128x1024
+ * fixture satisfies the production aligned-artifact predicate and installs
+ * temporary raw/derived resolver entries so the real fused and split helpers
+ * run.  It checks both aligned and raw paths bit-for-bit; the aligned split
+ * plus both HC epilogues are also recorded in a CUDA graph to catch accidental
+ * legacy-stream launches. */
+extern "C" int ds4_cuda_test_q8_hc_expand_oracle(void) {
+    g_q8_hc_expand_oracle_runs.fetch_add(1u, std::memory_order_relaxed);
+
+    constexpr uint32_t in_dim = 1024u;
+    constexpr uint32_t n_embd = 128u;
+    constexpr uint32_t n_hc = 4u;
+    constexpr uint32_t blocks = in_dim / 32u;
+    constexpr uint64_t n_weight_blocks =
+        (uint64_t)n_embd * blocks;
+    constexpr uint64_t weight_bytes =
+        n_weight_blocks * 34u;
+    constexpr uint64_t aligned_dq_bytes =
+        (n_weight_blocks * sizeof(__half) + 63u) & ~63ull;
+    constexpr uint64_t aligned_bytes =
+        aligned_dq_bytes + n_weight_blocks * 32u;
+    constexpr uint64_t x_bytes = (uint64_t)in_dim * sizeof(float);
+    constexpr uint64_t row_bytes = (uint64_t)n_embd * sizeof(float);
+    constexpr uint64_t hc_bytes =
+        (uint64_t)n_hc * n_embd * sizeof(float);
+    constexpr uint64_t split_count = 2u * n_hc + n_hc * n_hc;
+    constexpr uint64_t split_bytes = split_count * sizeof(float);
+
+    void *d_aligned = NULL;
+    void *d_raw = NULL;
+    void *d_x = NULL;
+    void *d_residual = NULL;
+    void *d_split = NULL;
+    void *d_add = NULL;
+    void *d_add2 = NULL;
+    void *d_block_fused_add = NULL;
+    void *d_block_fused_plain = NULL;
+    void *d_block_split = NULL;
+    void *d_hc_fused_add = NULL;
+    void *d_hc_fused_plain = NULL;
+    void *d_hc_split_add = NULL;
+    void *d_hc_split_plain = NULL;
+    cudaStream_t stream = NULL;
+    cudaGraph_t graph = NULL;
+    cudaGraphExec_t exec = NULL;
+    cudaStream_t saved_decode_graph_stream = g_decode_graph_stream;
+    const int saved_decode_graph_capturing = g_decode_graph_capturing;
+    const size_t saved_model_range_count = g_model_ranges.size();
+    const size_t saved_derived_range_count = g_derived_ranges.size();
+    const int saved_gb10_tier0 = g_cuda_is_gb10[0];
+    int oracle_ranges_installed = 0;
+    int capture_started = 0;
+    int ok = 0;
+
+    std::vector<void *> allocations;
+    auto alloc = [&allocations](void **ptr, uint64_t bytes) -> int {
+        if (cudaMalloc(ptr, (size_t)bytes) != cudaSuccess) return 0;
+        allocations.push_back(*ptr);
+        return 1;
+    };
+
+    do {
+        if (saved_decode_graph_capturing) break;
+        if (!alloc(&d_aligned, aligned_bytes) ||
+            !alloc(&d_raw, weight_bytes) ||
+            !alloc(&d_x, x_bytes) ||
+            !alloc(&d_residual, hc_bytes) ||
+            !alloc(&d_split, split_bytes) ||
+            !alloc(&d_add, row_bytes) ||
+            !alloc(&d_add2, row_bytes) ||
+            !alloc(&d_block_fused_add, row_bytes) ||
+            !alloc(&d_block_fused_plain, row_bytes) ||
+            !alloc(&d_block_split, row_bytes) ||
+            !alloc(&d_hc_fused_add, hc_bytes) ||
+            !alloc(&d_hc_fused_plain, hc_bytes) ||
+            !alloc(&d_hc_split_add, hc_bytes) ||
+            !alloc(&d_hc_split_plain, hc_bytes)) {
+            break;
+        }
+        if (cudaStreamCreate(&stream) != cudaSuccess) break;
+
+        std::vector<unsigned char> h_weight((size_t)weight_bytes);
+        for (uint32_t row = 0; row < n_embd; row++) {
+            for (uint32_t block = 0; block < blocks; block++) {
+                unsigned char *dst = h_weight.data() +
+                    ((uint64_t)row * blocks + block) * 34u;
+                const __half scale = __float2half(
+                    0.00390625f * (float)(1u + ((row + block) % 3u)));
+                memcpy(dst, &scale, sizeof(scale));
+                int8_t *codes = (int8_t *)(dst + sizeof(scale));
+                for (uint32_t i = 0; i < 32u; i++) {
+                    codes[i] = (int8_t)(
+                        (int)((row * 5u + block * 7u + i * 3u) % 23u) - 11);
+                }
+            }
+        }
+        std::vector<unsigned char> h_aligned((size_t)aligned_bytes);
+        for (uint64_t block = 0; block < n_weight_blocks; block++) {
+            const unsigned char *src = h_weight.data() + block * 34u;
+            memcpy(h_aligned.data() + block * sizeof(__half),
+                   src, sizeof(__half));
+            memcpy(h_aligned.data() + aligned_dq_bytes + block * 32u,
+                   src + sizeof(__half), 32u);
+        }
+        std::vector<float> h_x(in_dim);
+        std::vector<float> h_residual((size_t)n_hc * n_embd);
+        std::vector<float> h_split((size_t)split_count);
+        constexpr float host_guard = 12345.5f;
+        std::vector<float> h_add_storage(n_embd + 2u, host_guard);
+        std::vector<float> h_add2_storage(n_embd + 2u, host_guard);
+        float *h_add = h_add_storage.data() + 1u;
+        float *h_add2 = h_add2_storage.data() + 1u;
+        for (uint32_t i = 0; i < in_dim; i++) {
+            h_x[i] = (float)((int)(i % 17u) - 8) * 0.0625f;
+        }
+        for (uint32_t i = 0; i < n_embd; i++) {
+            h_add[i] = (float)((int)(i % 7u) - 3) * 0.03125f;
+            h_add2[i] = (float)((int)(i % 5u) - 2) * 0.015625f;
+        }
+        if (h_add_storage.front() != host_guard ||
+            h_add_storage.back() != host_guard ||
+            h_add2_storage.front() != host_guard ||
+            h_add2_storage.back() != host_guard) {
+            break;
+        }
+        for (uint32_t h = 0; h < n_hc; h++) {
+            for (uint32_t d = 0; d < n_embd; d++) {
+                h_residual[(uint64_t)h * n_embd + d] =
+                    (float)((int)((h * 13u + d) % 19u) - 9) * 0.0078125f;
+            }
+            h_split[n_hc + h] = 0.25f + (float)h * 0.0625f;
+        }
+        for (uint32_t src = 0; src < n_hc; src++) {
+            for (uint32_t dst = 0; dst < n_hc; dst++) {
+                h_split[2u * n_hc + dst + (uint64_t)src * n_hc] =
+                    src == dst ? 0.75f : 0.03125f * (float)(src + dst + 1u);
+            }
+        }
+
+        if (cudaMemcpy(d_aligned, h_aligned.data(), (size_t)aligned_bytes,
+                       cudaMemcpyHostToDevice) != cudaSuccess ||
+            cudaMemcpy(d_raw, h_weight.data(), (size_t)weight_bytes,
+                       cudaMemcpyHostToDevice) != cudaSuccess ||
+            cudaMemcpy(d_x, h_x.data(), (size_t)x_bytes,
+                       cudaMemcpyHostToDevice) != cudaSuccess ||
+            cudaMemcpy(d_residual, h_residual.data(), (size_t)hc_bytes,
+                       cudaMemcpyHostToDevice) != cudaSuccess ||
+            cudaMemcpy(d_split, h_split.data(), (size_t)split_bytes,
+                       cudaMemcpyHostToDevice) != cudaSuccess ||
+            cudaMemcpy(d_add, h_add, (size_t)row_bytes,
+                       cudaMemcpyHostToDevice) != cudaSuccess ||
+            cudaMemcpy(d_add2, h_add2, (size_t)row_bytes,
+                       cudaMemcpyHostToDevice) != cudaSuccess) {
+            break;
+        }
+
+        if (g_n_gpus != 1 || !cuda_q8_use_dp4a() ||
+            !cuda_aligned_q8_enabled() ||
+            getenv("DS4_CUDA_NO_Q8_FUSED_ALIGNED") != NULL ||
+            getenv("DS4_CUDA_NO_DERIVED_WEIGHTS") != NULL) {
+            break;
+        }
+        const void *oracle_model_map = h_weight.data();
+        g_model_ranges.push_back({
+            oracle_model_map, 0u, weight_bytes, (char *)d_raw,
+            NULL, NULL, 0u, 0, 1});
+        g_derived_ranges.push_back({
+            oracle_model_map, 0u, weight_bytes,
+            CUDA_DERIVED_Q8_0_ALIGNED_DENSE,
+            in_dim, n_embd, 1u, aligned_bytes, (char *)d_aligned});
+        g_cuda_is_gb10[0] = 1;
+        oracle_ranges_installed = 1;
+
+        ds4_gpu_tensor x = {};
+        ds4_gpu_tensor block_fused_add = {};
+        ds4_gpu_tensor block_fused_plain = {};
+        ds4_gpu_tensor block_split = {};
+        ds4_gpu_tensor add = {};
+        ds4_gpu_tensor add2 = {};
+        ds4_gpu_tensor residual = {};
+        ds4_gpu_tensor split = {};
+        ds4_gpu_tensor hc_fused_add = {};
+        ds4_gpu_tensor hc_fused_plain = {};
+        ds4_gpu_tensor hc_split_add = {};
+        ds4_gpu_tensor hc_split_plain = {};
+        x.ptr = d_x;
+        x.bytes = x_bytes;
+        x.device_id = 0;
+        block_fused_add.ptr = d_block_fused_add;
+        block_fused_add.bytes = row_bytes;
+        block_fused_add.device_id = 0;
+        block_fused_plain.ptr = d_block_fused_plain;
+        block_fused_plain.bytes = row_bytes;
+        block_fused_plain.device_id = 0;
+        block_split.ptr = d_block_split;
+        block_split.bytes = row_bytes;
+        block_split.device_id = 0;
+        add.ptr = d_add;
+        add.bytes = row_bytes;
+        add.device_id = 0;
+        add2.ptr = d_add2;
+        add2.bytes = row_bytes;
+        add2.device_id = 0;
+        residual.ptr = d_residual;
+        residual.bytes = hc_bytes;
+        residual.device_id = 0;
+        split.ptr = d_split;
+        split.bytes = split_bytes;
+        split.device_id = 0;
+        hc_fused_add.ptr = d_hc_fused_add;
+        hc_fused_add.bytes = hc_bytes;
+        hc_fused_add.device_id = 0;
+        hc_fused_plain.ptr = d_hc_fused_plain;
+        hc_fused_plain.bytes = hc_bytes;
+        hc_fused_plain.device_id = 0;
+        hc_split_add.ptr = d_hc_split_add;
+        hc_split_add.bytes = hc_bytes;
+        hc_split_add.device_id = 0;
+        hc_split_plain.ptr = d_hc_split_plain;
+        hc_split_plain.bytes = hc_bytes;
+        hc_split_plain.device_id = 0;
+
+        /* Exercise the actual fused preparation/resolver wrapper and warm
+         * its reusable Q8 scratch before graph capture. */
+        const cudaStream_t eager_helper_stream = cuda_decode_stream();
+        if (!cuda_matmul_q8_0_hc_expand_tensor_labeled(
+                &hc_fused_add, &block_fused_add,
+                oracle_model_map, weight_bytes, 0u,
+                in_dim, n_embd, &x, &add, &add2,
+                NULL, NULL, NULL, 0u, &residual, &split,
+                n_embd, n_hc, "q8_hc_oracle_aligned_add") ||
+            !cuda_matmul_q8_0_hc_expand_tensor_labeled(
+                &hc_fused_plain, &block_fused_plain,
+                oracle_model_map, weight_bytes, 0u,
+                in_dim, n_embd, &x, NULL, NULL,
+                NULL, NULL, NULL, 0u, &residual, &split,
+                n_embd, n_hc, "q8_hc_oracle_aligned_plain") ||
+            cudaStreamSynchronize(eager_helper_stream) != cudaSuccess) {
+            break;
+        }
+
+        if (cudaStreamBeginCapture(stream,
+                                   cudaStreamCaptureModeGlobal) != cudaSuccess) {
+            break;
+        }
+        capture_started = 1;
+        g_decode_graph_stream = stream;
+        g_decode_graph_capturing = 1;
+        const int matmul_ok =
+            cuda_matmul_q8_0_hc_split_matmul_tensor_labeled(
+                &block_split, oracle_model_map, weight_bytes, 0u,
+                in_dim, n_embd, &x, 0,
+                "q8_hc_oracle_aligned_split");
+        const int add_ok = ds4_gpu_hc_expand_add2_split_tensor(
+            &hc_split_add, &block_split, &add, &add2, &residual, &split,
+            n_embd, n_hc);
+        const int plain_ok = ds4_gpu_hc_expand_split_tensor(
+            &hc_split_plain, &block_split, &residual, &split,
+            n_embd, n_hc);
+        g_decode_graph_capturing = saved_decode_graph_capturing;
+        g_decode_graph_stream = saved_decode_graph_stream;
+        const cudaError_t capture_end = cudaStreamEndCapture(stream, &graph);
+        capture_started = 0;
+        if (!matmul_ok || !add_ok || !plain_ok ||
+            capture_end != cudaSuccess || !graph) {
+            break;
+        }
+        if (cudaGraphInstantiate(&exec, graph, NULL, NULL, 0) != cudaSuccess ||
+            !exec || cudaGraphLaunch(exec, stream) != cudaSuccess ||
+            cudaStreamSynchronize(stream) != cudaSuccess) {
+            break;
+        }
+
+        auto outputs_match = [&]() -> int {
+            std::vector<float> h_block_fused_add(n_embd);
+            std::vector<float> h_block_fused_plain(n_embd);
+            std::vector<float> h_block_split(n_embd);
+            std::vector<float> h_fused_add((size_t)n_hc * n_embd);
+            std::vector<float> h_split_add((size_t)n_hc * n_embd);
+            std::vector<float> h_fused_plain((size_t)n_hc * n_embd);
+            std::vector<float> h_split_plain((size_t)n_hc * n_embd);
+            if (cudaMemcpy(h_block_fused_add.data(), d_block_fused_add,
+                           (size_t)row_bytes, cudaMemcpyDeviceToHost) !=
+                    cudaSuccess ||
+                cudaMemcpy(h_block_fused_plain.data(), d_block_fused_plain,
+                           (size_t)row_bytes, cudaMemcpyDeviceToHost) !=
+                    cudaSuccess ||
+                cudaMemcpy(h_block_split.data(), d_block_split,
+                           (size_t)row_bytes, cudaMemcpyDeviceToHost) !=
+                    cudaSuccess ||
+                cudaMemcpy(h_fused_add.data(), d_hc_fused_add,
+                           (size_t)hc_bytes, cudaMemcpyDeviceToHost) !=
+                    cudaSuccess ||
+                cudaMemcpy(h_split_add.data(), d_hc_split_add,
+                           (size_t)hc_bytes, cudaMemcpyDeviceToHost) !=
+                    cudaSuccess ||
+                cudaMemcpy(h_fused_plain.data(), d_hc_fused_plain,
+                           (size_t)hc_bytes, cudaMemcpyDeviceToHost) !=
+                    cudaSuccess ||
+                cudaMemcpy(h_split_plain.data(), d_hc_split_plain,
+                           (size_t)hc_bytes, cudaMemcpyDeviceToHost) !=
+                    cudaSuccess) {
+                return 0;
+            }
+            return memcmp(h_block_fused_add.data(), h_block_split.data(),
+                          (size_t)row_bytes) == 0 &&
+                   memcmp(h_block_fused_plain.data(), h_block_split.data(),
+                          (size_t)row_bytes) == 0 &&
+                   memcmp(h_fused_add.data(), h_split_add.data(),
+                          (size_t)hc_bytes) == 0 &&
+                   memcmp(h_fused_plain.data(), h_split_plain.data(),
+                          (size_t)hc_bytes) == 0;
+        };
+        if (!outputs_match()) break;
+
+        /* Remove only the temporary derived entry to force the same public
+         * helpers through their raw-weight side, then repeat bit parity. */
+        g_derived_ranges.resize(saved_derived_range_count);
+        g_cuda_is_gb10[0] = 0;
+        const cudaStream_t raw_helper_stream = cuda_decode_stream();
+        if (!cuda_matmul_q8_0_hc_expand_tensor_labeled(
+                &hc_fused_add, &block_fused_add,
+                oracle_model_map, weight_bytes, 0u,
+                in_dim, n_embd, &x, &add, &add2,
+                NULL, NULL, NULL, 0u, &residual, &split,
+                n_embd, n_hc, "q8_hc_oracle_raw_add") ||
+            !cuda_matmul_q8_0_hc_expand_tensor_labeled(
+                &hc_fused_plain, &block_fused_plain,
+                oracle_model_map, weight_bytes, 0u,
+                in_dim, n_embd, &x, NULL, NULL,
+                NULL, NULL, NULL, 0u, &residual, &split,
+                n_embd, n_hc, "q8_hc_oracle_raw_plain") ||
+            !cuda_matmul_q8_0_hc_split_matmul_tensor_labeled(
+                &block_split, oracle_model_map, weight_bytes, 0u,
+                in_dim, n_embd, &x, 0, "q8_hc_oracle_raw_split") ||
+            !ds4_gpu_hc_expand_add2_split_tensor(
+                &hc_split_add, &block_split, &add, &add2,
+                &residual, &split, n_embd, n_hc) ||
+            !ds4_gpu_hc_expand_split_tensor(
+                &hc_split_plain, &block_split, &residual, &split,
+                n_embd, n_hc) ||
+            cudaStreamSynchronize(raw_helper_stream) != cudaSuccess) {
+            break;
+        }
+        ok = outputs_match();
+    } while (0);
+
+    g_decode_graph_capturing = saved_decode_graph_capturing;
+    g_decode_graph_stream = saved_decode_graph_stream;
+    if (capture_started && stream) {
+        cudaGraph_t abandoned = NULL;
+        (void)cudaStreamEndCapture(stream, &abandoned);
+        if (abandoned) (void)cudaGraphDestroy(abandoned);
+    }
+    if (oracle_ranges_installed) {
+        g_model_ranges.resize(saved_model_range_count);
+        g_derived_ranges.resize(saved_derived_range_count);
+        g_cuda_is_gb10[0] = saved_gb10_tier0;
+        oracle_ranges_installed = 0;
+    }
+    if (exec) (void)cudaGraphExecDestroy(exec);
+    if (graph) (void)cudaGraphDestroy(graph);
+    if (stream) (void)cudaStreamDestroy(stream);
+    for (void *ptr : allocations) (void)cudaFree(ptr);
+    if (!ok) {
+        (void)cudaGetLastError();
+        g_q8_hc_expand_oracle_failures.fetch_add(
+            1u, std::memory_order_relaxed);
+    }
+    return ok;
 }
 
 /* --gpu-vram auto probe. Defined here (in the .cu unit) so the
@@ -26632,23 +34371,18 @@ extern "C" int ds4_gpu_args_probe_auto_cuda(const int      *device_filter,
     return 0;
 }
 
-typedef struct ds4_gpu_stream_expert_table {
-    const void *model_map;
-    uint64_t    model_size;
-    uint32_t    layer;
-    uint32_t    n_total_expert;
-    uint64_t    gate_offset;
-    uint64_t    up_offset;
-    uint64_t    down_offset;
-    uint64_t    gate_expert_bytes;
-    uint64_t    down_expert_bytes;
-} ds4_gpu_stream_expert_table;
-
 static int cuda_stream_selected_ensure_bytes(
         char **ptr, uint64_t *capacity, uint64_t bytes, const char *label) {
     if (*ptr && *capacity >= bytes) return 1;
     if (*ptr) {
-        (void)cudaFree(*ptr);
+        const cudaError_t free_err = cudaFree(*ptr);
+        if (free_err != cudaSuccess) {
+            fprintf(stderr,
+                    "ds4: CUDA streaming %s resize free failed: %s\n",
+                    label, cudaGetErrorString(free_err));
+            (void)cudaGetLastError();
+            return 0;
+        }
         *ptr = NULL;
         *capacity = 0;
     }
@@ -26664,12 +34398,15 @@ static int cuda_stream_selected_ensure_bytes(
     return 1;
 }
 
-static int cuda_stream_selected_ensure_i32(uint64_t count) {
+static int cuda_stream_selected_ensure_i32(
+        cuda_stream_selected_transient_storage *storage,
+        uint64_t count) {
+    if (!storage) return 0;
     if (count == 0 || count > UINT64_MAX / sizeof(int32_t)) return 0;
     const uint64_t bytes = count * sizeof(int32_t);
     return cuda_stream_selected_ensure_bytes(
-            (char **)&g_stream_selected_cache.slot_selected_ptr,
-            &g_stream_selected_cache.slot_selected_capacity,
+            (char **)&storage->remap,
+            &storage->remap_capacity,
             bytes,
             "selected-id remap");
 }
@@ -26699,11 +34436,27 @@ static int cuda_stream_selected_ranges_valid(
            down_bytes <= table->model_size - table->down_offset;
 }
 
-static int cuda_stream_selected_cache_begin_load(
+/* 1 publishes persistent metadata, 0 is a safe pre-enqueue rejection, and
+ * -1 is a post-enqueue/fail-closed result.  Caller owns the writer epoch. */
+static int cuda_stream_expert_persistent_try_load_writer(
         const ds4_gpu_stream_expert_table *table,
         const int32_t *selected_ids,
-        uint32_t slot_count) {
-    cuda_stream_selected_cache_invalidate();
+        uint32_t slot_count,
+        uint64_t *upload_event_out,
+        int *submitted_any_out,
+        int force_for_oracle);
+
+static int cuda_stream_selected_cache_begin_load_impl(
+        const ds4_gpu_stream_expert_table *table,
+        const int32_t *selected_ids,
+        uint32_t slot_count,
+        uint64_t *upload_event_out,
+        int *submitted_any_out) {
+    if (upload_event_out) *upload_event_out = 0;
+    if (submitted_any_out) *submitted_any_out = 0;
+    cuda_stream_selected_writer_guard writer;
+    cuda_stream_selected_transient_storage *storage =
+        &g_stream_selected_transient_storage;
     if (!g_ssd_streaming_mode) return 1;
     if (!cuda_stream_selected_ranges_valid(table) || !selected_ids ||
         slot_count == 0) {
@@ -26714,6 +34467,39 @@ static int cuda_stream_selected_cache_begin_load(
                 "ds4: CUDA SSD streaming requires single-GPU placement\n");
         return 0;
     }
+    /* This function also runs on the selected-load service thread.  CUDA's
+     * current device is thread-local, while g_current_logical_tier is a
+     * process-global launch cache and may already say tier 0 from the main
+     * thread.  Always select the physical owner explicitly here. */
+    if (cudaSetDevice(g_gpu[0].device_id) != cudaSuccess) {
+        fprintf(stderr,
+                "ds4: CUDA selected cache could not select owner device %d\n",
+                g_gpu[0].device_id);
+        (void)cudaGetLastError();
+        return 0;
+    }
+    if (g_stream_expert_persistent_runtime_ready &&
+        cuda_stream_expert_persistent_requested()) {
+        const int persistent_rc =
+            cuda_stream_expert_persistent_try_load_writer(
+                table, selected_ids, slot_count,
+                upload_event_out, submitted_any_out,
+                /*force_for_oracle=*/0);
+        if (persistent_rc > 0) {
+            writer.publish_valid();
+            return 1;
+        }
+        if (persistent_rc < 0 ||
+            cuda_stream_expert_persistent_require_requested()) {
+            return 0;
+        }
+        g_stream_expert_persistent_fallbacks.fetch_add(
+            1, std::memory_order_relaxed);
+    }
+    if (storage->poisoned &&
+        !cuda_stream_selected_cache_free_storage_writer()) {
+        return 0;
+    }
 
     std::vector<int32_t> expert_to_slot;
     std::vector<int32_t> compact_ids;
@@ -26742,6 +34528,17 @@ static int cuda_stream_selected_cache_begin_load(
         }
         slot_ids[i] = compact;
     }
+    int top6_unique = (slot_count % 6u) == 0u;
+    for (uint32_t base = 0; top6_unique && base < slot_count; base += 6u) {
+        for (uint32_t i = 0; top6_unique && i < 6u; i++) {
+            for (uint32_t j = i + 1u; j < 6u; j++) {
+                if (selected_ids[base + i] == selected_ids[base + j]) {
+                    top6_unique = 0;
+                    break;
+                }
+            }
+        }
+    }
     if (compact_ids.empty() || compact_ids.size() > UINT32_MAX) return 0;
     const uint64_t compact_count = compact_ids.size();
     if (compact_count > UINT64_MAX / table->gate_expert_bytes ||
@@ -26751,68 +34548,191 @@ static int cuda_stream_selected_cache_begin_load(
     const uint64_t gate_bytes = compact_count * table->gate_expert_bytes;
     const uint64_t down_bytes = compact_count * table->down_expert_bytes;
     const int logical_tier = 0;
-    if (g_stream_selected_cache.logical_tier != logical_tier &&
-        (g_stream_selected_cache.gate_ptr ||
-         g_stream_selected_cache.up_ptr ||
-         g_stream_selected_cache.down_ptr ||
-         g_stream_selected_cache.slot_selected_ptr)) {
-        cuda_stream_selected_cache_release();
+    if ((storage->gate || storage->up || storage->down || storage->remap) &&
+        storage->owner_device != g_gpu[0].device_id) {
+        if (!cuda_stream_selected_cache_free_storage_writer()) return 0;
+    }
+    const uint64_t remap_bytes = (uint64_t)slot_count * sizeof(int32_t);
+    const int resize_binding =
+        (storage->gate && storage->gate_capacity < gate_bytes) ||
+        (storage->up && storage->up_capacity < gate_bytes) ||
+        (storage->down && storage->down_capacity < down_bytes) ||
+        (storage->remap && storage->remap_capacity < remap_bytes);
+    if (resize_binding && !cuda_stream_selected_consume_drain()) {
+        cuda_stream_selected_cache_invalidate();
+        return 0;
+    }
+    if (resize_binding && !cuda_stream_selected_upload_drain_checked()) {
+        cuda_stream_selected_cache_invalidate();
+        storage->poisoned = 1;
+        return 0;
     }
+    storage->owner_device = g_gpu[0].device_id;
     if (ds4_gpu_set_current_device(logical_tier) != 0 ||
         !cuda_stream_selected_ensure_bytes(
-                &g_stream_selected_cache.gate_ptr,
-                &g_stream_selected_cache.gate_capacity,
+                &storage->gate,
+                &storage->gate_capacity,
                 gate_bytes, "gate experts") ||
         !cuda_stream_selected_ensure_bytes(
-                &g_stream_selected_cache.up_ptr,
-                &g_stream_selected_cache.up_capacity,
+                &storage->up,
+                &storage->up_capacity,
                 gate_bytes, "up experts") ||
         !cuda_stream_selected_ensure_bytes(
-                &g_stream_selected_cache.down_ptr,
-                &g_stream_selected_cache.down_capacity,
+                &storage->down,
+                &storage->down_capacity,
                 down_bytes, "down experts") ||
-        !cuda_stream_selected_ensure_i32(slot_count)) {
+        !cuda_stream_selected_ensure_i32(storage, slot_count)) {
         cuda_stream_selected_cache_invalidate();
         return 0;
     }
 
-    for (uint32_t i = 0; i < compact_ids.size(); i++) {
-        const uint64_t expert = (uint32_t)compact_ids[i];
-        const uint64_t gate_src =
-            table->gate_offset + expert * table->gate_expert_bytes;
-        const uint64_t up_src =
-            table->up_offset + expert * table->gate_expert_bytes;
-        const uint64_t down_src =
-            table->down_offset + expert * table->down_expert_bytes;
-        const uint64_t gate_dst = (uint64_t)i * table->gate_expert_bytes;
-        const uint64_t down_dst = (uint64_t)i * table->down_expert_bytes;
-        if (!cuda_model_copy_to_device_streamed(
-                    g_stream_selected_cache.gate_ptr + gate_dst,
-                    table->model_map, table->model_size,
-                    gate_src, table->gate_expert_bytes,
-                    "stream gate expert copy") ||
-            !cuda_model_copy_to_device_streamed(
-                    g_stream_selected_cache.up_ptr + gate_dst,
-                    table->model_map, table->model_size,
-                    up_src, table->gate_expert_bytes,
-                    "stream up expert copy") ||
-            !cuda_model_copy_to_device_streamed(
-                    g_stream_selected_cache.down_ptr + down_dst,
-                    table->model_map, table->model_size,
-                    down_src, table->down_expert_bytes,
-                    "stream down expert copy")) {
-            cuda_stream_selected_cache_invalidate();
-            return 0;
+    g_stream_selected_batch_io_candidates++;
+    const int batch_io = cuda_stream_selected_batch_io_requested();
+    const int batch_io_required =
+        cuda_stream_selected_batch_io_require_requested();
+    int copied = 0;
+    if (upload_event_out && !batch_io) {
+        /* The explicit async contract never falls into the per-expert
+         * synchronous loader.  Its caller may retry through the legacy API
+         * only after learning that no upload was submitted. */
+        return 0;
+    }
+    if (batch_io) {
+        std::vector<cuda_stream_selected_copy_task> tasks;
+        int task_plan_ready = 1;
+        try {
+            if (compact_ids.size() > SIZE_MAX / 3u ||
+                compact_ids.size() > UINT32_MAX / 3u) {
+                task_plan_ready = 0;
+            } else {
+                tasks.reserve(compact_ids.size() * 3u);
+                uint32_t ordinal = 0;
+                for (uint32_t i = 0; i < compact_ids.size(); i++) {
+                    const uint64_t expert = (uint32_t)compact_ids[i];
+                    const uint64_t gate_src = table->gate_offset +
+                        expert * table->gate_expert_bytes;
+                    const uint64_t up_src = table->up_offset +
+                        expert * table->gate_expert_bytes;
+                    const uint64_t down_src = table->down_offset +
+                        expert * table->down_expert_bytes;
+                    const uint64_t gate_dst =
+                        (uint64_t)i * table->gate_expert_bytes;
+                    const uint64_t down_dst =
+                        (uint64_t)i * table->down_expert_bytes;
+                    tasks.push_back({
+                        storage->gate + gate_dst,
+                        gate_src, table->gate_expert_bytes, ordinal++,
+                    });
+                    tasks.push_back({
+                        storage->up + gate_dst,
+                        up_src, table->gate_expert_bytes, ordinal++,
+                    });
+                    tasks.push_back({
+                        storage->down + down_dst,
+                        down_src, table->down_expert_bytes, ordinal++,
+                    });
+                }
+            }
+        } catch (...) {
+            task_plan_ready = 0;
+        }
+
+        if (task_plan_ready) {
+            int submitted = 0;
+            g_stream_selected_batch_io_attempts++;
+            copied = cuda_model_copy_tasks_to_device_streamed(
+                tasks, table->model_map, table->model_size,
+                storage->remap,
+                slot_ids.data(), slot_count,
+                cuda_stream_selected_batch_io_oracle_requested(),
+                /*chunk_override=*/0, &submitted,
+                upload_event_out,
+                "selected expert batch");
+            if (submitted_any_out) *submitted_any_out = submitted;
+            if (copied) {
+                g_stream_selected_batch_io_completed++;
+            } else {
+                g_stream_selected_batch_io_failures++;
+                if (batch_io_required) {
+                    g_stream_selected_batch_io_required_failures++;
+                }
+                if (submitted || batch_io_required || upload_event_out) {
+                    cuda_stream_selected_cache_invalidate();
+                    return 0;
+                }
+                g_stream_selected_batch_io_safe_fallbacks++;
+            }
+        } else {
+            g_stream_selected_batch_io_failures++;
+            if (batch_io_required) {
+                g_stream_selected_batch_io_required_failures++;
+            }
+            if (batch_io_required || upload_event_out) {
+                if (batch_io_required) {
+                    fprintf(stderr,
+                            "ds4: required CUDA selected batched I/O could "
+                            "not allocate its copy plan\n");
+                }
+                cuda_stream_selected_cache_invalidate();
+                return 0;
+            }
+            g_stream_selected_batch_io_safe_fallbacks++;
         }
     }
-    if (!cuda_ok(cudaMemcpy(g_stream_selected_cache.slot_selected_ptr,
-                            slot_ids.data(),
-                            (size_t)slot_count * sizeof(int32_t),
-                            cudaMemcpyHostToDevice),
-                 "stream selected-id remap copy")) {
+
+    if (!copied && upload_event_out) {
         cuda_stream_selected_cache_invalidate();
         return 0;
     }
+    if (!copied) {
+        /* The legacy copy path may use blocking cudaMemcpy before an upload
+         * stream exists.  It therefore needs the host-side lease boundary;
+         * the batched path imports the same event directly into its stream. */
+        if (!cuda_stream_selected_consume_drain()) {
+            cuda_stream_selected_cache_invalidate();
+            return 0;
+        }
+        g_stream_selected_batch_io_legacy++;
+        for (uint32_t i = 0; i < compact_ids.size(); i++) {
+            const uint64_t expert = (uint32_t)compact_ids[i];
+            const uint64_t gate_src =
+                table->gate_offset + expert * table->gate_expert_bytes;
+            const uint64_t up_src =
+                table->up_offset + expert * table->gate_expert_bytes;
+            const uint64_t down_src =
+                table->down_offset + expert * table->down_expert_bytes;
+            const uint64_t gate_dst =
+                (uint64_t)i * table->gate_expert_bytes;
+            const uint64_t down_dst =
+                (uint64_t)i * table->down_expert_bytes;
+            if (!cuda_model_copy_to_device_streamed(
+                        storage->gate + gate_dst,
+                        table->model_map, table->model_size,
+                        gate_src, table->gate_expert_bytes,
+                        "stream gate expert copy") ||
+                !cuda_model_copy_to_device_streamed(
+                        storage->up + gate_dst,
+                        table->model_map, table->model_size,
+                        up_src, table->gate_expert_bytes,
+                        "stream up expert copy") ||
+                !cuda_model_copy_to_device_streamed(
+                        storage->down + down_dst,
+                        table->model_map, table->model_size,
+                        down_src, table->down_expert_bytes,
+                        "stream down expert copy")) {
+                cuda_stream_selected_cache_invalidate();
+                return 0;
+            }
+        }
+        if (!cuda_ok(cudaMemcpy(storage->remap,
+                                slot_ids.data(),
+                                (size_t)slot_count * sizeof(int32_t),
+                                cudaMemcpyHostToDevice),
+                     "stream selected-id remap copy")) {
+            cuda_stream_selected_cache_invalidate();
+            return 0;
+        }
+    }
 
     g_stream_selected_cache.logical_tier = logical_tier;
     g_stream_selected_cache.model_map = table->model_map;
@@ -26820,21 +34740,39 @@ static int cuda_stream_selected_cache_begin_load(
     g_stream_selected_cache.n_total_expert = table->n_total_expert;
     g_stream_selected_cache.slot_count = slot_count;
     g_stream_selected_cache.compact_count = (uint32_t)compact_count;
+    g_stream_selected_cache.slot_base = 0;
+    g_stream_selected_cache.weight_domain = (uint32_t)compact_count;
+    g_stream_selected_cache.top6_unique = top6_unique;
+    uint64_t generation = ++g_stream_selected_cache_generation;
+    if (generation == 0) generation = ++g_stream_selected_cache_generation;
+    g_stream_selected_cache.generation = generation;
+    g_stream_selected_cache.upload_event_value =
+        upload_event_out ? *upload_event_out : 0;
     g_stream_selected_cache.gate_offset = table->gate_offset;
     g_stream_selected_cache.up_offset = table->up_offset;
     g_stream_selected_cache.down_offset = table->down_offset;
     g_stream_selected_cache.gate_expert_bytes = table->gate_expert_bytes;
     g_stream_selected_cache.down_expert_bytes = table->down_expert_bytes;
-    g_stream_selected_cache.slot_selected_tensor.ptr =
-        g_stream_selected_cache.slot_selected_ptr;
-    g_stream_selected_cache.slot_selected_tensor.bytes =
-        (uint64_t)slot_count * sizeof(int32_t);
-    g_stream_selected_cache.slot_selected_tensor.owner = 0;
-    g_stream_selected_cache.slot_selected_tensor.device_id = logical_tier;
-    g_stream_selected_cache.valid = 1;
+    if (!cuda_stream_selected_cache_bind_transient_storage(
+            logical_tier, gate_bytes, down_bytes, remap_bytes)) {
+        cuda_stream_selected_cache_invalidate();
+        return 0;
+    }
+    /* Publish all binding metadata atomically with respect to consumer
+     * acquisition.  The upload itself may still be in flight; its exact
+     * completion token is part of the metadata guarded by this release. */
+    writer.publish_valid();
     return 1;
 }
 
+static int cuda_stream_selected_cache_begin_load(
+        const ds4_gpu_stream_expert_table *table,
+        const int32_t *selected_ids,
+        uint32_t slot_count) {
+    return cuda_stream_selected_cache_begin_load_impl(
+        table, selected_ids, slot_count, NULL, NULL);
+}
+
 __device__ __forceinline__ static float glm_rope_yarn_corr_factor_dev(
         int n_dims, int n_ctx_orig, float n_rot, float base) {
     return n_dims * logf(n_ctx_orig / (n_rot * 2.0f * (float)M_PI)) /
@@ -32440,6 +40378,884 @@ static int cuda_matmul_mmq_dense_quant(
     return 1;
 }
 
+/* Dense Q4_K matvec fallback for the AProjQ4 attention projections. MMQ below
+ * handles prefill as a tiled matrix multiply so weights are reused across
+ * tokens; this bandwidth-oriented kernel remains useful for decode and for
+ * shapes MMQ cannot accept. */
+__global__ static void matmul_q4_K_dense_kernel(
+        float *out,
+        const char *w_base,
+        const cuda_block_q8_K *xq,
+        uint64_t row_bytes,
+        uint32_t xq_blocks,
+        uint32_t out_dim,
+        uint32_t n_tok) {
+    uint32_t lane = threadIdx.x & 7u;
+    uint32_t row_lane = threadIdx.x >> 3u;   /* 32 rows per 256-thread block */
+    uint32_t tok = blockIdx.y;
+    uint32_t row = blockIdx.x * 32u + row_lane;
+    if (tok >= n_tok || row >= out_dim) return;
+    const cuda_block_q8_K *xqb = xq + (uint64_t)tok * xq_blocks;
+    const cuda_block_q4_K *wr =
+        (const cuda_block_q4_K *)(w_base + (uint64_t)row * row_bytes);
+    float acc = 0.0f;
+    for (uint32_t b = lane; b < xq_blocks; b += 8u) {
+        acc += dev_dot_q4_K_q8_K_block(wr + b, xqb + b);
+    }
+    acc = quarter_warp_sum_f32(acc, lane);
+    if (lane == 0) out[(uint64_t)tok * out_dim + row] = acc;
+}
+
+/* Two independently-sized Q4_K projections over one canonical Q8_K row.
+ * Each accumulator keeps the same block walk and quarter-warp reduction as
+ * matmul_q4_K_dense_kernel; interleaving the weight loads does not alter either
+ * output's arithmetic order. */
+__global__ static void matmul_q4_K_dense_pair_kernel(
+        float *out0,
+        float *out1,
+        const char *w0_base,
+        const char *w1_base,
+        const cuda_block_q8_K *xq,
+        uint64_t row_bytes,
+        uint32_t xq_blocks,
+        uint32_t out0_dim,
+        uint32_t out1_dim,
+        uint32_t n_tok) {
+    const uint32_t lane = threadIdx.x & 7u;
+    const uint32_t row_lane = threadIdx.x >> 3u;
+    const uint32_t tok = blockIdx.y;
+    const uint32_t row = blockIdx.x * 32u + row_lane;
+    if (tok >= n_tok || (row >= out0_dim && row >= out1_dim)) return;
+
+    const cuda_block_q8_K *xqb = xq + (uint64_t)tok * xq_blocks;
+    const cuda_block_q4_K *wr0 = row < out0_dim
+        ? (const cuda_block_q4_K *)(w0_base + (uint64_t)row * row_bytes)
+        : NULL;
+    const cuda_block_q4_K *wr1 = row < out1_dim
+        ? (const cuda_block_q4_K *)(w1_base + (uint64_t)row * row_bytes)
+        : NULL;
+    float acc0 = 0.0f;
+    float acc1 = 0.0f;
+    for (uint32_t b = lane; b < xq_blocks; b += 8u) {
+        if (wr0) acc0 += dev_dot_q4_K_q8_K_block(wr0 + b, xqb + b);
+        if (wr1) acc1 += dev_dot_q4_K_q8_K_block(wr1 + b, xqb + b);
+    }
+    acc0 = quarter_warp_sum_f32(acc0, lane);
+    acc1 = quarter_warp_sum_f32(acc1, lane);
+    if (lane == 0u) {
+        if (row < out0_dim) out0[(uint64_t)tok * out0_dim + row] = acc0;
+        if (row < out1_dim) out1[(uint64_t)tok * out1_dim + row] = acc1;
+    }
+}
+
+/* Decode attention-output tail for AProjQ4:
+ *
+ *     block_out = input @ Wob(Q4_K)
+ *     out_hc    = HCPost(block_out, residual_hc, split)
+ *
+ * Keep the Q4_K dot-product and quarter-warp reduction identical to
+ * matmul_q4_K_dense_kernel.  The row-owning lane then materializes the
+ * diagnostic block output and immediately expands the same F32 value into
+ * the four HC streams.  This removes the standalone hc_expand launch without
+ * changing the Q4 accumulation order used by the non-MMQ fallback. */
+__global__ static void matmul_q4_K_hc_expand4_kernel(
+        float *out_hc,
+        float *block_out,
+        const float *residual_hc,
+        const float *split,
+        const char *w_base,
+        const cuda_block_q8_K *xq,
+        uint64_t row_bytes,
+        uint32_t xq_blocks,
+        uint32_t out_dim,
+        uint32_t n_embd) {
+    const uint32_t lane = threadIdx.x & 7u;
+    const uint32_t row_lane = threadIdx.x >> 3u;
+    const uint32_t row = blockIdx.x * 32u + row_lane;
+    if (row >= out_dim) return;
+
+    const cuda_block_q4_K *wr =
+        (const cuda_block_q4_K *)(w_base + (uint64_t)row * row_bytes);
+    float acc = 0.0f;
+    const bool vector_aligned = (((uintptr_t)w_base & 15u) == 0u);
+    for (uint32_t b = lane; b < xq_blocks; b += 8u) {
+        if (vector_aligned) {
+            dev_dot_q4_K_q8_K_block_vec(wr + b, xq + b, &acc);
+        } else {
+            acc += dev_dot_q4_K_q8_K_block(wr + b, xq + b);
+        }
+    }
+    acc = quarter_warp_sum_f32(acc, lane);
+
+    if (lane == 0u) {
+        block_out[row] = acc;
+
+        const float *post = split + 4u;
+        const float *comb = split + 8u;
+#pragma unroll
+        for (uint32_t dst_hc = 0; dst_hc < 4u; dst_hc++) {
+            float hc_acc = acc * post[dst_hc];
+#pragma unroll
+            for (uint32_t src_hc = 0; src_hc < 4u; src_hc++) {
+                hc_acc += comb[dst_hc + src_hc * 4u] *
+                          residual_hc[(uint64_t)src_hc * n_embd + row];
+            }
+            out_hc[(uint64_t)dst_hc * n_embd + row] = hc_acc;
+        }
+    }
+}
+
+/* Diagnostic row-packed HC epilogue for the canonical MMVQ/Q8_1 Q4_K path.
+ * MMVQ materializes block_out using its established activation quantizer and
+ * reduction order; one thread per embedding row then reuses that value for
+ * all four HC destinations.  Its source-level multiply/add sequence mirrors
+ * hc_expand_kernel, but full unrolling under --use_fast_math is not assumed
+ * bit-exact.  Normal decoding uses hc_expand_kernel; the oracle below is the
+ * only consumer of this candidate until device parity is proven. */
+__global__ static void q4_K_hc_expand4_rows_kernel(
+        float *out_hc,
+        const float *block_out,
+        const float *residual_hc,
+        const float *split,
+        uint32_t n_embd) {
+    const uint32_t row = blockIdx.x * blockDim.x + threadIdx.x;
+    if (row >= n_embd) return;
+
+    const float block_v = block_out[row];
+    const float *post = split + 4u;
+    const float *comb = split + 8u;
+#pragma unroll
+    for (uint32_t dst_hc = 0; dst_hc < 4u; dst_hc++) {
+        float acc = block_v * post[dst_hc];
+#pragma unroll
+        for (uint32_t src_hc = 0; src_hc < 4u; src_hc++) {
+            acc += comb[dst_hc + src_hc * 4u] *
+                   residual_hc[(uint64_t)src_hc * n_embd + row];
+        }
+        out_hc[(uint64_t)dst_hc * n_embd + row] = acc;
+    }
+}
+
+__global__ static void q4_K_attn_hc_bitwise_compare_kernel(
+        uint32_t *mismatch,
+        const float *reference,
+        const float *candidate,
+        uint64_t count) {
+    const uint64_t i = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x;
+    if (i >= count) return;
+    if (__float_as_uint(reference[i]) != __float_as_uint(candidate[i])) {
+        atomicExch(mismatch, 1u);
+    }
+}
+
+static int cuda_matmul_q4_K_tensor(
+        ds4_gpu_tensor *out,
+        const void     *model_map,
+        uint64_t        model_size,
+        uint64_t        weight_offset,
+        uint64_t        in_dim,
+        uint64_t        out_dim,
+        const ds4_gpu_tensor *x,
+        uint64_t        n_tok) {
+    if (!out || !x || !model_map || n_tok == 0) return 0;
+    if (in_dim == 0 || (in_dim % CUDA_QK_K) != 0) return 0;
+    const uint64_t blocks = in_dim / CUDA_QK_K;
+    const uint64_t row_bytes = blocks * sizeof(cuda_block_q4_K);
+    if (weight_offset > model_size || row_bytes == 0 ||
+        out_dim > UINT64_MAX / row_bytes) {
+        return 0;
+    }
+    const uint64_t weight_bytes = out_dim * row_bytes;
+    if (weight_bytes > model_size - weight_offset) return 0;
+    if (x->bytes < n_tok * in_dim * sizeof(float) ||
+        out->bytes < n_tok * out_dim * sizeof(float)) {
+        return 0;
+    }
+    const int logical_tier = ds4_tensor_device_idx(out);
+    int weight_device_resident = 0;
+    const char *wptr = cuda_resolve_weight_ptr(
+        model_map, weight_offset, weight_bytes, logical_tier,
+        "q4_K dense", &weight_device_resident);
+    if (!wptr) return 0;
+    // The 16-warp experiment is a prefill specialization.  Keep decode and
+    // speculative micro-batches on MMVQ, but make a strict prefill request
+    // fail closed even when the global MMQ selector was disabled before this
+    // call (otherwise the legacy Q8_K fallback could be measured silently).
+    const int require_q4_16warp = n_tok > 8u && cuda_env_flag_enabled(
+        "DS4_CUDA_REQUIRE_Q4_MMQ_16WARP", 0);
+    /* The scalar-token kernel below rereads every weight row for every token,
+     * making prefill scale almost linearly with batch length. MMQ tiles both
+     * axes and shares the Q4_K weights across activation columns, matching the
+     * fast Q8 prefill path while retaining native quantized arithmetic. */
+    if (in_dim <= INT_MAX && out_dim <= INT_MAX && n_tok <= INT_MAX &&
+        cuda_use_mmq()) {
+        /* MMVQ has lower setup cost for decode and speculative micro-batches;
+         * regular MMQ wins once enough token columns can share each weight
+         * tile. Both consume the GGUF Q4_K layout directly. */
+        const int rc = n_tok <= 8u
+            ? ds4_mmq_q4_K_dense_vec_with_weight_residency(
+                  wptr, (const float *)x->ptr, (float *)out->ptr,
+                  (int)out_dim, (int)n_tok, (int)in_dim,
+                  weight_device_resident,
+                  cuda_decode_stream())
+            : ds4_mmq_q4_K_dense(
+                  wptr, (const float *)x->ptr, (float *)out->ptr,
+                  (int)out_dim, (int)n_tok, (int)in_dim,
+                  cuda_decode_stream());
+        if (rc == 0) return 1;
+        fprintf(stderr,
+                "ds4: Q4_K MMQ returned %d "
+                "(in=%llu out=%llu n_tok=%llu)%s\n",
+                rc, (unsigned long long)in_dim,
+                (unsigned long long)out_dim,
+                (unsigned long long)n_tok,
+                g_cuda_test_q4_mmq_strict
+                    ? "; strict benchmark mode rejects fallback"
+                    : "; falling back");
+        if (require_q4_16warp) return 0;
+        if (g_cuda_test_q4_mmq_strict) return 0;
+        if (in_dim == 1024u && out_dim == 32768u && n_tok == 1u &&
+            getenv("DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT") != NULL) {
+            return 0;
+        }
+    }
+    if (g_cuda_test_q4_mmq_strict) {
+        fprintf(stderr,
+                "ds4: Q4_K strict benchmark mode found no MMQ dispatch "
+                "(in=%llu out=%llu n_tok=%llu)\n",
+                (unsigned long long)in_dim,
+                (unsigned long long)out_dim,
+                (unsigned long long)n_tok);
+        return 0;
+    }
+    if (require_q4_16warp) {
+        fprintf(stderr,
+                "ds4: required Q4 16-warp prefill found no MMQ dispatch "
+                "(in=%llu out=%llu n_tok=%llu)\n",
+                (unsigned long long)in_dim,
+                (unsigned long long)out_dim,
+                (unsigned long long)n_tok);
+        return 0;
+    }
+    void *tmp = cuda_tmp_alloc_on(logical_tier,
+                                  n_tok * blocks * sizeof(cuda_block_q8_K),
+                                  "q4_K dense prequant");
+    if (!tmp) return 0;
+    cuda_block_q8_K *xq = (cuda_block_q8_K *)tmp;
+    dim3 qgrid((unsigned)blocks, (unsigned)n_tok, 1);
+    q8_K_quantize_kernel<<<qgrid, 256, 0, cuda_decode_stream()>>>(
+            xq, (const float *)x->ptr,
+            (uint32_t)in_dim, (uint32_t)n_tok);
+    if (!cuda_ok(cudaGetLastError(), "q4_K dense quantize launch")) return 0;
+    dim3 grid(((unsigned)out_dim + 31u) / 32u, (unsigned)n_tok, 1);
+    matmul_q4_K_dense_kernel<<<grid, 256, 0, cuda_decode_stream()>>>(
+            (float *)out->ptr,
+            wptr,
+            xq,
+            row_bytes,
+            (uint32_t)blocks,
+            (uint32_t)out_dim,
+            (uint32_t)n_tok);
+    return cuda_ok(cudaGetLastError(), "q4_K dense matmul launch");
+}
+
+static int cuda_matmul_q4_K_pair_tensor_impl(
+        ds4_gpu_tensor *out0,
+        ds4_gpu_tensor *out1,
+        const void *model_map,
+        uint64_t model_size,
+        uint64_t weight0_offset,
+        uint64_t weight1_offset,
+        uint64_t in_dim,
+        uint64_t out0_dim,
+        uint64_t out1_dim,
+        const ds4_gpu_tensor *x,
+        uint64_t n_tok) {
+    if (!out0 || !out1 || !x || !model_map || n_tok == 0u ||
+        n_tok > INT_MAX ||
+        in_dim == 0u || (in_dim % CUDA_QK_K) != 0u ||
+        in_dim > INT_MAX || out0_dim == 0u || out1_dim == 0u ||
+        out0_dim > INT_MAX || out1_dim > INT_MAX) {
+        return 0;
+    }
+
+    const uint64_t blocks = in_dim / CUDA_QK_K;
+    if (blocks == 0u || blocks > UINT64_MAX / sizeof(cuda_block_q4_K)) {
+        return 0;
+    }
+    const uint64_t row_bytes = blocks * sizeof(cuda_block_q4_K);
+    if (out0_dim > UINT64_MAX / row_bytes ||
+        out1_dim > UINT64_MAX / row_bytes ||
+        weight0_offset > model_size || weight1_offset > model_size) {
+        return 0;
+    }
+    const uint64_t weight0_bytes = out0_dim * row_bytes;
+    const uint64_t weight1_bytes = out1_dim * row_bytes;
+    if (weight0_bytes > model_size - weight0_offset ||
+        weight1_bytes > model_size - weight1_offset ||
+        n_tok > UINT64_MAX / in_dim ||
+        n_tok * in_dim > UINT64_MAX / sizeof(float) ||
+        n_tok > UINT64_MAX / out0_dim ||
+        n_tok * out0_dim > UINT64_MAX / sizeof(float) ||
+        n_tok > UINT64_MAX / out1_dim ||
+        n_tok * out1_dim > UINT64_MAX / sizeof(float)) {
+        return 0;
+    }
+    const uint64_t x_bytes = n_tok * in_dim * sizeof(float);
+    const uint64_t out0_bytes = n_tok * out0_dim * sizeof(float);
+    const uint64_t out1_bytes = n_tok * out1_dim * sizeof(float);
+    if (x->bytes < x_bytes || out0->bytes < out0_bytes ||
+        out1->bytes < out1_bytes) {
+        return 0;
+    }
+    const uintptr_t out0_addr = (uintptr_t)out0->ptr;
+    const uintptr_t out1_addr = (uintptr_t)out1->ptr;
+    const bool outputs_overlap = out0_addr <= out1_addr
+        ? (uint64_t)(out1_addr - out0_addr) < out0_bytes
+        : (uint64_t)(out0_addr - out1_addr) < out1_bytes;
+    if (outputs_overlap) return 0;
+
+    const int logical_tier = ds4_tensor_device_idx(out0);
+    if (logical_tier < 0 || logical_tier >= g_n_gpus ||
+        ds4_tensor_device_idx(out1) != logical_tier ||
+        ds4_tensor_device_idx(x) != logical_tier) {
+        return 0;
+    }
+    const char *w0 = cuda_resolve_weight_ptr(
+        model_map, weight0_offset, weight0_bytes, logical_tier,
+        "q4_K dense pair0");
+    const char *w1 = cuda_resolve_weight_ptr(
+        model_map, weight1_offset, weight1_bytes, logical_tier,
+        "q4_K dense pair1");
+    if (!w0 || !w1) return 0;
+
+    const int gb10_canonical = cuda_q4_gb10_fast_path_enabled(
+        logical_tier, "DS4_CUDA_DISABLE_Q4_DENSE_PAIR");
+    const int require_q4_16warp = n_tok > 8u && cuda_env_flag_enabled(
+        "DS4_CUDA_REQUIRE_Q4_MMQ_16WARP", 0);
+    if (cuda_use_mmq()) {
+        /* Share one canonical Q8_1 activation across both projections:
+         * MMVQ covers decode/speculative widths and token-tiled MMQ covers
+         * prefill.  On GB10 a rejected launch fails closed to ds4.c's
+         * independent projections; the Q8_K fallback below remains the
+         * established non-GB10 rollback. */
+        const int rc = n_tok <= 8u
+            ? ds4_mmq_q4_K_dense_pair_vec(
+                  w0, w1, (const float *)x->ptr,
+                  (float *)out0->ptr, (float *)out1->ptr,
+                  (int)out0_dim, (int)out1_dim,
+                  (int)n_tok, (int)in_dim, cuda_decode_stream())
+            : ds4_mmq_q4_K_dense_pair(
+                  w0, w1, (const float *)x->ptr,
+                  (float *)out0->ptr, (float *)out1->ptr,
+                  (int)out0_dim, (int)out1_dim,
+                  (int)n_tok, (int)in_dim, cuda_decode_stream());
+        if (rc == 0) return 1;
+        fprintf(stderr,
+                "ds4: Q4_K %s pair returned %d "
+                "(in=%llu out0=%llu out1=%llu n_tok=%llu)%s\n",
+                n_tok <= 8u ? "MMVQ" : "MMQ", rc,
+                (unsigned long long)in_dim,
+                (unsigned long long)out0_dim,
+                (unsigned long long)out1_dim,
+                (unsigned long long)n_tok,
+                rc == DS4_MMQ_NOT_APPLICABLE
+                    ? (require_q4_16warp
+                           ? "; required 16-warp path rejects fallback"
+                           : g_cuda_test_q4_mmq_strict
+                           ? "; strict benchmark mode rejects fallback"
+                           : "; falling back")
+                    : "; attempted path failed closed");
+        if (rc != DS4_MMQ_NOT_APPLICABLE) return -1;
+        if (require_q4_16warp) return -1;
+        if (g_cuda_test_q4_mmq_strict) return -1;
+        if (gb10_canonical) return 0;
+    }
+    if (require_q4_16warp) {
+        fprintf(stderr,
+                "ds4: required Q4 16-warp prefill found no pair MMQ "
+                "dispatch (in=%llu out0=%llu out1=%llu n_tok=%llu)\n",
+                (unsigned long long)in_dim,
+                (unsigned long long)out0_dim,
+                (unsigned long long)out1_dim,
+                (unsigned long long)n_tok);
+        return -1;
+    }
+    if (g_cuda_test_q4_mmq_strict) {
+        fprintf(stderr,
+                "ds4: Q4_K pair strict benchmark mode found no MMQ "
+                "dispatch (in=%llu out0=%llu out1=%llu n_tok=%llu)\n",
+                (unsigned long long)in_dim,
+                (unsigned long long)out0_dim,
+                (unsigned long long)out1_dim,
+                (unsigned long long)n_tok);
+        return -1;
+    }
+
+    /* The Q8_K pair is the established decode/microbatch rollback only.
+     * For prefill, preserve DS4_CUDA_MMQ=0 and MMQ rejection semantics by
+     * returning control to the caller's two independent dense projections. */
+    if (n_tok > 8u) return 0;
+
+    if (n_tok > UINT64_MAX / blocks ||
+        n_tok * blocks > UINT64_MAX / sizeof(cuda_block_q8_K)) {
+        return 0;
+    }
+    cuda_block_q8_K *xq = (cuda_block_q8_K *)cuda_tmp_alloc_on(
+        logical_tier, n_tok * blocks * sizeof(cuda_block_q8_K),
+        "q4_K dense pair prequant");
+    if (!xq) return 0;
+
+    const dim3 qgrid((unsigned)blocks, (unsigned)n_tok, 1u);
+    q8_K_quantize_kernel<<<qgrid, 256, 0, cuda_decode_stream()>>>(
+        xq, (const float *)x->ptr, (uint32_t)in_dim, (uint32_t)n_tok);
+    if (!cuda_ok(cudaGetLastError(), "q4_K dense pair quantize launch")) {
+        return -1;
+    }
+
+    const uint64_t max_out = out0_dim > out1_dim ? out0_dim : out1_dim;
+    const dim3 grid(((unsigned)max_out + 31u) / 32u,
+                    (unsigned)n_tok, 1u);
+    matmul_q4_K_dense_pair_kernel<<<grid, 256, 0, cuda_decode_stream()>>>(
+        (float *)out0->ptr,
+        (float *)out1->ptr,
+        w0,
+        w1,
+        xq,
+        row_bytes,
+        (uint32_t)blocks,
+        (uint32_t)out0_dim,
+        (uint32_t)out1_dim,
+        (uint32_t)n_tok);
+    return cuda_ok(cudaGetLastError(), "q4_K dense pair matmul launch")
+        ? 1
+        : -1;
+}
+
+static uint64_t g_q4_attn_hc_oracle_calls;
+static uint64_t g_q4_attn_hc_oracle_epilogue_mismatches;
+static uint64_t g_q4_attn_hc_oracle_q8k_mismatches;
+static uint64_t g_q4_attn_hc_oracle_skips;
+static int g_q4_attn_hc_oracle_report_registered;
+static int g_q4_attn_hc_oracle_epilogue_reported;
+static int g_q4_attn_hc_oracle_q8k_reported;
+
+static void cuda_q4_attn_hc_oracle_report(void) {
+    fprintf(stderr,
+            "ds4: CUDA Q4 attention-output/HC oracle: "
+            "calls=%llu epilogue_mismatches=%llu "
+            "q8k_mismatches=%llu skips=%llu "
+            "(canonical output retained)\n",
+            (unsigned long long)g_q4_attn_hc_oracle_calls,
+            (unsigned long long)g_q4_attn_hc_oracle_epilogue_mismatches,
+            (unsigned long long)g_q4_attn_hc_oracle_q8k_mismatches,
+            (unsigned long long)g_q4_attn_hc_oracle_skips);
+}
+
+static void cuda_q4_attn_hc_oracle_register_report(void) {
+    if (!g_q4_attn_hc_oracle_report_registered) {
+        g_q4_attn_hc_oracle_report_registered = 1;
+        (void)atexit(cuda_q4_attn_hc_oracle_report);
+    }
+}
+
+static int cuda_q4_K_hc_expand_canonical(
+        ds4_gpu_tensor       *out_hc,
+        ds4_gpu_tensor       *block_out,
+        const void           *model_map,
+        uint64_t              model_size,
+        uint64_t              weight_offset,
+        uint64_t              in_dim,
+        uint64_t              out_dim,
+        const ds4_gpu_tensor *x,
+        const ds4_gpu_tensor *residual_hc,
+        const ds4_gpu_tensor *split,
+        int                   row_packed_epilogue) {
+    /* Reuse the exact ordinary Q4 dispatcher.  With MMQ enabled this is the
+     * canonical MMVQ/Q8_1 path (including sanitize); if MMVQ rejects the
+     * shape it retains the ordinary Q8_K materialized fallback. */
+    if (!cuda_matmul_q4_K_tensor(block_out, model_map, model_size,
+                                 weight_offset, in_dim, out_dim, x, 1u)) {
+        return 0;
+    }
+    if (row_packed_epilogue) {
+        q4_K_hc_expand4_rows_kernel<<<((unsigned)out_dim + 255u) / 256u,
+                                        256, 0, cuda_decode_stream()>>>(
+            (float *)out_hc->ptr,
+            (const float *)block_out->ptr,
+            (const float *)residual_hc->ptr,
+            (const float *)split->ptr,
+            (uint32_t)out_dim);
+    } else {
+        /* Oracle reference: byte-for-byte the ordinary one-token
+         * ds4_gpu_hc_expand_tensor launch and argument layout. */
+        const uint64_t n_elem = 4u * out_dim;
+        const float *base = (const float *)split->ptr;
+        hc_expand_kernel<<<(unsigned)((n_elem + 255u) / 256u),
+                           256, 0, cuda_decode_stream()>>>(
+            (float *)out_hc->ptr,
+            (const float *)block_out->ptr,
+            (const float *)block_out->ptr,
+            (const float *)block_out->ptr,
+            (const float *)residual_hc->ptr,
+            base + 4u,
+            base + 8u,
+            (uint32_t)out_dim, 4u, 1u, 4u, 16u, 0, 0);
+    }
+    return cuda_ok(cudaGetLastError(),
+                   "q4_K canonical MMVQ hc expand launch");
+}
+
+static int cuda_q4_K_hc_expand_q8k_launch(
+        ds4_gpu_tensor       *out_hc,
+        ds4_gpu_tensor       *block_out,
+        const ds4_gpu_tensor *x,
+        const ds4_gpu_tensor *residual_hc,
+        const ds4_gpu_tensor *split,
+        const char           *wptr,
+        cuda_block_q8_K      *xq,
+        uint64_t              row_bytes,
+        uint32_t              blocks,
+        uint32_t              out_dim,
+        uint32_t              n_embd) {
+    q8_K_quantize_kernel<<<blocks, 256, 0, cuda_decode_stream()>>>(
+        xq, (const float *)x->ptr, blocks * CUDA_QK_K, 1u);
+    if (!cuda_ok(cudaGetLastError(),
+                 "q4_K Q8_K hc expand quantize launch")) {
+        return 0;
+    }
+
+    matmul_q4_K_hc_expand4_kernel<<<(out_dim + 31u) / 32u,
+                                      256, 0, cuda_decode_stream()>>>(
+        (float *)out_hc->ptr,
+        (float *)block_out->ptr,
+        (const float *)residual_hc->ptr,
+        (const float *)split->ptr,
+        wptr,
+        xq,
+        row_bytes,
+        blocks,
+        out_dim,
+        n_embd);
+    return cuda_ok(cudaGetLastError(), "q4_K Q8_K hc expand launch");
+}
+
+extern "C" int ds4_gpu_matmul_q4_K_hc_expand_available(void) {
+    if (getenv("DS4_CUDA_DISABLE_Q4_ATTN_OUT_HC_FUSE") != NULL) return 0;
+    if (cuda_env_flag_enabled("DS4_CUDA_Q4_ATTN_OUT_HC_ORACLE", 0)) {
+        /* Always emit a summary when the oracle was requested, including a
+         * zero-call summary that makes an ineligible/unused A/B visible. */
+        cuda_q4_attn_hc_oracle_register_report();
+    }
+    /* On GB10 the default fused call path keeps both the ordinary Q4_K MMVQ
+     * result and the ordinary HC epilogue.  This preserves decode-graph island
+     * B without claiming that the row-packed epilogue is bit-exact.  The
+     * row-packed and Q8_K candidates remain oracle-only diagnostics until
+     * device tests prove their numerical contract.  The existing disable
+     * variable (plus DS4_CUDA_NO_Q4_GB10_FAST) is a complete rollback to the
+     * caller's separate B projection + HC expansion. */
+    if (cuda_q4_gb10_fast_path_enabled(
+            g_current_logical_tier, NULL)) {
+        return 1;
+    }
+    return !cuda_use_mmq() ||
+           cuda_env_flag_enabled("DS4_CUDA_ENABLE_Q4_ATTN_OUT_HC_FUSE", 0) ||
+           cuda_env_flag_enabled(
+               "DS4_CUDA_Q4_ATTN_OUT_HC_Q8K_EXPERIMENT", 0) ||
+           cuda_env_flag_enabled("DS4_CUDA_Q4_ATTN_OUT_HC_ORACLE", 0);
+}
+
+extern "C" int ds4_gpu_matmul_q4_K_hc_expand_tensor(
+        ds4_gpu_tensor       *out_hc,
+        ds4_gpu_tensor       *block_out,
+        const void           *model_map,
+        uint64_t              model_size,
+        uint64_t              weight_offset,
+        uint64_t              in_dim,
+        uint64_t              out_dim,
+        const ds4_gpu_tensor *x,
+        const ds4_gpu_tensor *residual_hc,
+        const ds4_gpu_tensor *split,
+        uint32_t              n_embd,
+        uint32_t              n_hc) {
+    if (!ds4_gpu_matmul_q4_K_hc_expand_available() ||
+        !out_hc || !block_out || !model_map || !x || !residual_hc || !split ||
+        in_dim == 0u || (in_dim % CUDA_QK_K) != 0u ||
+        out_dim == 0u || out_dim != n_embd || (out_dim & 1u) != 0u ||
+        n_hc != 4u || in_dim > UINT32_MAX || out_dim > UINT32_MAX) {
+        return 0;
+    }
+
+    const uint64_t blocks = in_dim / CUDA_QK_K;
+    if (blocks == 0u || blocks > UINT64_MAX / sizeof(cuda_block_q4_K)) {
+        return 0;
+    }
+    const uint64_t row_bytes = blocks * sizeof(cuda_block_q4_K);
+    if (out_dim > UINT64_MAX / row_bytes || weight_offset > model_size) {
+        return 0;
+    }
+    const uint64_t weight_bytes = out_dim * row_bytes;
+    if (weight_bytes > model_size - weight_offset ||
+        in_dim > UINT64_MAX / sizeof(float) ||
+        out_dim > UINT64_MAX / sizeof(float)) {
+        return 0;
+    }
+    const uint64_t x_bytes = in_dim * sizeof(float);
+    const uint64_t embd_bytes = out_dim * sizeof(float);
+    const uint64_t hc_bytes = 4u * embd_bytes;
+    const uint64_t split_bytes = 24u * sizeof(float);
+    if (x->bytes < x_bytes || block_out->bytes < embd_bytes ||
+        residual_hc->bytes < hc_bytes || split->bytes < split_bytes ||
+        out_hc->bytes < hc_bytes) {
+        return 0;
+    }
+
+    const int logical_tier = ds4_tensor_device_idx(out_hc);
+    if (ds4_tensor_device_idx(block_out) != logical_tier ||
+        ds4_tensor_device_idx(x) != logical_tier ||
+        ds4_tensor_device_idx(residual_hc) != logical_tier ||
+        ds4_tensor_device_idx(split) != logical_tier) {
+        return 0;
+    }
+    const char *wptr = cuda_resolve_weight_ptr(
+        model_map, weight_offset, weight_bytes, logical_tier,
+        "q4_K hc expand");
+    if (!wptr) return 0;
+
+    const int q8k_experiment =
+        !cuda_use_mmq() ||
+        cuda_env_flag_enabled(
+            "DS4_CUDA_Q4_ATTN_OUT_HC_Q8K_EXPERIMENT", 0);
+    const int oracle = cuda_env_flag_enabled(
+        "DS4_CUDA_Q4_ATTN_OUT_HC_ORACLE", 0);
+
+    if (!q8k_experiment && !oracle) {
+        return cuda_q4_K_hc_expand_canonical(
+            out_hc, block_out, model_map, model_size, weight_offset,
+            in_dim, out_dim, x, residual_hc, split, 0);
+    }
+
+    /* The oracle is deliberately diagnostic and fail-closed: compute and
+     * retain canonical output first, then compare the true one-kernel Q8_K
+     * candidate bit-for-bit.  It synchronizes only when explicitly enabled
+     * and never attempts a host read while the decode stream is captured. */
+    if (oracle) {
+        cuda_q4_attn_hc_oracle_register_report();
+        if (!cuda_q4_K_hc_expand_canonical(
+                out_hc, block_out, model_map, model_size, weight_offset,
+                in_dim, out_dim, x, residual_hc, split, 0)) {
+            return 0;
+        }
+
+        cudaStream_t stream = cuda_decode_stream();
+        cudaStreamCaptureStatus capture = cudaStreamCaptureStatusNone;
+        const cudaError_t capture_err =
+            cudaStreamIsCapturing(stream, &capture);
+        if (capture_err != cudaSuccess ||
+            capture != cudaStreamCaptureStatusNone) {
+            (void)cudaGetLastError();
+            g_q4_attn_hc_oracle_skips++;
+            return 1;
+        }
+
+        const uint64_t xq_bytes =
+            blocks * sizeof(cuda_block_q8_K);
+        const uint64_t candidate_block_off =
+            (xq_bytes + 255u) & ~255ull;
+        const uint64_t candidate_q8k_hc_off =
+            (candidate_block_off + embd_bytes + 255u) & ~255ull;
+        const uint64_t candidate_epilogue_hc_off =
+            (candidate_q8k_hc_off + hc_bytes + 255u) & ~255ull;
+        const uint64_t mismatch_off =
+            (candidate_epilogue_hc_off + hc_bytes + 255u) & ~255ull;
+        const uint64_t scratch_bytes =
+            mismatch_off + 2u * sizeof(uint32_t);
+        unsigned char *scratch = (unsigned char *)cuda_tmp_alloc_on(
+            logical_tier, scratch_bytes,
+            "q4_K attention-output HC oracle");
+        if (!scratch) {
+            g_q4_attn_hc_oracle_skips++;
+            return 1;
+        }
+
+        cuda_block_q8_K *candidate_xq =
+            (cuda_block_q8_K *)scratch;
+        ds4_gpu_tensor candidate_block = *block_out;
+        candidate_block.ptr = scratch + candidate_block_off;
+        candidate_block.bytes = embd_bytes;
+        candidate_block.owner = 0;
+        ds4_gpu_tensor candidate_q8k_hc = *out_hc;
+        candidate_q8k_hc.ptr = scratch + candidate_q8k_hc_off;
+        candidate_q8k_hc.bytes = hc_bytes;
+        candidate_q8k_hc.owner = 0;
+        ds4_gpu_tensor candidate_epilogue_hc = *out_hc;
+        candidate_epilogue_hc.ptr = scratch + candidate_epilogue_hc_off;
+        candidate_epilogue_hc.bytes = hc_bytes;
+        candidate_epilogue_hc.owner = 0;
+        uint32_t *mismatch_device =
+            (uint32_t *)(scratch + mismatch_off);
+
+        if (!cuda_ok(cudaMemsetAsync(mismatch_device, 0,
+                                     2u * sizeof(uint32_t),
+                                     stream),
+                     "clear q4_K attention-output HC oracle") ||
+            !cuda_q4_K_hc_expand_q8k_launch(
+                &candidate_q8k_hc, &candidate_block, x, residual_hc, split,
+                wptr, candidate_xq, row_bytes, (uint32_t)blocks,
+                (uint32_t)out_dim, n_embd)) {
+            g_q4_attn_hc_oracle_skips++;
+            return 1;
+        }
+
+        q4_K_hc_expand4_rows_kernel<<<((unsigned)out_dim + 255u) / 256u,
+                                        256, 0, stream>>>(
+            (float *)candidate_epilogue_hc.ptr,
+            (const float *)block_out->ptr,
+            (const float *)residual_hc->ptr,
+            (const float *)split->ptr,
+            (uint32_t)out_dim);
+
+        q4_K_attn_hc_bitwise_compare_kernel
+            <<<(embd_bytes / sizeof(float) + 255u) / 256u,
+               256, 0, stream>>>(
+                mismatch_device,
+                (const float *)block_out->ptr,
+                (const float *)candidate_block.ptr,
+                embd_bytes / sizeof(float));
+        q4_K_attn_hc_bitwise_compare_kernel
+            <<<(hc_bytes / sizeof(float) + 255u) / 256u,
+               256, 0, stream>>>(
+                mismatch_device,
+                (const float *)out_hc->ptr,
+                (const float *)candidate_q8k_hc.ptr,
+                hc_bytes / sizeof(float));
+        q4_K_attn_hc_bitwise_compare_kernel
+            <<<(hc_bytes / sizeof(float) + 255u) / 256u,
+               256, 0, stream>>>(
+                mismatch_device + 1u,
+                (const float *)out_hc->ptr,
+                (const float *)candidate_epilogue_hc.ptr,
+                hc_bytes / sizeof(float));
+        if (!cuda_ok(cudaGetLastError(),
+                     "q4_K attention-output HC oracle compare launch")) {
+            return 0;
+        }
+
+        uint32_t mismatch_host[2] = {0u, 0u};
+        if (!cuda_ok(cudaMemcpyAsync(mismatch_host, mismatch_device,
+                                     sizeof(mismatch_host),
+                                     cudaMemcpyDeviceToHost, stream),
+                     "read q4_K attention-output HC oracle") ||
+            !cuda_ok(cudaStreamSynchronize(stream),
+                     "synchronize q4_K attention-output HC oracle")) {
+            return 0;
+        }
+        g_q4_attn_hc_oracle_calls++;
+        if (mismatch_host[0] != 0u) {
+            g_q4_attn_hc_oracle_q8k_mismatches++;
+            if (!g_q4_attn_hc_oracle_q8k_reported) {
+                g_q4_attn_hc_oracle_q8k_reported = 1;
+                fprintf(stderr,
+                        "ds4: CUDA Q4 attention-output/HC Q8_K oracle "
+                        "found a bitwise mismatch; retaining canonical "
+                        "MMVQ/Q8_1 output\n");
+            }
+        }
+        if (mismatch_host[1] != 0u) {
+            g_q4_attn_hc_oracle_epilogue_mismatches++;
+            if (!g_q4_attn_hc_oracle_epilogue_reported) {
+                g_q4_attn_hc_oracle_epilogue_reported = 1;
+                fprintf(stderr,
+                        "ds4: CUDA Q4 attention-output/HC row-packed "
+                        "epilogue oracle found a bitwise mismatch; "
+                        "retaining the ordinary HC expansion\n");
+            }
+        }
+        return 1;
+    }
+
+    cuda_block_q8_K *xq = (cuda_block_q8_K *)cuda_tmp_alloc_on(
+        logical_tier, blocks * sizeof(cuda_block_q8_K),
+        "q4_K hc expand prequant");
+    if (xq && cuda_q4_K_hc_expand_q8k_launch(
+                    out_hc, block_out, x, residual_hc, split,
+                    wptr, xq, row_bytes, (uint32_t)blocks,
+                    (uint32_t)out_dim, n_embd)) {
+        return 1;
+    }
+
+    /* Allocation/launch rejection of the optional candidate must not make
+     * decoding unavailable.  Re-materialize through the ordinary dispatcher
+     * and finish with the ordinary HC epilogue. */
+    return cuda_q4_K_hc_expand_canonical(
+        out_hc, block_out, model_map, model_size, weight_offset,
+        in_dim, out_dim, x, residual_hc, split, 0);
+}
+
+__global__ static void matmul_q4_K_kslice_kernel(
+        float *out,
+        const char *w_base,
+        const cuda_block_q8_K *xq,
+        uint64_t full_row_bytes,
+        uint32_t block0,
+        uint32_t block_count,
+        uint32_t out_dim) {
+    const uint32_t lane = threadIdx.x & 7u;
+    const uint32_t row_lane = threadIdx.x >> 3u;
+    const uint32_t row = blockIdx.x * 32u + row_lane;
+    if (row >= out_dim) return;
+    const cuda_block_q4_K *wr = (const cuda_block_q4_K *)(
+        w_base + (uint64_t)row * full_row_bytes) + block0;
+    float acc = 0.0f;
+    for (uint32_t b = lane; b < block_count; b += 8u) {
+        acc += dev_dot_q4_K_q8_K_block(wr + b, xq + b);
+    }
+    acc = quarter_warp_sum_f32(acc, lane);
+    if (lane == 0) out[row] = acc;
+}
+
+static int cuda_matmul_q4_K_kslice_tensor(
+        ds4_gpu_tensor *out, const void *model_map, uint64_t model_size,
+        uint64_t weight_offset, uint64_t full_in_dim, uint64_t k_off,
+        uint64_t k_cnt, uint64_t out_dim, const ds4_gpu_tensor *x,
+        uint64_t x_elem_off) {
+    if (!out || !x || !model_map || full_in_dim == 0 || k_cnt == 0 ||
+        full_in_dim > UINT32_MAX || k_cnt > UINT32_MAX ||
+        out_dim > UINT32_MAX || out_dim > UINT64_MAX / sizeof(float) ||
+        (full_in_dim % CUDA_QK_K) != 0u ||
+        (k_off % CUDA_QK_K) != 0u || (k_cnt % CUDA_QK_K) != 0u ||
+        k_off > full_in_dim || k_cnt > full_in_dim - k_off ||
+        x_elem_off > x->bytes / sizeof(float) ||
+        k_cnt > x->bytes / sizeof(float) - x_elem_off ||
+        out->bytes < out_dim * sizeof(float)) {
+        return 0;
+    }
+    const uint64_t full_blocks = full_in_dim / CUDA_QK_K;
+    const uint64_t slice_blocks = k_cnt / CUDA_QK_K;
+    if (full_blocks > UINT64_MAX / sizeof(cuda_block_q4_K)) return 0;
+    const uint64_t row_bytes = full_blocks * sizeof(cuda_block_q4_K);
+    if (out_dim > UINT64_MAX / row_bytes || weight_offset > model_size) return 0;
+    const uint64_t weight_bytes = out_dim * row_bytes;
+    if (weight_bytes > model_size - weight_offset) return 0;
+    const int logical_tier = ds4_tensor_device_idx(out);
+    const char *wptr = cuda_resolve_weight_ptr(
+        model_map, weight_offset, weight_bytes, logical_tier, "q4_K kslice");
+    if (!wptr) return 0;
+    cuda_block_q8_K *xq = (cuda_block_q8_K *)cuda_tmp_alloc_on(
+        logical_tier, slice_blocks * sizeof(cuda_block_q8_K),
+        "q4_K kslice prequant");
+    if (!xq) return 0;
+    const float *xptr = (const float *)x->ptr + x_elem_off;
+    q8_K_quantize_kernel<<<(unsigned)slice_blocks, 256>>>(
+        xq, xptr, (uint32_t)k_cnt, 1u);
+    if (!cuda_ok(cudaGetLastError(), "q4_K kslice quantize launch")) return 0;
+    matmul_q4_K_kslice_kernel<<<((unsigned)out_dim + 31u) / 32u, 256>>>(
+        (float *)out->ptr, wptr, xq, row_bytes,
+        (uint32_t)(k_off / CUDA_QK_K), (uint32_t)slice_blocks,
+        (uint32_t)out_dim);
+    return cuda_ok(cudaGetLastError(), "q4_K kslice matmul launch");
+}
+
 extern "C" int ds4_gpu_matmul_quant_tensor(
         ds4_gpu_tensor       *out,
         const void             *model_map,
@@ -32455,12 +41271,15 @@ extern "C" int ds4_gpu_matmul_quant_tensor(
         return ds4_gpu_matmul_q8_0_tensor(out, model_map, model_size,
                                           weight_offset, in_dim, out_dim,
                                           x, n_tok);
+    case 12u:  /* Q4_K (AProjQ4 dense attention projections) */
+        return cuda_matmul_q4_K_tensor(out, model_map, model_size,
+                                       weight_offset, in_dim, out_dim,
+                                       x, n_tok);
     case 1u:   /* F16 */
         return ds4_gpu_matmul_f16_tensor(out, model_map, model_size,
                                          weight_offset, in_dim, out_dim,
                                          x, n_tok);
     case 10u:  /* Q2_K */
-    case 12u:  /* Q4_K */
     case 16u:  /* IQ2_XXS */
     case 39u:  /* MXFP4 */
         return cuda_matmul_mmq_dense_quant(
@@ -32491,7 +41310,8 @@ extern "C" int ds4_gpu_routed_moe_set_selected_override(const int32_t *selected,
 }
 
 extern "C" void ds4_gpu_set_glm_streaming_prefill_full_layer(bool enabled) {
-    (void)enabled;   /* SSD streaming is not used on the CUDA backend */
+    /* CUDA streams selected experts rather than pinning whole routed layers. */
+    (void)enabled;
 }
 
 extern "C" void ds4_gpu_set_glm_mtp_verify_mode(bool enabled) {
@@ -32735,93 +41555,1172 @@ extern "C" int ds4_gpu_shared_gate_up_swiglu_q8_0_rows_tensor(
             }
         }
     }
-    if (n_tok == 1 &&
-        cuda_matmul_q8_0_aligned_pair_try(
-            gate, up, model_map, model_size,
-            gate_offset, up_offset, in_dim, out_dim, out_dim, x) > 0) {
-        return ds4_gpu_swiglu_tensor(mid, gate, up,
-                                     (uint32_t)out_dim, clamp, 1.0f);
+    if (n_tok == 1 &&
+        cuda_matmul_q8_0_aligned_pair_try(
+            gate, up, model_map, model_size,
+            gate_offset, up_offset, in_dim, out_dim, out_dim, x) > 0) {
+        return ds4_gpu_swiglu_tensor(mid, gate, up,
+                                     (uint32_t)out_dim, clamp, 1.0f);
+    }
+    if (n_tok == 1 && (in_dim & 31u) == 0u &&
+        !getenv("DS4_GLM_SHARED_SPLIT") &&
+        mid->bytes >= out_dim * sizeof(float) &&
+        x->bytes >= in_dim * sizeof(float)) {
+        const uint64_t row_bytes = (in_dim / 32u) * 34u;
+        const uint64_t wb = out_dim * row_bytes;
+        if (gate_offset <= model_size && wb <= model_size - gate_offset &&
+            up_offset <= model_size && wb <= model_size - up_offset) {
+            const int logical_tier = cuda_current_tier();
+            const char *gw = cuda_resolve_weight_ptr(model_map, gate_offset,
+                    wb, logical_tier, "glm_shared_gate");
+            const char *uw = cuda_resolve_weight_ptr(model_map, up_offset,
+                    wb, logical_tier, "glm_shared_up");
+            if (gw && uw) {
+                const uint32_t warps = 8u;
+                const uint32_t sh = (uint32_t)in_dim * sizeof(float);
+                glm_shared_gate_up_swiglu_one_kernel
+                        <<<(unsigned)((out_dim + warps - 1u) / warps),
+                           warps * 32u, sh>>>(
+                        (float *)mid->ptr, gw, uw, (const float *)x->ptr,
+                        (uint32_t)in_dim, (uint32_t)out_dim, clamp);
+                return cuda_ok(cudaGetLastError(), "glm shared swiglu one");
+            }
+        }
+    }
+    if (!ds4_gpu_matmul_q8_0_tensor(gate, model_map, model_size, gate_offset,
+                                    in_dim, out_dim, x, n_tok) ||
+        !ds4_gpu_matmul_q8_0_tensor(up, model_map, model_size, up_offset,
+                                    in_dim, out_dim, x, n_tok)) {
+        return 0;
+    }
+    return ds4_gpu_swiglu_tensor(mid, gate, up,
+                                 (uint32_t)(out_dim * n_tok), clamp, 1.0f);
+}
+
+extern "C" int ds4_gpu_shared_mid_swiglu_q8_0_tensor(
+        ds4_gpu_tensor       *mid,
+        const void             *model_map,
+        uint64_t                model_size,
+        uint64_t                gate_offset,
+        uint64_t                up_offset,
+        uint64_t                in_dim,
+        uint64_t                out_dim,
+        const ds4_gpu_tensor *x,
+        float                   clamp) {
+    static ds4_gpu_tensor *gu_scratch[DS4_MAX_GPUS][2] = {{0}};
+    const int dev = cuda_current_tier();
+    const uint64_t need = out_dim * sizeof(float);
+    for (int i = 0; i < 2; i++) {
+        if (!gu_scratch[dev][i] || gu_scratch[dev][i]->bytes < need) {
+            if (gu_scratch[dev][i]) ds4_gpu_tensor_free(gu_scratch[dev][i]);
+            gu_scratch[dev][i] = ds4_gpu_tensor_alloc(need);
+        }
+        if (!gu_scratch[dev][i]) return 0;
+    }
+    return ds4_gpu_shared_gate_up_swiglu_q8_0_rows_tensor(
+            gu_scratch[dev][0], gu_scratch[dev][1], mid,
+            model_map, model_size, gate_offset, up_offset,
+            in_dim, out_dim, x, 1, clamp);
+}
+
+extern "C" int ds4_gpu_signal_selected_readback_ready(uint64_t *event_value) {
+    if (event_value) *event_value = 1;
+    return cuda_ok(cudaDeviceSynchronize(), "selected readback signal");
+}
+
+extern "C" int ds4_gpu_cuda_stream_selected_set_owner_device(void) {
+    if (g_n_gpus != 1) return 0;
+    const int owner = g_stream_selected_event_owner_device >= 0 ?
+        g_stream_selected_event_owner_device : g_gpu[0].device_id;
+    if (owner != g_gpu[0].device_id) return 0;
+    const cudaError_t err = cudaSetDevice(owner);
+    if (err != cudaSuccess) {
+        fprintf(stderr,
+                "ds4: CUDA selected event owner-device switch to %d failed: %s\n",
+                owner, cudaGetErrorString(err));
+        (void)cudaGetLastError();
+        return 0;
+    }
+    return 1;
+}
+
+extern "C" int ds4_cuda_test_stream_selected_owner_device(void) {
+    if (g_n_gpus != 1) return 0;
+    int saved_device = -1;
+    int device_count = 0;
+    if (cudaGetDevice(&saved_device) != cudaSuccess ||
+        cudaGetDeviceCount(&device_count) != cudaSuccess) {
+        (void)cudaGetLastError();
+        return 0;
+    }
+    const int saved_logical_tier = g_current_logical_tier;
+    const int owner = g_gpu[0].device_id;
+    int wrong_device = owner;
+    for (int device = 0; device < device_count; device++) {
+        if (device != owner) {
+            wrong_device = device;
+            break;
+        }
+    }
+    /* Reproduce the service-thread hazard: the process-global logical cache
+     * claims tier 0 even though this thread is current on another physical
+     * device.  The owner setter must still issue cudaSetDevice(owner). */
+    g_current_logical_tier = 0;
+    int ok = cudaSetDevice(wrong_device) == cudaSuccess &&
+             ds4_gpu_cuda_stream_selected_set_owner_device() != 0;
+    int got_device = -1;
+    if (ok) ok = cudaGetDevice(&got_device) == cudaSuccess &&
+                 got_device == owner;
+    if (saved_device >= 0) (void)cudaSetDevice(saved_device);
+    g_current_logical_tier = saved_logical_tier;
+    if (!ok) (void)cudaGetLastError();
+    return ok;
+}
+
+/* Event-only boundary used by the DeepSeek SSD selected-expert worker.  It is
+ * intentionally separate from the compatibility API above: callers which do
+ * not opt in retain its full-device synchronization semantics. */
+extern "C" int ds4_gpu_signal_selected_readback_ready_async(
+        uint64_t *event_value) {
+    if (event_value) *event_value = 0;
+    if (!event_value ||
+        !ds4_gpu_cuda_stream_selected_event_pipeline_enabled() ||
+        g_n_gpus != 1 ||
+        !ds4_gpu_cuda_stream_selected_set_owner_device() ||
+        !cuda_stream_selected_event_pipeline_ensure()) {
+        (void)cudaGetLastError();
+        return 0;
+    }
+
+    const cudaStream_t stream = cuda_decode_stream();
+    cudaStreamCaptureStatus capture_status = cudaStreamCaptureStatusNone;
+    cudaError_t err = cudaStreamIsCapturing(stream, &capture_status);
+    if (err != cudaSuccess || capture_status != cudaStreamCaptureStatusNone) {
+        if (err != cudaSuccess) (void)cudaGetLastError();
+        return 0;
+    }
+    err = cudaEventRecord(g_stream_selected_compute_ready_event, stream);
+    if (err != cudaSuccess) {
+        fprintf(stderr,
+                "ds4: CUDA selected compute-ready event record failed: %s\n",
+                cudaGetErrorString(err));
+        (void)cudaGetLastError();
+        return 0;
+    }
+    uint64_t value = ++g_stream_selected_compute_event_value;
+    if (value == 0) value = ++g_stream_selected_compute_event_value;
+    *event_value = value;
+    g_stream_selected_event_signals.fetch_add(1, std::memory_order_relaxed);
+    return 1;
+}
+
+extern "C" int ds4_gpu_stream_expert_cache_begin_selected_load(
+        const ds4_gpu_stream_expert_table *table,
+        const int32_t                     *selected_ids,
+        uint32_t                           n_selected) {
+    return cuda_stream_selected_cache_begin_load(table, selected_ids,
+                                                 n_selected);
+}
+
+/* Tri-state result: 1 published (event_value may be zero for an oracle),
+ * 0 failed before enqueue and is safe for the caller's legacy retry, -1
+ * failed after enqueue and must fail closed. */
+extern "C" int ds4_gpu_stream_expert_cache_begin_selected_load_async(
+        const ds4_gpu_stream_expert_table *table,
+        const int32_t                     *selected_ids,
+        uint32_t                           n_selected,
+        uint64_t                          *upload_event_value) {
+    if (upload_event_value) *upload_event_value = 0;
+    if (!upload_event_value ||
+        !ds4_gpu_cuda_stream_selected_event_pipeline_enabled() ||
+        g_n_gpus != 1 ||
+        !ds4_gpu_cuda_stream_selected_set_owner_device()) {
+        (void)cudaGetLastError();
+        return 0;
+    }
+    int submitted = 0;
+    const int ok = cuda_stream_selected_cache_begin_load_impl(
+        table, selected_ids, n_selected, upload_event_value, &submitted);
+    if (ok) return 1;
+    return submitted ? -1 : 0;
+}
+
+static int cuda_stream_expert_persistent_align256(
+        uint64_t value, uint64_t *out) {
+    if (!out || value > UINT64_MAX - 255u) return 0;
+    *out = (value + 255u) & ~UINT64_C(255);
+    return 1;
+}
+
+static int cuda_stream_expert_persistent_arena_layout(
+        uint32_t capacity,
+        uint64_t gate_expert_bytes,
+        uint64_t down_expert_bytes,
+        uint64_t *up_offset,
+        uint64_t *down_offset,
+        uint64_t *total_bytes) {
+    uint64_t gate_plane = 0;
+    uint64_t down_plane = 0;
+    uint64_t after_up = 0;
+    uint64_t after_down = 0;
+    if (!up_offset || !down_offset || !total_bytes || capacity == 0 ||
+        gate_expert_bytes == 0 || down_expert_bytes == 0 ||
+        !cuda_stream_expert_persistent_mul_u64(
+            capacity, gate_expert_bytes, &gate_plane) ||
+        !cuda_stream_expert_persistent_mul_u64(
+            capacity, down_expert_bytes, &down_plane) ||
+        !cuda_stream_expert_persistent_align256(gate_plane, up_offset) ||
+        !cuda_stream_expert_persistent_add_u64(
+            *up_offset, gate_plane, &after_up) ||
+        !cuda_stream_expert_persistent_align256(after_up, down_offset) ||
+        !cuda_stream_expert_persistent_add_u64(
+            *down_offset, down_plane, &after_down) ||
+        !cuda_stream_expert_persistent_align256(
+            after_down, total_bytes) ||
+        *total_bytes > SIZE_MAX) {
+        return 0;
+    }
+    return *up_offset >= gate_plane && *down_offset >= after_up &&
+           *total_bytes >= after_down;
+}
+
+static int cuda_stream_expert_persistent_arena_release_locked(
+        int reset_class) {
+    cuda_stream_expert_persistent_arena *arena =
+        &g_stream_expert_persistent_arena;
+    const uint32_t configured_budget = arena->configured_budget;
+    /* The engine seeds the majority total before installing a new model map.
+     * Preserve that startup pin while resetting the learned exact split. */
+    const uint64_t configured_expert_bytes =
+        arena->configured_expert_bytes;
+    const uint64_t gate_expert_bytes = reset_class ? 0 :
+        arena->gate_expert_bytes;
+    const uint64_t down_expert_bytes = reset_class ? 0 :
+        arena->down_expert_bytes;
+    int previous_device = -1;
+    const int has_storage = arena->base || arena->remap;
+    if (has_storage) {
+        (void)cudaGetDevice(&previous_device);
+        /* The arena is not a transient-cache alias, but it shares the upload
+         * and consumer epoch boundary which its future loader will use. */
+        int drained = cuda_stream_selected_consume_drain() &&
+                      cuda_stream_selected_upload_drain_checked();
+        if (!drained) {
+            drained = cudaSetDevice(arena->owner_device) == cudaSuccess &&
+                      cudaDeviceSynchronize() == cudaSuccess;
+            if (!drained) (void)cudaGetLastError();
+        }
+        if (!drained ||
+            cudaSetDevice(arena->owner_device) != cudaSuccess) {
+            (void)cudaGetLastError();
+            arena->poisoned = 1;
+            g_stream_expert_persistent_arena_failures.fetch_add(
+                1, std::memory_order_relaxed);
+            if (previous_device >= 0) (void)cudaSetDevice(previous_device);
+            return 0;
+        }
+        if (arena->remap && cudaFree(arena->remap) != cudaSuccess) {
+            (void)cudaGetLastError();
+            arena->poisoned = 1;
+            g_stream_expert_persistent_arena_failures.fetch_add(
+                1, std::memory_order_relaxed);
+            if (previous_device >= 0) (void)cudaSetDevice(previous_device);
+            return 0;
+        }
+        arena->remap = NULL;
+        arena->remap_capacity = 0;
+        if (arena->base && cudaFree(arena->base) != cudaSuccess) {
+            (void)cudaGetLastError();
+            arena->poisoned = 1;
+            g_stream_expert_persistent_arena_failures.fetch_add(
+                1, std::memory_order_relaxed);
+            if (previous_device >= 0) (void)cudaSetDevice(previous_device);
+            return 0;
+        }
+        g_stream_expert_persistent_arena_releases.fetch_add(
+            1, std::memory_order_relaxed);
+    }
+    cuda_stream_expert_persistent_state_clear(
+        &g_stream_expert_persistent_state);
+    memset(arena, 0, sizeof(*arena));
+    arena->owner_device = -1;
+    arena->configured_budget = configured_budget;
+    arena->configured_expert_bytes = configured_expert_bytes;
+    arena->gate_expert_bytes = gate_expert_bytes;
+    arena->down_expert_bytes = down_expert_bytes;
+    if (previous_device >= 0) (void)cudaSetDevice(previous_device);
+    return 1;
+}
+
+static int cuda_stream_expert_persistent_arena_release(int reset_class) {
+    std::lock_guard<std::mutex> lock(
+        g_stream_expert_persistent_arena_mutex);
+    return cuda_stream_expert_persistent_arena_release_locked(reset_class);
+}
+
+static int cuda_stream_expert_persistent_arena_ensure_locked(void) {
+    cuda_stream_expert_persistent_arena *arena =
+        &g_stream_expert_persistent_arena;
+    if (arena->poisoned || g_n_gpus != 1 ||
+        arena->configured_budget == 0 ||
+        arena->gate_expert_bytes == 0 || arena->down_expert_bytes == 0) {
+        return 0;
+    }
+    uint64_t up_offset = 0;
+    uint64_t down_offset = 0;
+    uint64_t total_bytes = 0;
+    if (!cuda_stream_expert_persistent_arena_layout(
+            arena->configured_budget, arena->gate_expert_bytes,
+            arena->down_expert_bytes, &up_offset, &down_offset,
+            &total_bytes)) {
+        g_stream_expert_persistent_arena_failures.fetch_add(
+            1, std::memory_order_relaxed);
+        return 0;
+    }
+    const int owner = g_gpu[0].device_id;
+    if (arena->base && arena->owner_device == owner &&
+        arena->capacity == arena->configured_budget &&
+        arena->bytes == total_bytes && arena->up_offset == up_offset &&
+        arena->down_offset == down_offset) {
+        g_stream_expert_persistent_arena_reuses.fetch_add(
+            1, std::memory_order_relaxed);
+        return 1;
+    }
+    if (!cuda_stream_expert_persistent_arena_release_locked(0)) return 0;
+    int previous_device = -1;
+    (void)cudaGetDevice(&previous_device);
+    void *base = NULL;
+    cudaError_t err = cudaSetDevice(owner);
+    if (err == cudaSuccess) err = cudaMalloc(&base, (size_t)total_bytes);
+    if (err != cudaSuccess || !base ||
+        (((uintptr_t)base) & 255u) != 0u) {
+        if (base) (void)cudaFree(base);
+        (void)cudaGetLastError();
+        g_stream_expert_persistent_arena_failures.fetch_add(
+            1, std::memory_order_relaxed);
+        if (previous_device >= 0) (void)cudaSetDevice(previous_device);
+        return 0;
+    }
+    arena->base = base;
+    arena->gate = (char *)base;
+    arena->up = (char *)base + up_offset;
+    arena->down = (char *)base + down_offset;
+    arena->bytes = total_bytes;
+    arena->up_offset = up_offset;
+    arena->down_offset = down_offset;
+    arena->capacity = arena->configured_budget;
+    arena->valid_count = 0;
+    arena->owner_device = owner;
+    g_stream_expert_persistent_arena_allocations.fetch_add(
+        1, std::memory_order_relaxed);
+    if (previous_device >= 0) (void)cudaSetDevice(previous_device);
+    return 1;
+}
+
+/* The persistent remap has a different lifetime/shape from the three weight
+ * planes, so it is a distinct allocation owned by the arena.  Phase 3a only
+ * exercises this through the device oracle; the production loader remains
+ * on g_stream_selected_transient_storage. */
+static int cuda_stream_expert_persistent_remap_ensure_locked(
+        uint64_t count) {
+    cuda_stream_expert_persistent_arena *arena =
+        &g_stream_expert_persistent_arena;
+    uint64_t bytes = 0;
+    if (!arena->base || arena->poisoned || arena->owner_device < 0 ||
+        count == 0 ||
+        !cuda_stream_expert_persistent_mul_u64(
+            count, sizeof(int32_t), &bytes) ||
+        bytes > SIZE_MAX) {
+        return 0;
+    }
+    if (arena->remap && arena->remap_capacity >= bytes) return 1;
+
+    int previous_device = -1;
+    (void)cudaGetDevice(&previous_device);
+    int32_t *replacement = NULL;
+    cudaError_t err = cudaSetDevice(arena->owner_device);
+    if (err == cudaSuccess) {
+        err = cudaMalloc((void **)&replacement, (size_t)bytes);
+    }
+    if (err != cudaSuccess || !replacement) {
+        if (replacement) (void)cudaFree(replacement);
+        (void)cudaGetLastError();
+        g_stream_expert_persistent_arena_failures.fetch_add(
+            1, std::memory_order_relaxed);
+        if (previous_device >= 0) (void)cudaSetDevice(previous_device);
+        return 0;
+    }
+    if (arena->remap) {
+        int drained = cuda_stream_selected_consume_drain() &&
+                      cuda_stream_selected_upload_drain_checked();
+        if (!drained) {
+            drained = cudaDeviceSynchronize() == cudaSuccess;
+        }
+        if (!drained || cudaFree(arena->remap) != cudaSuccess) {
+            (void)cudaGetLastError();
+            (void)cudaFree(replacement);
+            arena->poisoned = 1;
+            g_stream_expert_persistent_arena_failures.fetch_add(
+                1, std::memory_order_relaxed);
+            if (previous_device >= 0) (void)cudaSetDevice(previous_device);
+            return 0;
+        }
+    }
+    arena->remap = replacement;
+    arena->remap_capacity = bytes;
+    if (previous_device >= 0) (void)cudaSetDevice(previous_device);
+    return 1;
+}
+
+static int cuda_stream_expert_persistent_remap_upload(
+        int32_t *dst,
+        const uint32_t *src,
+        uint64_t count,
+        uint64_t *upload_event_out,
+        int *submitted_out) {
+    if (upload_event_out) *upload_event_out = 0;
+    if (submitted_out) *submitted_out = 0;
+    if (!dst || !src || count == 0 ||
+        count > SIZE_MAX / sizeof(int32_t)) {
+        return 0;
+    }
+    const size_t bytes = (size_t)count * sizeof(int32_t);
+    if (!upload_event_out ||
+        cuda_stream_selected_event_pipeline_oracle_requested()) {
+        if (!cuda_stream_selected_consume_drain()) return 0;
+        if (submitted_out) *submitted_out = 1;
+        const cudaError_t err = cudaMemcpy(
+            dst, src, bytes, cudaMemcpyHostToDevice);
+        if (err != cudaSuccess) {
+            fprintf(stderr,
+                    "ds4: CUDA persistent expert remap copy failed: %s\n",
+                    cudaGetErrorString(err));
+            (void)cudaGetLastError();
+            return 0;
+        }
+        return 1;
+    }
+
+    if (!cuda_stream_selected_stage_pool_alloc(1u) ||
+        !cuda_stream_selected_event_pipeline_ensure() ||
+        !cuda_stream_selected_remap_stage_ensure(count) ||
+        !cuda_stream_selected_consume_wait_on_upload()) {
+        return 0;
+    }
+    memcpy(g_stream_selected_remap_stage, src, bytes);
+    cudaError_t err = cudaMemcpyAsync(
+        dst, g_stream_selected_remap_stage, bytes,
+        cudaMemcpyHostToDevice, g_stream_selected_upload_stream);
+    if (err != cudaSuccess) {
+        (void)cudaGetLastError();
+        return 0;
+    }
+    if (submitted_out) *submitted_out = 1;
+    err = cudaEventRecord(g_stream_selected_upload_done_event,
+                          g_stream_selected_upload_stream);
+    if (err != cudaSuccess) {
+        (void)cudaGetLastError();
+        return cuda_stream_selected_upload_fail(
+            "persistent expert remap");
+    }
+    uint64_t value = ++g_stream_selected_upload_event_value;
+    if (value == 0) value = ++g_stream_selected_upload_event_value;
+    *upload_event_out = value;
+    g_stream_selected_event_uploads.fetch_add(
+        1, std::memory_order_relaxed);
+    return 1;
+}
+
+static void cuda_stream_expert_persistent_invalidate_all_locked(
+        cuda_stream_expert_persistent_arena *arena) {
+    const uint32_t invalidated =
+        cuda_stream_expert_persistent_state_count(
+            &g_stream_expert_persistent_state);
+    if (!cuda_stream_expert_persistent_state_init(
+            &g_stream_expert_persistent_state,
+            arena->capacity, arena->gate_expert_bytes,
+            arena->down_expert_bytes)) {
+        cuda_stream_expert_persistent_state_clear(
+            &g_stream_expert_persistent_state);
+    }
+    arena->valid_count = 0;
+    g_stream_expert_persistent_slot_invalidations.fetch_add(
+        invalidated, std::memory_order_relaxed);
+}
+
+static int cuda_stream_expert_persistent_try_load_writer(
+        const ds4_gpu_stream_expert_table *table,
+        const int32_t *selected_ids,
+        uint32_t slot_count,
+        uint64_t *upload_event_out,
+        int *submitted_any_out,
+        int force_for_oracle) {
+    if (upload_event_out) *upload_event_out = 0;
+    if (submitted_any_out) *submitted_any_out = 0;
+    g_stream_expert_persistent_epochs_attempted.fetch_add(
+        1, std::memory_order_relaxed);
+    if (!g_stream_selected_writer_active || !g_ssd_streaming_mode ||
+        g_n_gpus != 1 || !cuda_stream_selected_ranges_valid(table) ||
+        !selected_ids || slot_count == 0 ||
+        (!force_for_oracle &&
+         (!g_stream_expert_persistent_runtime_ready ||
+          !cuda_stream_expert_persistent_requested()))) {
+        return 0;
+    }
+
+    std::lock_guard<std::mutex> lock(
+        g_stream_expert_persistent_arena_mutex);
+    cuda_stream_expert_persistent_arena *arena =
+        &g_stream_expert_persistent_arena;
+    uint64_t two_gate = 0;
+    uint64_t total_class = 0;
+    if (arena->poisoned) return -1;
+    if (g_stream_selected_stage_poisoned &&
+        !cuda_stream_selected_stage_release_checked()) {
+        arena->poisoned = 1;
+        g_stream_expert_persistent_poisons.fetch_add(
+            1, std::memory_order_relaxed);
+        return -1;
+    }
+    if (arena->configured_budget == 0 ||
+        !cuda_stream_expert_persistent_mul_u64(
+            table->gate_expert_bytes, 2u, &two_gate) ||
+        !cuda_stream_expert_persistent_add_u64(
+            two_gate, table->down_expert_bytes, &total_class)) {
+        return 0;
+    }
+    if (arena->configured_expert_bytes == 0) {
+        arena->configured_expert_bytes = total_class;
+    }
+    if (arena->configured_expert_bytes != total_class) return 0;
+    if (arena->gate_expert_bytes == 0 && arena->down_expert_bytes == 0) {
+        arena->gate_expert_bytes = table->gate_expert_bytes;
+        arena->down_expert_bytes = table->down_expert_bytes;
+    }
+    if (arena->gate_expert_bytes != table->gate_expert_bytes ||
+        arena->down_expert_bytes != table->down_expert_bytes) {
+        return 0;
+    }
+    if (!cuda_stream_expert_persistent_arena_ensure_locked()) {
+        return arena->poisoned ? -1 : 0;
+    }
+
+    cuda_stream_expert_persistent_class expected_class = {};
+    if (!cuda_stream_expert_persistent_class_make(
+            &expected_class, arena->capacity,
+            arena->gate_expert_bytes, arena->down_expert_bytes)) {
+        return 0;
+    }
+    if (g_stream_expert_persistent_state.slots.empty()) {
+        if (!cuda_stream_expert_persistent_state_init(
+                &g_stream_expert_persistent_state,
+                arena->capacity, arena->gate_expert_bytes,
+                arena->down_expert_bytes)) {
+            return 0;
+        }
+    } else if (!cuda_stream_expert_persistent_class_equal(
+                   &g_stream_expert_persistent_state.size_class,
+                   &expected_class) ||
+               !cuda_stream_expert_persistent_state_valid(
+                   &g_stream_expert_persistent_state)) {
+        cuda_stream_expert_persistent_invalidate_all_locked(arena);
+        if (!cuda_stream_expert_persistent_state_valid(
+                &g_stream_expert_persistent_state)) {
+            return 0;
+        }
+    }
+
+    cuda_stream_expert_persistent_plan plan = {};
+    if (!cuda_stream_expert_persistent_plan_build(
+            &g_stream_expert_persistent_state, table,
+            selected_ids, slot_count, &plan)) {
+        cuda_stream_expert_persistent_plan_rollback(&plan);
+        return 0;
     }
-    if (n_tok == 1 && (in_dim & 31u) == 0u &&
-        !getenv("DS4_GLM_SHARED_SPLIT") &&
-        mid->bytes >= out_dim * sizeof(float) &&
-        x->bytes >= in_dim * sizeof(float)) {
-        const uint64_t row_bytes = (in_dim / 32u) * 34u;
-        const uint64_t wb = out_dim * row_bytes;
-        if (gate_offset <= model_size && wb <= model_size - gate_offset &&
-            up_offset <= model_size && wb <= model_size - up_offset) {
-            const int logical_tier = cuda_current_tier();
-            const char *gw = cuda_resolve_weight_ptr(model_map, gate_offset,
-                    wb, logical_tier, "glm_shared_gate");
-            const char *uw = cuda_resolve_weight_ptr(model_map, up_offset,
-                    wb, logical_tier, "glm_shared_up");
-            if (gw && uw) {
-                const uint32_t warps = 8u;
-                const uint32_t sh = (uint32_t)in_dim * sizeof(float);
-                glm_shared_gate_up_swiglu_one_kernel
-                        <<<(unsigned)((out_dim + warps - 1u) / warps),
-                           warps * 32u, sh>>>(
-                        (float *)mid->ptr, gw, uw, (const float *)x->ptr,
-                        (uint32_t)in_dim, (uint32_t)out_dim, clamp);
-                return cuda_ok(cudaGetLastError(), "glm shared swiglu one");
+    if (!cuda_stream_expert_persistent_remap_ensure_locked(slot_count)) {
+        cuda_stream_expert_persistent_plan_rollback(&plan);
+        return arena->poisoned ? -1 : 0;
+    }
+
+    std::vector<cuda_stream_selected_copy_task> tasks;
+    uint64_t weight_bytes = 0;
+    if (plan.loads.size() > UINT32_MAX / 3u) {
+        cuda_stream_expert_persistent_plan_rollback(&plan);
+        return 0;
+    }
+    int task_plan_ok = 1;
+    try {
+        tasks.reserve(plan.loads.size() * 3u);
+        uint32_t ordinal = 0;
+        for (const cuda_stream_expert_persistent_load &load : plan.loads) {
+            if (load.slot >= arena->capacity) {
+                task_plan_ok = 0;
+                break;
             }
+            const uint64_t gate_dst =
+                (uint64_t)load.slot * table->gate_expert_bytes;
+            const uint64_t down_dst =
+                (uint64_t)load.slot * table->down_expert_bytes;
+            const uint64_t expert = load.key.expert_id;
+            tasks.push_back({arena->gate + gate_dst,
+                table->gate_offset + expert * table->gate_expert_bytes,
+                table->gate_expert_bytes, ordinal++});
+            tasks.push_back({arena->up + gate_dst,
+                table->up_offset + expert * table->gate_expert_bytes,
+                table->gate_expert_bytes, ordinal++});
+            tasks.push_back({arena->down + down_dst,
+                table->down_offset + expert * table->down_expert_bytes,
+                table->down_expert_bytes, ordinal++});
         }
+    } catch (...) {
+        task_plan_ok = 0;
     }
-    if (!ds4_gpu_matmul_q8_0_tensor(gate, model_map, model_size, gate_offset,
-                                    in_dim, out_dim, x, n_tok) ||
-        !ds4_gpu_matmul_q8_0_tensor(up, model_map, model_size, up_offset,
-                                    in_dim, out_dim, x, n_tok)) {
+    if (!task_plan_ok) {
+        cuda_stream_expert_persistent_plan_rollback(&plan);
         return 0;
     }
-    return ds4_gpu_swiglu_tensor(mid, gate, up,
-                                 (uint32_t)(out_dim * n_tok), clamp, 1.0f);
-}
+    if (!cuda_stream_expert_persistent_mul_u64(
+            plan.loads.size(), expected_class.bytes_per_slot,
+            &weight_bytes)) {
+        cuda_stream_expert_persistent_plan_rollback(&plan);
+        return 0;
+    }
+    uint64_t gate_capacity = 0;
+    uint64_t down_capacity = 0;
+    if (!cuda_stream_expert_persistent_mul_u64(
+            arena->capacity, table->gate_expert_bytes, &gate_capacity) ||
+        !cuda_stream_expert_persistent_mul_u64(
+            arena->capacity, table->down_expert_bytes, &down_capacity)) {
+        cuda_stream_expert_persistent_plan_rollback(&plan);
+        return 0;
+    }
+    const uint64_t remap_bytes =
+        (uint64_t)slot_count * sizeof(int32_t);
+    int submitted = 0;
+    int copied = 0;
+    if (tasks.empty()) {
+        copied = cuda_stream_expert_persistent_remap_upload(
+            arena->remap, plan.remap.data(), plan.remap.size(),
+            upload_event_out, &submitted);
+    } else {
+        copied = cuda_model_copy_tasks_to_device_streamed(
+            tasks, table->model_map, table->model_size,
+            arena->remap, (const int32_t *)plan.remap.data(),
+            plan.remap.size(), /*run_oracle=*/0,
+            /*chunk_override=*/0, &submitted,
+            upload_event_out, "persistent expert cache");
+    }
+    if (submitted_any_out) *submitted_any_out = submitted;
+    if (g_stream_expert_persistent_test_fail_after_enqueue && submitted) {
+        copied = 0;
+        g_stream_expert_persistent_test_fail_after_enqueue = 0;
+    }
+    const int expect_async_token = upload_event_out != NULL &&
+        !cuda_stream_selected_event_pipeline_oracle_requested();
+    if (!copied ||
+        (expect_async_token && *upload_event_out == 0)) {
+        if (!submitted) {
+            cuda_stream_expert_persistent_plan_rollback(&plan);
+            if (g_stream_selected_stage_poisoned) {
+                arena->poisoned = 1;
+                g_stream_expert_persistent_poisons.fetch_add(
+                    1, std::memory_order_relaxed);
+                return -1;
+            }
+            return 0;
+        }
+        const int drained = cuda_stream_selected_upload_drain_checked();
+        cuda_stream_expert_persistent_invalidate_all_locked(arena);
+        if (!drained) {
+            arena->poisoned = 1;
+            g_stream_expert_persistent_poisons.fetch_add(
+                1, std::memory_order_relaxed);
+        }
+        cuda_stream_expert_persistent_plan_rollback(&plan);
+        g_stream_expert_persistent_upload_failures.fetch_add(
+            1, std::memory_order_relaxed);
+        return -1;
+    }
 
-extern "C" int ds4_gpu_shared_mid_swiglu_q8_0_tensor(
-        ds4_gpu_tensor       *mid,
-        const void             *model_map,
-        uint64_t                model_size,
-        uint64_t                gate_offset,
-        uint64_t                up_offset,
-        uint64_t                in_dim,
-        uint64_t                out_dim,
-        const ds4_gpu_tensor *x,
-        float                   clamp) {
-    static ds4_gpu_tensor *gu_scratch[DS4_MAX_GPUS][2] = {{0}};
-    const int dev = cuda_current_tier();
-    const uint64_t need = out_dim * sizeof(float);
-    for (int i = 0; i < 2; i++) {
-        if (!gu_scratch[dev][i] || gu_scratch[dev][i]->bytes < need) {
-            if (gu_scratch[dev][i]) ds4_gpu_tensor_free(gu_scratch[dev][i]);
-            gu_scratch[dev][i] = ds4_gpu_tensor_alloc(need);
+    if (!cuda_stream_expert_persistent_plan_commit(
+            &g_stream_expert_persistent_state, &plan)) {
+        const int drained = cuda_stream_selected_upload_drain_checked();
+        cuda_stream_expert_persistent_invalidate_all_locked(arena);
+        if (!drained) {
+            arena->poisoned = 1;
+            g_stream_expert_persistent_poisons.fetch_add(
+                1, std::memory_order_relaxed);
         }
-        if (!gu_scratch[dev][i]) return 0;
+        cuda_stream_expert_persistent_plan_rollback(&plan);
+        g_stream_expert_persistent_upload_failures.fetch_add(
+            1, std::memory_order_relaxed);
+        return -1;
     }
-    return ds4_gpu_shared_gate_up_swiglu_q8_0_rows_tensor(
-            gu_scratch[dev][0], gu_scratch[dev][1], mid,
-            model_map, model_size, gate_offset, up_offset,
-            in_dim, out_dim, x, 1, clamp);
+    arena->valid_count = cuda_stream_expert_persistent_state_count(
+        &g_stream_expert_persistent_state);
+
+    int top6_unique = (slot_count % 6u) == 0u;
+    for (uint32_t base = 0; top6_unique && base < slot_count; base += 6u) {
+        for (uint32_t i = 0; top6_unique && i < 6u; i++) {
+            for (uint32_t j = i + 1u; j < 6u; j++) {
+                if (selected_ids[base + i] == selected_ids[base + j]) {
+                    top6_unique = 0;
+                    break;
+                }
+            }
+        }
+    }
+    g_stream_selected_cache.storage_kind =
+        CUDA_STREAM_SELECTED_STORAGE_PERSISTENT;
+    g_stream_selected_cache.logical_tier = 0;
+    g_stream_selected_cache.model_map = table->model_map;
+    g_stream_selected_cache.layer = table->layer;
+    g_stream_selected_cache.n_total_expert = table->n_total_expert;
+    g_stream_selected_cache.slot_count = slot_count;
+    g_stream_selected_cache.compact_count = plan.unique_count;
+    g_stream_selected_cache.slot_base = plan.slot_base;
+    g_stream_selected_cache.weight_domain = plan.weight_domain;
+    g_stream_selected_cache.top6_unique = top6_unique;
+    uint64_t generation = ++g_stream_selected_cache_generation;
+    if (generation == 0) generation = ++g_stream_selected_cache_generation;
+    g_stream_selected_cache.generation = generation;
+    g_stream_selected_cache.upload_event_value =
+        upload_event_out ? *upload_event_out : 0;
+    g_stream_selected_cache.gate_offset = table->gate_offset;
+    g_stream_selected_cache.up_offset = table->up_offset;
+    g_stream_selected_cache.down_offset = table->down_offset;
+    g_stream_selected_cache.gate_expert_bytes = table->gate_expert_bytes;
+    g_stream_selected_cache.down_expert_bytes = table->down_expert_bytes;
+    g_stream_selected_cache.gate_ptr = arena->gate;
+    g_stream_selected_cache.up_ptr = arena->up;
+    g_stream_selected_cache.down_ptr = arena->down;
+    g_stream_selected_cache.gate_capacity = gate_capacity;
+    g_stream_selected_cache.up_capacity = gate_capacity;
+    g_stream_selected_cache.down_capacity = down_capacity;
+    g_stream_selected_cache.slot_selected_ptr = arena->remap;
+    g_stream_selected_cache.slot_selected_capacity = arena->remap_capacity;
+    g_stream_selected_cache.slot_selected_tensor.ptr = arena->remap;
+    g_stream_selected_cache.slot_selected_tensor.bytes = remap_bytes;
+    g_stream_selected_cache.slot_selected_tensor.owner = 0;
+    g_stream_selected_cache.slot_selected_tensor.device_id = 0;
+
+    g_stream_expert_persistent_epochs_published.fetch_add(
+        1, std::memory_order_relaxed);
+    if (tasks.empty()) {
+        g_stream_expert_persistent_all_hit_epochs.fetch_add(
+            1, std::memory_order_relaxed);
+    } else {
+        g_stream_expert_persistent_miss_epochs.fetch_add(
+            1, std::memory_order_relaxed);
+    }
+    g_stream_expert_persistent_miss_experts.fetch_add(
+        plan.loads.size(), std::memory_order_relaxed);
+    g_stream_expert_persistent_weight_bytes.fetch_add(
+        weight_bytes, std::memory_order_relaxed);
+    g_stream_expert_persistent_remap_bytes.fetch_add(
+        remap_bytes, std::memory_order_relaxed);
+    return 1;
 }
 
-extern "C" int ds4_gpu_signal_selected_readback_ready(uint64_t *event_value) {
-    if (event_value) *event_value = 1;
-    return cuda_ok(cudaDeviceSynchronize(), "selected readback signal");
+extern "C" int ds4_cuda_test_stream_expert_persistent_runtime(void) {
+    g_stream_expert_persistent_runtime_oracle_runs.fetch_add(
+        1, std::memory_order_relaxed);
+    if (g_n_gpus != 1) {
+        g_stream_expert_persistent_runtime_oracle_failures.fetch_add(
+            1, std::memory_order_relaxed);
+        return 0;
+    }
+
+    cuda_stream_expert_persistent_arena *arena =
+        &g_stream_expert_persistent_arena;
+    uint32_t saved_budget = 0;
+    uint64_t saved_total = 0;
+    uint64_t saved_gate = 0;
+    uint64_t saved_down = 0;
+    {
+        std::lock_guard<std::mutex> lock(
+            g_stream_expert_persistent_arena_mutex);
+        saved_budget = arena->configured_budget;
+        saved_total = arena->configured_expert_bytes;
+        saved_gate = arena->gate_expert_bytes;
+        saved_down = arena->down_expert_bytes;
+    }
+    const int saved_ssd = g_ssd_streaming_mode;
+    const int saved_model_fd = g_model_fd;
+    int ok = cuda_stream_expert_storage_release(1);
+    g_ssd_streaming_mode = 1;
+    g_model_fd = -1;
+    {
+        std::lock_guard<std::mutex> lock(
+            g_stream_expert_persistent_arena_mutex);
+        arena->configured_budget = 4u;
+        arena->configured_expert_bytes = 40u;
+        arena->gate_expert_bytes = 0;
+        arena->down_expert_bytes = 0;
+    }
+
+    unsigned char *model = NULL;
+    const uint32_t n_total = 6u;
+    const uint64_t gate_stride = 16u;
+    const uint64_t down_stride = 8u;
+    const uint64_t gate_offset = 0u;
+    const uint64_t up_offset = n_total * gate_stride;
+    const uint64_t down_offset = up_offset + n_total * gate_stride;
+    const uint64_t model_size = down_offset + n_total * down_stride;
+    if (ok) {
+        ok = cudaMallocHost((void **)&model, (size_t)model_size) ==
+             cudaSuccess;
+        if (!ok) (void)cudaGetLastError();
+    }
+    if (ok) {
+        for (uint32_t expert = 0; expert < n_total; expert++) {
+            memset(model + gate_offset + expert * gate_stride,
+                   0x10 + expert, (size_t)gate_stride);
+            memset(model + up_offset + expert * gate_stride,
+                   0x40 + expert, (size_t)gate_stride);
+            memset(model + down_offset + expert * down_stride,
+                   0x70 + expert, (size_t)down_stride);
+        }
+    }
+    ds4_gpu_stream_expert_table table = {};
+    table.model_map = model;
+    table.model_size = model_size;
+    table.layer = 17u;
+    table.n_total_expert = n_total;
+    table.gate_offset = gate_offset;
+    table.up_offset = up_offset;
+    table.down_offset = down_offset;
+    table.gate_expert_bytes = gate_stride;
+    table.down_expert_bytes = down_stride;
+
+    auto run_epoch = [&](const int32_t *ids, uint32_t count,
+                         uint64_t *upload_event_out,
+                         int *submitted_out) {
+        cuda_stream_selected_writer_guard writer;
+        const int rc = cuda_stream_expert_persistent_try_load_writer(
+            &table, ids, count, upload_event_out, submitted_out,
+            /*force_for_oracle=*/1);
+        if (rc > 0) writer.publish_valid();
+        return rc;
+    };
+    auto verify_epoch = [&](const int32_t *ids,
+                            const int32_t *expected_remap,
+                            uint32_t count, uint32_t expected_slot_base,
+                            uint32_t expected_domain) {
+        cuda_stream_selected_binding binding = {};
+        if (!cuda_stream_selected_binding_acquire(
+                &binding, 0, table.model_map, table.layer,
+                table.n_total_expert, table.gate_offset,
+                table.up_offset, table.down_offset,
+                table.gate_expert_bytes, table.down_expert_bytes,
+                count) ||
+            binding.storage_kind !=
+                CUDA_STREAM_SELECTED_STORAGE_PERSISTENT ||
+            binding.slot_base != expected_slot_base ||
+            binding.weight_domain != expected_domain ||
+            binding.gate != arena->gate +
+                (uint64_t)expected_slot_base * gate_stride ||
+            binding.up != arena->up +
+                (uint64_t)expected_slot_base * gate_stride ||
+            binding.down != arena->down +
+                (uint64_t)expected_slot_base * down_stride) {
+            return 0;
+        }
+        std::vector<int32_t> remap;
+        try {
+            remap.resize(count);
+        } catch (...) {
+            return 0;
+        }
+        if (cudaMemcpy(remap.data(), binding.selected->ptr,
+                       (size_t)count * sizeof(int32_t),
+                       cudaMemcpyDeviceToHost) != cudaSuccess) {
+            (void)cudaGetLastError();
+            return 0;
+        }
+        for (uint32_t i = 0; i < count; i++) {
+            if (remap[i] != expected_remap[i] || remap[i] < 0 ||
+                (uint32_t)remap[i] >= binding.weight_domain ||
+                (uint64_t)binding.slot_base + (uint32_t)remap[i] >=
+                    arena->capacity) {
+                return 0;
+            }
+            unsigned char gate_got[16] = {};
+            unsigned char up_got[16] = {};
+            unsigned char down_got[8] = {};
+            const uint64_t gate_delta =
+                (uint64_t)(uint32_t)remap[i] * gate_stride;
+            const uint64_t down_delta =
+                (uint64_t)(uint32_t)remap[i] * down_stride;
+            if (cudaMemcpy(gate_got, binding.gate + gate_delta,
+                           sizeof(gate_got),
+                           cudaMemcpyDeviceToHost) != cudaSuccess ||
+                cudaMemcpy(up_got, binding.up + gate_delta,
+                           sizeof(up_got),
+                           cudaMemcpyDeviceToHost) != cudaSuccess ||
+                cudaMemcpy(down_got, binding.down + down_delta,
+                           sizeof(down_got),
+                           cudaMemcpyDeviceToHost) != cudaSuccess) {
+                (void)cudaGetLastError();
+                return 0;
+            }
+            for (uint32_t b = 0; b < sizeof(gate_got); b++) {
+                if (gate_got[b] != (unsigned char)(0x10 + ids[i]) ||
+                    up_got[b] != (unsigned char)(0x40 + ids[i])) {
+                    return 0;
+                }
+            }
+            for (uint32_t b = 0; b < sizeof(down_got); b++) {
+                if (down_got[b] != (unsigned char)(0x70 + ids[i])) {
+                    return 0;
+                }
+            }
+        }
+        return 1;
+    };
+
+    const int32_t cold[] = {0, 1};
+    const int32_t mixed[] = {1, 2};
+    const int32_t all_hit_shifted[] = {1, 2, 1};
+    const int32_t all_hit_sparse[] = {0, 2};
+    const int32_t eviction[] = {3, 4};
+    const int32_t fault[] = {5};
+    const int32_t remap01[] = {0, 1};
+    const int32_t remap010[] = {0, 1, 0};
+    const int32_t remap02[] = {0, 2};
+    const int32_t remap20[] = {2, 0};
+    int submitted = 0;
+    uint64_t upload_event = 0;
+    if (ok) ok = run_epoch(cold, 2u, &upload_event, &submitted) == 1 &&
+                 submitted &&
+                 (upload_event != 0 ||
+                  cuda_stream_selected_event_pipeline_oracle_requested());
+    uint64_t consume_generation = 0;
+    if (ok) {
+        ok = cuda_stream_selected_consumer_begin(
+                 /*allow_streaming=*/1, cuda_decode_stream(),
+                 &consume_generation) &&
+             cuda_stream_selected_consumer_end(
+                 consume_generation, cuda_decode_stream()) &&
+             cuda_stream_selected_consume_drain() &&
+             verify_epoch(cold, remap01, 2u, 0u, 2u);
+    }
+    submitted = 0;
+    if (ok) ok = run_epoch(mixed, 2u, NULL, &submitted) == 1 &&
+                 submitted && verify_epoch(mixed, remap01, 2u, 1u, 2u);
+    submitted = 0;
+    if (ok) ok = run_epoch(all_hit_shifted, 3u, NULL, &submitted) == 1 &&
+                 submitted && verify_epoch(
+                     all_hit_shifted, remap010, 3u, 1u, 2u);
+    submitted = 0;
+    if (ok) ok = run_epoch(all_hit_sparse, 2u, NULL, &submitted) == 1 &&
+                 submitted && verify_epoch(
+                     all_hit_sparse, remap02, 2u, 0u, 3u);
+    submitted = 0;
+    if (ok) ok = run_epoch(eviction, 2u, NULL, &submitted) == 1 &&
+                 submitted && verify_epoch(
+                     eviction, remap20, 2u, 1u, 3u);
+    if (ok) g_stream_expert_persistent_test_fail_after_enqueue = 1;
+    submitted = 0;
+    upload_event = 0;
+    if (ok) {
+        ok = run_epoch(fault, 1u, &upload_event, &submitted) == -1 &&
+             submitted &&
+             (upload_event != 0 ||
+              cuda_stream_selected_event_pipeline_oracle_requested()) &&
+             !g_stream_selected_cache.valid;
+    }
+    if (ok) {
+        std::lock_guard<std::mutex> lock(
+            g_stream_expert_persistent_arena_mutex);
+        ok = arena->valid_count == 0u &&
+             cuda_stream_expert_persistent_state_count(
+                 &g_stream_expert_persistent_state) == 0u;
+    }
+    if (ok) g_stream_selected_stage_test_fail_drain = 1;
+    submitted = 0;
+    if (ok) ok = run_epoch(cold, 2u, NULL, &submitted) == -1 && !submitted;
+    {
+        std::lock_guard<std::mutex> lock(
+            g_stream_expert_persistent_arena_mutex);
+        ok = ok && arena->poisoned &&
+             g_stream_selected_stage_poisoned;
+    }
+
+    const int released = cuda_stream_expert_storage_release(1);
+    const int stage_released =
+        cuda_stream_selected_stage_release_checked();
+    ok = ok && released && stage_released &&
+         !g_stream_selected_stage_poisoned;
+    if (released) {
+        std::lock_guard<std::mutex> lock(
+            g_stream_expert_persistent_arena_mutex);
+        ok = ok && !arena->base && !arena->remap &&
+             arena->valid_count == 0u &&
+             g_stream_expert_persistent_state.slots.empty();
+        arena->configured_budget = saved_budget;
+        arena->configured_expert_bytes = saved_total;
+        arena->gate_expert_bytes = saved_gate;
+        arena->down_expert_bytes = saved_down;
+    }
+    g_stream_expert_persistent_test_fail_after_enqueue = 0;
+    g_stream_selected_stage_test_fail_drain = 0;
+    g_model_fd = saved_model_fd;
+    g_ssd_streaming_mode = saved_ssd;
+    if (model) (void)cudaFreeHost(model);
+    if (!ok) {
+        g_stream_expert_persistent_runtime_oracle_failures.fetch_add(
+            1, std::memory_order_relaxed);
+    }
+    return ok;
 }
 
-extern "C" int ds4_gpu_stream_expert_cache_begin_selected_load(
-        const ds4_gpu_stream_expert_table *table,
-        const int32_t                     *selected_ids,
-        uint32_t                           n_selected) {
-    return cuda_stream_selected_cache_begin_load(table, selected_ids,
-                                                 n_selected);
+extern "C" int ds4_cuda_test_stream_expert_persistent_arena(void) {
+    g_stream_expert_persistent_arena_oracle_runs.fetch_add(
+        1, std::memory_order_relaxed);
+    std::lock_guard<std::mutex> lock(
+        g_stream_expert_persistent_arena_mutex);
+    cuda_stream_expert_persistent_arena *arena =
+        &g_stream_expert_persistent_arena;
+    const uint32_t saved_budget = arena->configured_budget;
+    const uint64_t saved_total = arena->configured_expert_bytes;
+    const uint64_t saved_gate = arena->gate_expert_bytes;
+    const uint64_t saved_down = arena->down_expert_bytes;
+    int ok = g_n_gpus == 1 && !arena->base && !arena->remap &&
+             !arena->poisoned;
+    arena->configured_budget = 3u;
+    arena->configured_expert_bytes = 40u;
+    arena->gate_expert_bytes = 16u;
+    arena->down_expert_bytes = 8u;
+
+    void *first_base = NULL;
+    int32_t *first_remap = NULL;
+    if (ok) {
+        ok = cuda_stream_expert_persistent_arena_ensure_locked() &&
+             cuda_stream_expert_persistent_remap_ensure_locked(7u);
+    }
+    uint64_t expected_up = 0;
+    uint64_t expected_down = 0;
+    uint64_t expected_total = 0;
+    if (ok) {
+        first_base = arena->base;
+        first_remap = arena->remap;
+        ok = cuda_stream_expert_persistent_arena_layout(
+                 3u, 16u, 8u, &expected_up, &expected_down,
+                 &expected_total) &&
+             arena->owner_device == g_gpu[0].device_id &&
+             arena->capacity == 3u && arena->bytes == expected_total &&
+             arena->up_offset == expected_up &&
+             arena->down_offset == expected_down &&
+             arena->gate == (char *)arena->base &&
+             arena->up == (char *)arena->base + expected_up &&
+             arena->down == (char *)arena->base + expected_down &&
+             arena->remap &&
+             arena->remap_capacity >= 7u * sizeof(int32_t) &&
+             (((uintptr_t)arena->base | (uintptr_t)arena->gate |
+               (uintptr_t)arena->up | (uintptr_t)arena->down |
+               (uintptr_t)arena->remap) & 255u) == 0u;
+    }
+
+    int previous_device = -1;
+    unsigned char got[3] = {0, 0, 0};
+    int32_t remap_got = 0;
+    const int32_t remap_canary = INT32_C(0x13572468);
+    if (ok) {
+        (void)cudaGetDevice(&previous_device);
+        ok = cudaSetDevice(arena->owner_device) == cudaSuccess &&
+             cudaMemset(arena->base, 0, (size_t)arena->bytes) == cudaSuccess &&
+             cudaMemset(arena->gate + 47u, 0xa1, 1u) == cudaSuccess &&
+             cudaMemset(arena->up + 47u, 0xb2, 1u) == cudaSuccess &&
+             cudaMemset(arena->down + 23u, 0xc3, 1u) == cudaSuccess &&
+             cudaMemcpy(arena->remap + 6u, &remap_canary,
+                        sizeof(remap_canary),
+                        cudaMemcpyHostToDevice) == cudaSuccess &&
+             cudaMemcpy(&got[0], arena->gate + 47u, 1u,
+                        cudaMemcpyDeviceToHost) == cudaSuccess &&
+             cudaMemcpy(&got[1], arena->up + 47u, 1u,
+                        cudaMemcpyDeviceToHost) == cudaSuccess &&
+             cudaMemcpy(&got[2], arena->down + 23u, 1u,
+                        cudaMemcpyDeviceToHost) == cudaSuccess &&
+             cudaMemcpy(&remap_got, arena->remap + 6u,
+                        sizeof(remap_got),
+                        cudaMemcpyDeviceToHost) == cudaSuccess &&
+             got[0] == 0xa1 && got[1] == 0xb2 && got[2] == 0xc3 &&
+             remap_got == remap_canary;
+        if (!ok) (void)cudaGetLastError();
+        if (previous_device >= 0) (void)cudaSetDevice(previous_device);
+    }
+    if (ok) {
+        ok = cuda_stream_expert_persistent_arena_ensure_locked() &&
+             cuda_stream_expert_persistent_remap_ensure_locked(3u) &&
+             arena->base == first_base && arena->remap == first_remap;
+    }
+    if (ok) {
+        ok = cuda_stream_expert_persistent_arena_release_locked(0) &&
+             !arena->base && !arena->remap &&
+             arena->remap_capacity == 0 &&
+             arena->gate_expert_bytes == 16u &&
+             arena->down_expert_bytes == 8u &&
+             cuda_stream_expert_persistent_arena_ensure_locked() &&
+             cuda_stream_expert_persistent_remap_ensure_locked(9u) &&
+             arena->base && arena->remap;
+    }
+    const int cleanup_ok =
+        cuda_stream_expert_persistent_arena_release_locked(1);
+    if (cleanup_ok) {
+        ok = ok && !arena->base && !arena->remap &&
+             arena->remap_capacity == 0 &&
+             arena->gate_expert_bytes == 0 &&
+             arena->down_expert_bytes == 0 &&
+             arena->configured_expert_bytes == 40u;
+        arena->configured_budget = saved_budget;
+        arena->configured_expert_bytes = saved_total;
+        arena->gate_expert_bytes = saved_gate;
+        arena->down_expert_bytes = saved_down;
+    } else {
+        ok = 0;
+    }
+    if (!ok) {
+        g_stream_expert_persistent_arena_oracle_failures.fetch_add(
+            1, std::memory_order_relaxed);
+    }
+    return ok;
 }
 
 extern "C" uint32_t ds4_gpu_stream_expert_cache_budget_for_expert_size(
         uint64_t gate_expert_bytes,
         uint64_t down_expert_bytes) {
-    (void)gate_expert_bytes;
-    (void)down_expert_bytes;
-    return 0;
+    uint64_t two_gate = 0;
+    uint64_t total_class = 0;
+    if (gate_expert_bytes == 0 || down_expert_bytes == 0 ||
+        !cuda_stream_expert_persistent_mul_u64(
+            gate_expert_bytes, 2u, &two_gate) ||
+        !cuda_stream_expert_persistent_add_u64(
+            two_gate, down_expert_bytes, &total_class) ||
+        total_class == 0) {
+        return 0;
+    }
+    std::lock_guard<std::mutex> lock(
+        g_stream_expert_persistent_arena_mutex);
+    cuda_stream_expert_persistent_arena *arena =
+        &g_stream_expert_persistent_arena;
+    if (arena->configured_expert_bytes == 0) {
+        arena->configured_expert_bytes = total_class;
+    }
+    if (arena->configured_expert_bytes != total_class) return 0;
+    if (arena->gate_expert_bytes == 0 && arena->down_expert_bytes == 0) {
+        arena->gate_expert_bytes = gate_expert_bytes;
+        arena->down_expert_bytes = down_expert_bytes;
+    } else if (arena->gate_expert_bytes != gate_expert_bytes ||
+               arena->down_expert_bytes != down_expert_bytes) {
+        return 0;
+    }
+    if (!g_stream_expert_persistent_runtime_ready ||
+        !g_ssd_streaming_mode || arena->configured_budget == 0 ||
+        !cuda_stream_expert_persistent_requested()) {
+        return 0;
+    }
+    return arena->configured_budget;
 }
 
 extern "C" int ds4_gpu_tensor_copy_f32_to_f16(ds4_gpu_tensor *dst, uint64_t dst_offset,
@@ -32857,18 +42756,88 @@ extern "C" int ds4_gpu_tensor_read_after_selected_event(const ds4_gpu_tensor *te
                                              uint64_t bytes,
                                              uint64_t event_value,
                                              const char *label) {
-    (void)event_value;
     if (!tensor || !data || offset > tensor->bytes ||
-        bytes > tensor->bytes - offset) {
+        bytes > tensor->bytes - offset || bytes > SIZE_MAX) {
+        return 0;
+    }
+    const int tier = ds4_tensor_device_idx(tensor);
+    if (event_value == 0 ||
+        event_value != g_stream_selected_compute_event_value ||
+        tier < 0 || tier >= g_n_gpus ||
+        g_stream_selected_event_owner_device != g_gpu[tier].device_id ||
+        !g_stream_selected_readback_stream ||
+        !g_stream_selected_compute_ready_event ||
+        !g_stream_selected_readback_done_event ||
+        cudaSetDevice(g_stream_selected_event_owner_device) != cudaSuccess ||
+        (bytes != 0 && !cuda_stream_selected_readback_stage_ensure(bytes))) {
+        (void)cudaGetLastError();
         return 0;
     }
-    if (!cuda_ok(cudaDeviceSynchronize(),
-                 label ? label : "selected readback wait")) {
+
+    cudaError_t err = cudaStreamWaitEvent(
+        g_stream_selected_readback_stream,
+        g_stream_selected_compute_ready_event, 0);
+    if (err == cudaSuccess && bytes != 0) {
+        err = cudaMemcpyAsync(
+            g_stream_selected_readback_stage,
+            (const char *)tensor->ptr + offset,
+            (size_t)bytes,
+            cudaMemcpyDeviceToHost,
+            g_stream_selected_readback_stream);
+    }
+    if (err == cudaSuccess) {
+        err = cudaEventRecord(g_stream_selected_readback_done_event,
+                              g_stream_selected_readback_stream);
+    }
+    if (err == cudaSuccess) {
+        err = cudaEventSynchronize(g_stream_selected_readback_done_event);
+    }
+    if (err != cudaSuccess) {
+        fprintf(stderr,
+                "ds4: CUDA %s failed: %s\n",
+                label ? label : "selected event readback",
+                cudaGetErrorString(err));
+        (void)cudaGetLastError();
         return 0;
     }
-    return cuda_ok(cudaMemcpy(data, (const char *)tensor->ptr + offset,
-                              (size_t)bytes, cudaMemcpyDeviceToHost),
-                   "selected tensor read");
+    if (bytes != 0) {
+        memcpy(data, g_stream_selected_readback_stage, (size_t)bytes);
+    }
+    g_stream_selected_event_readbacks.fetch_add(
+        1, std::memory_order_relaxed);
+
+    if (cuda_stream_selected_event_pipeline_oracle_requested()) {
+        g_stream_selected_event_oracle_runs.fetch_add(
+            1, std::memory_order_relaxed);
+        std::vector<unsigned char> reference;
+        try {
+            reference.resize((size_t)bytes);
+        } catch (...) {
+            g_stream_selected_event_oracle_failures.fetch_add(
+                1, std::memory_order_relaxed);
+            return 0;
+        }
+        err = cudaDeviceSynchronize();
+        if (err == cudaSuccess && bytes != 0) {
+            err = cudaMemcpy(reference.data(),
+                             (const char *)tensor->ptr + offset,
+                             (size_t)bytes,
+                             cudaMemcpyDeviceToHost);
+        }
+        if (err != cudaSuccess ||
+            (bytes != 0 && memcmp(reference.data(), data,
+                                  (size_t)bytes) != 0)) {
+            fprintf(stderr,
+                    "ds4: CUDA selected event readback oracle %s\n",
+                    err == cudaSuccess ? "mismatch" :
+                    cudaGetErrorString(err));
+            if (err != cudaSuccess) (void)cudaGetLastError();
+            g_stream_selected_event_oracle_failures.fetch_add(
+                1, std::memory_order_relaxed);
+            return 0;
+        }
+    }
+    return 1;
 }
 
 extern "C" int ds4_gpu_tp_big_gate_encode(uint32_t layer, uint32_t rows,
@@ -32894,6 +42863,614 @@ extern "C" int ds4_gpu_wait_selected_readback_ready(uint64_t event_value, const
                    label ? label : "selected readback wait");
 }
 
+extern "C" int ds4_gpu_stream_expert_cache_wait_selected_upload(
+        uint64_t event_value, const char *label) {
+    return cuda_stream_selected_wait_upload_on(
+        event_value, cuda_decode_stream(), label);
+}
+
+static int cuda_stream_selected_wait_upload_on(
+        uint64_t event_value, cudaStream_t stream, const char *label) {
+    if (event_value == 0) return 1;
+    if (event_value != g_stream_selected_upload_event_value ||
+        !g_stream_selected_upload_done_event ||
+        g_stream_selected_event_owner_device < 0 ||
+        g_n_gpus != 1 ||
+        g_stream_selected_event_owner_device != g_gpu[0].device_id ||
+        g_stream_selected_upload_owner_device !=
+            g_stream_selected_event_owner_device ||
+        cudaSetDevice(g_stream_selected_event_owner_device) != cudaSuccess) {
+        (void)cudaGetLastError();
+        return 0;
+    }
+    const cudaError_t err = cudaStreamWaitEvent(
+        stream, g_stream_selected_upload_done_event, 0);
+    if (err != cudaSuccess) {
+        fprintf(stderr,
+                "ds4: CUDA %s failed: %s\n",
+                label ? label : "selected upload wait",
+                cudaGetErrorString(err));
+        (void)cudaGetLastError();
+        return 0;
+    }
+    g_stream_selected_event_compute_waits.fetch_add(
+        1, std::memory_order_relaxed);
+    return 1;
+}
+
+extern "C" int ds4_gpu_cuda_stream_selected_event_abort(void) {
+    /* Stop new consumers and wait until every routed caller has published
+     * its consume frontier before the device-wide abort drain. */
+    cuda_stream_selected_writer_guard writer;
+    int device = g_stream_selected_event_owner_device;
+    if (device < 0 && g_n_gpus == 1) device = g_gpu[0].device_id;
+    if (device >= 0 && cudaSetDevice(device) != cudaSuccess) {
+        (void)cudaGetLastError();
+        return 0;
+    }
+    const cudaError_t err = cudaDeviceSynchronize();
+    uint64_t compute_value = ++g_stream_selected_compute_event_value;
+    if (compute_value == 0) ++g_stream_selected_compute_event_value;
+    uint64_t upload_value = ++g_stream_selected_upload_event_value;
+    if (upload_value == 0) ++g_stream_selected_upload_event_value;
+    if (err != cudaSuccess) {
+        fprintf(stderr,
+                "ds4: CUDA selected event abort sync failed: %s\n",
+                cudaGetErrorString(err));
+        (void)cudaGetLastError();
+        return 0;
+    }
+    return 1;
+}
+
+__global__ static void cuda_stream_selected_lease_delay_store_kernel(
+        int32_t *dst, int32_t value, uint64_t delay_clocks) {
+    if (blockIdx.x != 0 || threadIdx.x != 0) return;
+    const uint64_t begin = clock64();
+    while (clock64() - begin < delay_clocks) {
+        __nanosleep(64u);
+    }
+    *dst = value;
+}
+
+/* End-to-end compact-binding protocol oracle.  It publishes generation A
+ * through the real begin_load_impl call site, acquires it through the real
+ * consumer and binding helpers, then starts a second writer.  A probe inside
+ * guard proves that writer B is waiting while reader A is held.  Once A
+ * records consume_done, B imports that event into the nonblocking upload
+ * stream, overwrites the remap, and publishes generation B before upload
+ * completion.  A second consumer validates both generation and upload token.
+ * fails if writer exclusion, publish_valid(), consume record/wait, generation
+ * matching, or upload completion are bypassed. */
+extern "C" int ds4_cuda_test_iq2_ssd_grouped_lease(void) {
+    if (g_n_gpus != 1 ||
+        cudaSetDevice(g_gpu[0].device_id) != cudaSuccess ||
+        !cuda_stream_selected_consume_drain() ||
+        !cuda_stream_selected_consume_prepare()) {
+        (void)cudaGetLastError();
+        return 0;
+    }
+
+    const int saved_ssd_streaming_mode = g_ssd_streaming_mode;
+    const int saved_model_fd = g_model_fd;
+    const void *saved_model_fd_host_base = g_model_fd_host_base;
+    const int saved_model_direct_fd = g_model_direct_fd;
+    const uint64_t saved_model_direct_align = g_model_direct_align;
+    const uint64_t saved_model_file_size = g_model_file_size;
+    cuda_stream_selected_cache_release();
+    if (g_stream_selected_transient_storage.gate ||
+        g_stream_selected_transient_storage.up ||
+        g_stream_selected_transient_storage.down ||
+        g_stream_selected_transient_storage.remap ||
+        g_stream_selected_cache.valid ||
+        g_stream_selected_cache.storage_kind !=
+            CUDA_STREAM_SELECTED_STORAGE_NONE) {
+        return 0;
+    }
+
+    uint32_t saved_arena_budget = 0;
+    uint64_t saved_arena_total = 0;
+    uint64_t saved_arena_gate = 0;
+    uint64_t saved_arena_down = 0;
+    void *persistent_base = NULL;
+    int32_t *persistent_remap = NULL;
+    uint64_t persistent_bytes = 0;
+    const unsigned char persistent_base_canary = 0x5au;
+    const int32_t persistent_remap_canary = INT32_C(0x31415926);
+    int persistent_touched = 0;
+    int ok = 1;
+    {
+        std::lock_guard<std::mutex> lock(
+            g_stream_expert_persistent_arena_mutex);
+        cuda_stream_expert_persistent_arena *arena =
+            &g_stream_expert_persistent_arena;
+        saved_arena_budget = arena->configured_budget;
+        saved_arena_total = arena->configured_expert_bytes;
+        saved_arena_gate = arena->gate_expert_bytes;
+        saved_arena_down = arena->down_expert_bytes;
+        if (arena->base || arena->remap || arena->poisoned) {
+            ok = 0;
+        } else {
+            persistent_touched = 1;
+            arena->configured_budget = 3u;
+            arena->configured_expert_bytes = 40u;
+            arena->gate_expert_bytes = 16u;
+            arena->down_expert_bytes = 8u;
+            ok = cuda_stream_expert_persistent_arena_ensure_locked() &&
+                 cuda_stream_expert_persistent_remap_ensure_locked(7u);
+            if (ok) {
+                persistent_base = arena->base;
+                persistent_remap = arena->remap;
+                persistent_bytes = arena->bytes;
+                ok = persistent_base && persistent_remap &&
+                     persistent_bytes != 0 &&
+                     cudaMemset((char *)persistent_base +
+                                    persistent_bytes - 1u,
+                                persistent_base_canary, 1u) == cudaSuccess &&
+                     cudaMemcpy(persistent_remap + 6u,
+                                &persistent_remap_canary,
+                                sizeof(persistent_remap_canary),
+                                cudaMemcpyHostToDevice) == cudaSuccess;
+                if (!ok) (void)cudaGetLastError();
+            }
+        }
+    }
+    {
+        cuda_stream_selected_writer_guard writer;
+        g_ssd_streaming_mode = 1;
+    }
+    /* Force begin_load_impl through its host-map copy contract regardless of
+     * any model fd configured by a surrounding process.  No saved descriptor
+     * is closed; the exact globals are restored at the single cleanup exit. */
+    g_model_fd = -1;
+    g_model_fd_host_base = NULL;
+    g_model_direct_fd = -1;
+    g_model_direct_align = 1;
+
+    unsigned char *test_model = NULL;
+    int32_t *host_words = NULL;
+    const uint64_t expert_bytes = 16u;
+    const uint32_t weight_domain = 6u;
+    const uint32_t test_total_expert = 8u;
+    const uint64_t full_table_bytes =
+        expert_bytes * test_total_expert;
+    const uint64_t test_gate_offset = 0u;
+    const uint64_t test_up_offset = full_table_bytes;
+    const uint64_t test_down_offset = 2u * full_table_bytes;
+    const uint64_t test_model_bytes = 3u * full_table_bytes;
+    g_model_file_size = test_model_bytes;
+    ok = ok && cuda_stream_selected_stage_pool_alloc(4096u) &&
+             cuda_stream_selected_event_pipeline_ensure() &&
+             cudaMallocHost((void **)&test_model,
+                            (size_t)test_model_bytes) == cudaSuccess &&
+             cudaMallocHost((void **)&host_words,
+                            3u * sizeof(*host_words)) == cudaSuccess;
+    if (!ok) (void)cudaGetLastError();
+    if (ok) {
+        for (uint64_t i = 0; i < test_model_bytes; i++) {
+            test_model[i] = (unsigned char)(i * 29u + 7u);
+        }
+        host_words[0] = 0x13572468;
+        host_words[1] = 0x24681357;
+        host_words[2] = 0;
+    }
+
+    const void *test_model_map = test_model;
+    const uint32_t test_layer = 0x7f00u;
+    const int32_t selected_ids[6] = {0, 1, 2, 3, 4, 5};
+    ds4_gpu_stream_expert_table table = {};
+    table.model_map = test_model_map;
+    table.model_size = test_model_bytes;
+    table.layer = test_layer;
+    table.n_total_expert = test_total_expert;
+    table.gate_offset = test_gate_offset;
+    table.up_offset = test_up_offset;
+    table.down_offset = test_down_offset;
+    table.gate_expert_bytes = expert_bytes;
+    table.down_expert_bytes = expert_bytes;
+    if (ok) {
+        ok = cuda_stream_selected_cache_begin_load_impl(
+            &table, selected_ids, weight_domain,
+            /*upload_event_out=*/NULL,
+            /*submitted_any_out=*/NULL);
+    }
+    if (ok) {
+        std::lock_guard<std::mutex> lock(
+            g_stream_expert_persistent_arena_mutex);
+        const cuda_stream_expert_persistent_arena *arena =
+            &g_stream_expert_persistent_arena;
+        const uintptr_t arena_begin = (uintptr_t)persistent_base;
+        const uintptr_t arena_end = arena_begin + persistent_bytes;
+        const uintptr_t transient_gate =
+            (uintptr_t)g_stream_selected_transient_storage.gate;
+        const uintptr_t transient_up =
+            (uintptr_t)g_stream_selected_transient_storage.up;
+        const uintptr_t transient_down =
+            (uintptr_t)g_stream_selected_transient_storage.down;
+        ok = arena->base == persistent_base &&
+             arena->remap == persistent_remap &&
+             arena_begin < arena_end &&
+             (transient_gate < arena_begin || transient_gate >= arena_end) &&
+             (transient_up < arena_begin || transient_up >= arena_end) &&
+             (transient_down < arena_begin || transient_down >= arena_end) &&
+             g_stream_selected_transient_storage.remap != persistent_remap;
+    }
+    const uint64_t generation_a = ok
+        ? g_stream_selected_cache.generation : 0;
+    if (ok && (!g_stream_selected_cache.valid || generation_a == 0 ||
+               g_stream_selected_cache.storage_kind !=
+                   CUDA_STREAM_SELECTED_STORAGE_TRANSIENT ||
+               g_stream_selected_cache.upload_event_value != 0 ||
+               g_stream_selected_cache.weight_domain != weight_domain ||
+               g_stream_selected_cache.gate_ptr !=
+                   g_stream_selected_transient_storage.gate ||
+               g_stream_selected_cache.up_ptr !=
+                   g_stream_selected_transient_storage.up ||
+               g_stream_selected_cache.down_ptr !=
+                   g_stream_selected_transient_storage.down ||
+               g_stream_selected_cache.slot_selected_ptr !=
+                   g_stream_selected_transient_storage.remap)) {
+        ok = 0;
+    }
+
+    uint64_t reader_generation = 0;
+    int reader_held = 0;
+    cuda_stream_selected_binding binding_a = {};
+    if (ok) {
+        ok = cuda_stream_selected_consumer_begin(
+                 /*allow_streaming=*/1, (cudaStream_t)0,
+                 &reader_generation) &&
+             reader_generation == generation_a;
+        reader_held = reader_generation != 0;
+    }
+    if (ok) {
+        ok = cuda_stream_selected_binding_acquire(
+                 &binding_a, 0, test_model_map, test_layer,
+                 test_total_expert,
+                 test_gate_offset, test_up_offset, test_down_offset,
+                 expert_bytes, expert_bytes, weight_domain) &&
+             binding_a.generation == generation_a &&
+             binding_a.storage_kind ==
+                 CUDA_STREAM_SELECTED_STORAGE_TRANSIENT &&
+             binding_a.weight_domain == weight_domain &&
+             binding_a.selected &&
+             binding_a.selected->ptr ==
+                 g_stream_selected_transient_storage.remap;
+    }
+    if (ok) {
+        /* A is deliberately longer than writer B's 1M-clock store.  If B
+         * omits the consume fence it writes first and A restores sentinel A;
+         * with the fence B cannot start until A has completed. */
+        cuda_stream_selected_lease_delay_store_kernel<<<1, 1, 0, 0>>>(
+            (int32_t *)binding_a.selected->ptr,
+            host_words[0], 50000000u);
+        ok = cudaGetLastError() == cudaSuccess;
+    }
+
+    uint64_t generation_b = ++g_stream_selected_cache_generation;
+    if (generation_b == 0) {
+        generation_b = ++g_stream_selected_cache_generation;
+    }
+    std::atomic<int> writer_waiting{0};
+    std::atomic<int> writer_acquired{0};
+    std::atomic<int> writer_published{0};
+    std::atomic<int> writer_ok{1};
+    std::thread writer_thread;
+    if (reader_held) {
+        try {
+            writer_thread = std::thread([&] {
+                cuda_stream_selected_writer_guard writer(&writer_waiting);
+                writer_acquired.store(1, std::memory_order_release);
+                int local_ok =
+                    cudaSetDevice(g_gpu[0].device_id) == cudaSuccess &&
+                    cuda_stream_selected_consume_wait_on_upload();
+                if (local_ok) {
+                    /* Keep publication observably ahead of upload completion.
+                     * Without consumer B's upload-token wait, its default-
+                     * stream read runs after consume A but during this delay
+                     * and deterministically observes sentinel A. */
+                    cuda_stream_selected_lease_delay_store_kernel<<<
+                        1, 1, 0, g_stream_selected_upload_stream>>>(
+                        g_stream_selected_transient_storage.remap,
+                        host_words[1], 1000000u);
+                    local_ok = cudaGetLastError() == cudaSuccess;
+                }
+                if (local_ok) {
+                    local_ok = cudaEventRecord(
+                        g_stream_selected_upload_done_event,
+                        g_stream_selected_upload_stream) == cudaSuccess;
+                }
+                uint64_t upload_value = 0;
+                if (local_ok) {
+                    upload_value = ++g_stream_selected_upload_event_value;
+                    if (upload_value == 0) {
+                        upload_value =
+                            ++g_stream_selected_upload_event_value;
+                    }
+                    local_ok =
+                        cuda_stream_selected_cache_bind_transient_storage(
+                            /*logical_tier=*/0,
+                            expert_bytes * weight_domain,
+                            expert_bytes * weight_domain,
+                            sizeof(selected_ids));
+                }
+                if (local_ok) {
+                    g_stream_selected_cache.generation = generation_b;
+                    g_stream_selected_cache.upload_event_value =
+                        upload_value;
+                    writer.publish_valid();
+                    writer_published.store(1, std::memory_order_release);
+                } else {
+                    (void)cudaGetLastError();
+                }
+                writer_ok.store(local_ok, std::memory_order_release);
+            });
+        } catch (...) {
+            ok = 0;
+        }
+    }
+
+    if (writer_thread.joinable()) {
+        uint32_t yields = 0;
+        while (!writer_waiting.load(std::memory_order_acquire) &&
+               yields++ < 1000000u) {
+            std::this_thread::yield();
+        }
+        /* waiting is set while the writer owns the protocol mutex, before
+         * its predicate wait.  With reader A held, acquired must still be 0
+         * and generation A must remain the published binding. */
+        if (!writer_waiting.load(std::memory_order_acquire) ||
+            writer_acquired.load(std::memory_order_acquire) ||
+            !g_stream_selected_cache.valid ||
+            g_stream_selected_cache.generation != generation_a) {
+            ok = 0;
+        }
+    }
+
+    if (reader_held) {
+        if (!cuda_stream_selected_consumer_end(
+                reader_generation, (cudaStream_t)0)) {
+            ok = 0;
+        }
+        reader_held = 0;
+    }
+    if (writer_thread.joinable()) writer_thread.join();
+    if (!writer_ok.load(std::memory_order_acquire) ||
+        !writer_acquired.load(std::memory_order_acquire) ||
+        !writer_published.load(std::memory_order_acquire) ||
+        !g_stream_selected_cache.valid ||
+        g_stream_selected_cache.generation != generation_b) {
+        ok = 0;
+    }
+
+    uint64_t reader_generation_b = 0;
+    cuda_stream_selected_binding binding_b = {};
+    int reader_b_held = 0;
+    if (writer_published.load(std::memory_order_acquire)) {
+        const int began = cuda_stream_selected_consumer_begin(
+            /*allow_streaming=*/1, (cudaStream_t)0,
+            &reader_generation_b);
+        reader_b_held = reader_generation_b != 0;
+        if (!began || reader_generation_b != generation_b ||
+            !cuda_stream_selected_binding_acquire(
+                &binding_b, 0, test_model_map, test_layer,
+                test_total_expert,
+                test_gate_offset, test_up_offset, test_down_offset,
+                expert_bytes, expert_bytes, weight_domain) ||
+            binding_b.generation != generation_b ||
+            binding_b.storage_kind !=
+                CUDA_STREAM_SELECTED_STORAGE_TRANSIENT ||
+            binding_b.upload_event_value == 0) {
+            ok = 0;
+        } else if (cudaMemcpyAsync(
+                       &host_words[2], binding_b.selected->ptr,
+                       sizeof(host_words[2]), cudaMemcpyDeviceToHost,
+                       (cudaStream_t)0) != cudaSuccess) {
+            (void)cudaGetLastError();
+            ok = 0;
+        }
+    }
+    if (reader_b_held) {
+        if (!cuda_stream_selected_consumer_end(
+                reader_generation_b, (cudaStream_t)0)) {
+            ok = 0;
+        }
+        reader_b_held = 0;
+    }
+    if (cudaStreamSynchronize((cudaStream_t)0) != cudaSuccess ||
+        !host_words || host_words[2] != host_words[1]) {
+        (void)cudaGetLastError();
+        ok = 0;
+    }
+
+    if (!ok) {
+        (void)cudaGetLastError();
+        (void)cudaDeviceSynchronize();
+    }
+    cuda_stream_selected_cache_release();
+    if (g_stream_selected_transient_storage.gate ||
+        g_stream_selected_transient_storage.up ||
+        g_stream_selected_transient_storage.down ||
+        g_stream_selected_transient_storage.remap ||
+        g_stream_selected_cache.valid || g_stream_selected_cache.gate_ptr ||
+        g_stream_selected_cache.up_ptr || g_stream_selected_cache.down_ptr ||
+        g_stream_selected_cache.slot_selected_ptr ||
+        g_stream_selected_cache.storage_kind !=
+            CUDA_STREAM_SELECTED_STORAGE_NONE) {
+        ok = 0;
+    }
+    if (persistent_touched) {
+        std::lock_guard<std::mutex> lock(
+            g_stream_expert_persistent_arena_mutex);
+        cuda_stream_expert_persistent_arena *arena =
+            &g_stream_expert_persistent_arena;
+        unsigned char base_got = 0;
+        int32_t remap_got = 0;
+        const int32_t resized_canary = INT32_C(0x27182818);
+        int isolation_ok = arena->base == persistent_base &&
+                           arena->remap == persistent_remap &&
+                           arena->bytes == persistent_bytes &&
+                           arena->remap_capacity >=
+                               7u * sizeof(int32_t) &&
+                           cudaSetDevice(arena->owner_device) == cudaSuccess &&
+                           cudaMemcpy(&base_got,
+                                      (char *)arena->base +
+                                          arena->bytes - 1u,
+                                      1u, cudaMemcpyDeviceToHost) ==
+                               cudaSuccess &&
+                           cudaMemcpy(&remap_got, arena->remap + 6u,
+                                      sizeof(remap_got),
+                                      cudaMemcpyDeviceToHost) == cudaSuccess &&
+                           base_got == persistent_base_canary &&
+                           remap_got == persistent_remap_canary;
+        if (isolation_ok) {
+            isolation_ok =
+                cuda_stream_expert_persistent_remap_ensure_locked(9u) &&
+                arena->base == persistent_base &&
+                arena->remap && arena->remap != persistent_remap &&
+                arena->remap_capacity >= 9u * sizeof(int32_t) &&
+                cudaMemcpy(arena->remap + 8u, &resized_canary,
+                           sizeof(resized_canary),
+                           cudaMemcpyHostToDevice) == cudaSuccess &&
+                cudaMemcpy(&remap_got, arena->remap + 8u,
+                           sizeof(remap_got),
+                           cudaMemcpyDeviceToHost) == cudaSuccess &&
+                cudaMemcpy(&base_got,
+                           (char *)arena->base + arena->bytes - 1u,
+                           1u, cudaMemcpyDeviceToHost) == cudaSuccess &&
+                remap_got == resized_canary &&
+                base_got == persistent_base_canary;
+        }
+        if (!isolation_ok) (void)cudaGetLastError();
+        const int arena_cleanup_ok =
+            cuda_stream_expert_persistent_arena_release_locked(1);
+        if (arena_cleanup_ok) {
+            arena->configured_budget = saved_arena_budget;
+            arena->configured_expert_bytes = saved_arena_total;
+            arena->gate_expert_bytes = saved_arena_gate;
+            arena->down_expert_bytes = saved_arena_down;
+        }
+        ok = ok && isolation_ok && arena_cleanup_ok;
+    }
+    if (test_model) (void)cudaFreeHost(test_model);
+    if (host_words) (void)cudaFreeHost(host_words);
+    cuda_stream_selected_stage_release();
+    {
+        cuda_stream_selected_writer_guard writer;
+        g_ssd_streaming_mode = saved_ssd_streaming_mode;
+    }
+    g_model_fd = saved_model_fd;
+    g_model_fd_host_base = saved_model_fd_host_base;
+    g_model_direct_fd = saved_model_direct_fd;
+    g_model_direct_align = saved_model_direct_align;
+    g_model_file_size = saved_model_file_size;
+    return ok;
+}
+
+/* Device-backed ordering oracle for CI/DGX bring-up.  It exercises the same
+ * event objects and streams as decode without requiring a GGUF or mutating
+ * process-environment policy after its once_flag has resolved. */
+extern "C" int ds4_cuda_test_stream_selected_event_pipeline(void) {
+    if (g_n_gpus != 1 ||
+        cudaSetDevice(g_gpu[0].device_id) != cudaSuccess ||
+        !cuda_stream_selected_event_pipeline_ensure() ||
+        !cuda_stream_selected_stage_pool_alloc(4096u) ||
+        !cuda_stream_selected_remap_stage_ensure(1u)) {
+        (void)cudaGetLastError();
+        return 0;
+    }
+
+    int32_t *source = NULL;
+    int32_t *uploaded = NULL;
+    int32_t *consumed = NULL;
+    int32_t *host = NULL;
+    int ok = cudaMalloc((void **)&source, sizeof(*source)) == cudaSuccess &&
+             cudaMalloc((void **)&uploaded, sizeof(*uploaded)) == cudaSuccess &&
+             cudaMalloc((void **)&consumed, sizeof(*consumed)) == cudaSuccess &&
+             cudaMallocHost((void **)&host, 2u * sizeof(*host)) == cudaSuccess;
+    if (!ok) (void)cudaGetLastError();
+
+    ds4_gpu_tensor tensor = {};
+    tensor.ptr = source;
+    tensor.bytes = sizeof(*source);
+    tensor.owner = 0;
+    tensor.device_id = 0;
+    if (ok) {
+        host[0] = 0x13572468;
+        host[1] = 0x24681357;
+        ok = cudaMemcpyAsync(source, &host[0], sizeof(host[0]),
+                             cudaMemcpyHostToDevice,
+                             cuda_decode_stream()) == cudaSuccess &&
+             cudaEventRecord(g_stream_selected_compute_ready_event,
+                             cuda_decode_stream()) == cudaSuccess;
+    }
+    uint64_t compute_value = 0;
+    if (ok) {
+        compute_value = ++g_stream_selected_compute_event_value;
+        if (compute_value == 0) {
+            compute_value = ++g_stream_selected_compute_event_value;
+        }
+        g_stream_selected_event_candidates.fetch_add(
+            1, std::memory_order_relaxed);
+        g_stream_selected_event_signals.fetch_add(
+            1, std::memory_order_relaxed);
+        int32_t got = 0;
+        ok = ds4_gpu_tensor_read_after_selected_event(
+                 &tensor, 0, &got, sizeof(got), compute_value,
+                 "selected event selftest readback") != 0 &&
+             got == host[0];
+    }
+
+    uint64_t upload_value = 0;
+    if (ok) {
+        g_stream_selected_remap_stage[0] = host[1];
+        ok = cudaMemcpyAsync(uploaded,
+                             g_stream_selected_remap_stage,
+                             sizeof(g_stream_selected_remap_stage[0]),
+                             cudaMemcpyHostToDevice,
+                             g_stream_selected_upload_stream) == cudaSuccess &&
+             cudaEventRecord(g_stream_selected_upload_done_event,
+                             g_stream_selected_upload_stream) == cudaSuccess;
+    }
+    if (ok) {
+        upload_value = ++g_stream_selected_upload_event_value;
+        if (upload_value == 0) {
+            upload_value = ++g_stream_selected_upload_event_value;
+        }
+        g_stream_selected_event_uploads.fetch_add(
+            1, std::memory_order_relaxed);
+        ok = ds4_gpu_stream_expert_cache_wait_selected_upload(
+                 upload_value, "selected event selftest upload") != 0 &&
+             cudaMemcpyAsync(consumed, uploaded, sizeof(*consumed),
+                             cudaMemcpyDeviceToDevice,
+                             cuda_decode_stream()) == cudaSuccess &&
+             cudaStreamSynchronize(cuda_decode_stream()) == cudaSuccess;
+    }
+    if (ok) {
+        int32_t got = 0;
+        ok = cudaMemcpy(&got, consumed, sizeof(got),
+                        cudaMemcpyDeviceToHost) == cudaSuccess &&
+             got == host[1];
+    }
+    g_stream_selected_event_oracle_runs.fetch_add(
+        1, std::memory_order_relaxed);
+    if (!ok) {
+        (void)cudaGetLastError();
+        g_stream_selected_event_oracle_failures.fetch_add(
+            1, std::memory_order_relaxed);
+        (void)cudaDeviceSynchronize();
+    }
+    if (source) (void)cudaFree(source);
+    if (uploaded) (void)cudaFree(uploaded);
+    if (consumed) (void)cudaFree(consumed);
+    if (host) (void)cudaFreeHost(host);
+    /* Keep the compute/readback/upload-done event resources for the report,
+     * but restore the selected-upload pool to the same cold state expected by
+     * the production batch-copy selftest which runs next. */
+    cuda_stream_selected_stage_release();
+    return ok;
+}
+
 /* Compatibility surface shared with the canonical Metal/ROCm graph. CUDA
  * either delegates to its equivalent primitive or reports an unavailable
  * optional fast path so the graph can use its established fallback. */
@@ -32931,33 +43508,82 @@ extern "C" void ds4_gpu_set_glm_model(bool enabled) {
 }
 
 extern "C" void ds4_gpu_set_ssd_streaming(bool enabled) {
+    if (enabled && !g_ssd_streaming_mode &&
+        !ds4_gpu_release_q4_attn_q_b_f16_sidecars()) {
+        fprintf(stderr,
+                "ds4: CUDA could not safely release Q4 attn_q_b F16 "
+                "sidecars while enabling SSD streaming\n");
+    }
+    if (enabled && !g_ssd_streaming_mode &&
+        !cuda_q4_attn_q_b_transient_f16_scratch_release()) {
+        fprintf(stderr,
+                "ds4: CUDA could not safely release Q4 attn_q_b transient "
+                "F16 scratch while enabling SSD streaming\n");
+    }
+    cuda_stream_selected_writer_guard writer;
     g_ssd_streaming_mode = enabled ? 1 : 0;
-    cuda_stream_selected_cache_invalidate();
-    if (!g_ssd_streaming_mode) cuda_stream_selected_cache_release();
+    if (!g_ssd_streaming_mode) {
+        (void)cuda_stream_expert_storage_release_writer(0);
+    }
 }
 
 extern "C" void ds4_gpu_set_streaming_expert_cache_budget(uint32_t experts) {
-    (void)experts;
+    {
+        std::lock_guard<std::mutex> lock(
+            g_stream_expert_persistent_arena_mutex);
+        if (experts ==
+            g_stream_expert_persistent_arena.configured_budget) return;
+    }
+    cuda_stream_selected_writer_guard writer;
+    std::lock_guard<std::mutex> lock(
+        g_stream_expert_persistent_arena_mutex);
+    if (!cuda_stream_expert_persistent_arena_release_locked(0)) return;
+    g_stream_expert_persistent_arena.configured_budget = experts;
 }
 
 extern "C" void ds4_gpu_set_streaming_expert_cache_expert_bytes(uint64_t bytes) {
-    (void)bytes;
+    {
+        std::lock_guard<std::mutex> lock(
+            g_stream_expert_persistent_arena_mutex);
+        if (bytes ==
+            g_stream_expert_persistent_arena.configured_expert_bytes) return;
+    }
+    cuda_stream_selected_writer_guard writer;
+    std::lock_guard<std::mutex> lock(
+        g_stream_expert_persistent_arena_mutex);
+    if (!cuda_stream_expert_persistent_arena_release_locked(1)) return;
+    g_stream_expert_persistent_arena.configured_expert_bytes = bytes;
 }
 
 extern "C" uint32_t ds4_gpu_stream_expert_cache_configured_count(void) {
-    return 0;
+    std::lock_guard<std::mutex> lock(
+        g_stream_expert_persistent_arena_mutex);
+    return g_stream_expert_persistent_runtime_ready &&
+           g_ssd_streaming_mode &&
+           cuda_stream_expert_persistent_requested() ?
+        g_stream_expert_persistent_arena.configured_budget : 0;
 }
 
 extern "C" uint32_t ds4_gpu_stream_expert_cache_current_count(void) {
-    return g_stream_selected_cache.valid ?
-        g_stream_selected_cache.compact_count : 0;
+    if (!g_stream_expert_persistent_runtime_ready ||
+        !g_ssd_streaming_mode ||
+        !cuda_stream_expert_persistent_requested()) {
+        std::lock_guard<std::mutex> lock(g_stream_selected_consume_mutex);
+        return !g_stream_selected_writer_active &&
+               g_stream_selected_cache.valid ?
+            g_stream_selected_cache.compact_count : 0;
+    }
+    std::lock_guard<std::mutex> lock(
+        g_stream_expert_persistent_arena_mutex);
+    return g_stream_expert_persistent_arena.base ?
+        g_stream_expert_persistent_arena.valid_count : 0;
 }
 
 extern "C" void ds4_gpu_stream_expert_cache_reset_route_hotness(void) {
 }
 
 extern "C" void ds4_gpu_stream_expert_cache_release_resident(void) {
-    cuda_stream_selected_cache_release();
+    (void)cuda_stream_expert_storage_release(1);
 }
 
 extern "C" int ds4_gpu_stream_expert_cache_seed_selected(
@@ -33075,10 +43701,17 @@ extern "C" int ds4_gpu_matmul_quant_kslice_tensor(
         uint64_t out_dim,
         const ds4_gpu_tensor *x,
         uint64_t x_elem_off) {
-    if (weight_type != 8u) return 0;
-    return ds4_gpu_matmul_q8_0_kslice_tensor(
-            out, model_map, model_size, weight_offset,
-            full_in_dim, k_off, k_cnt, out_dim, x, x_elem_off);
+    if (weight_type == 8u) {
+        return ds4_gpu_matmul_q8_0_kslice_tensor(
+                out, model_map, model_size, weight_offset,
+                full_in_dim, k_off, k_cnt, out_dim, x, x_elem_off);
+    }
+    if (weight_type == 12u) {
+        return cuda_matmul_q4_K_kslice_tensor(
+                out, model_map, model_size, weight_offset,
+                full_in_dim, k_off, k_cnt, out_dim, x, x_elem_off);
+    }
+    return 0;
 }
 
 extern "C" int ds4_gpu_matmul_q8_0_f16_out_tensor(
@@ -33098,18 +43731,291 @@ extern "C" int ds4_gpu_matmul_q8_0_f16_out_tensor(
 extern "C" int ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor(
         ds4_gpu_tensor *out, ds4_gpu_tensor *q_half,
         const void *model_map, uint64_t model_size, uint64_t weight_offset,
+        uint32_t weight_type,
         uint64_t in_dim, uint64_t out_dim, const ds4_gpu_tensor *x,
         uint32_t n_tok, uint32_t n_head, uint32_t head_dim,
         uint32_t n_rot, uint32_t pos0, uint32_t n_ctx_orig, bool inverse,
         float freq_base, float freq_scale, float ext_factor,
         float attn_factor, float beta_fast, float beta_slow, float eps) {
-    (void)out; (void)q_half; (void)model_map; (void)model_size;
-    (void)weight_offset; (void)in_dim; (void)out_dim; (void)x;
-    (void)n_tok; (void)n_head; (void)head_dim; (void)n_rot; (void)pos0;
-    (void)n_ctx_orig; (void)inverse; (void)freq_base; (void)freq_scale;
-    (void)ext_factor; (void)attn_factor; (void)beta_fast; (void)beta_slow;
-    (void)eps;
-    return 0;
+    /* The pre-existing CUDA Q8 specialization was a stub.  Preserve that
+     * fallback behavior and arm only resident Q4_K.  The release path keeps
+     * its established F32 projection boundary; an explicit diagnostic can
+     * stage an F16 result in the resident-only arena for CUDA validation. */
+    if (weight_type != CUDA_Q4_ATTN_Q_B_TYPE) return 0;
+    if (n_tok < 32u) return 0;
+    const int required = cuda_q4_attn_q_b_f16_required();
+    const int persistent_requested =
+        cuda_q4_attn_q_b_f16_requested();
+    const int persistent_disabled =
+        cuda_q4_attn_q_b_f16_disabled();
+    const int persistent_selected =
+        required || (persistent_requested && !persistent_disabled);
+    const uint32_t persistent_min_tokens =
+        cuda_q4_attn_q_b_f16_min_tokens();
+    if (persistent_selected && n_tok < persistent_min_tokens) return 0;
+    const int persistent_candidate =
+        persistent_selected && n_tok >= persistent_min_tokens;
+    const int strict = required && persistent_candidate;
+    const int use_persistent =
+        persistent_candidate && !persistent_disabled;
+    const int fallback = strict ? -1 : 0;
+    const int transient_candidate =
+        !cuda_q4_attn_q_b_transient_f16_disabled() &&
+        n_tok >= cuda_q4_attn_q_b_transient_f16_min_tokens();
+    if (!persistent_candidate && !transient_candidate) return 0;
+
+    if ((strict && persistent_disabled) ||
+        g_ssd_streaming_mode || g_quality_mode || g_n_gpus != 1 ||
+        !g_cublas_ready ||
+        !g_gpu[0].cublas_ready || !g_gpu[0].cublas ||
+        cuda_q4_attn_q_b_f16_multi_model_policy_active() ||
+        g_decode_graph_capturing || !out || !x || !model_map ||
+        !out->ptr || !x->ptr || n_head == 0u ||
+        head_dim == 0u || in_dim != CUDA_Q4_ATTN_Q_B_IN_DIM ||
+        out_dim != CUDA_Q4_ATTN_Q_B_OUT_DIM ||
+        out_dim != (uint64_t)n_head * head_dim || n_rot > head_dim ||
+        (n_rot & 1u) != 0u ||
+        ds4_tensor_device_idx(out) != 0 ||
+        ds4_tensor_device_idx(x) != 0 ||
+        n_tok > (uint32_t)INT_MAX ||
+        (uint64_t)n_tok * in_dim > (uint64_t)UINT32_MAX * 256u ||
+        (uint64_t)n_tok * n_head > UINT32_MAX ||
+        pos0 > UINT32_MAX - (n_tok - 1u)) {
+        return fallback;
+    }
+
+    const uint64_t x_bytes =
+        (uint64_t)n_tok * in_dim * sizeof(float);
+    const uint64_t out_bytes =
+        (uint64_t)n_tok * out_dim * sizeof(float);
+    if (x->bytes < x_bytes || out->bytes < out_bytes) {
+        return fallback;
+    }
+    const int use_f16_output =
+        cuda_q4_attn_q_b_f16_output_requested();
+    const uint64_t q_half_bytes =
+        (uint64_t)n_tok * out_dim * sizeof(__half);
+    int use_graph_q_half = 0;
+    if (use_f16_output && q_half && q_half->ptr &&
+        q_half->bytes >= q_half_bytes &&
+        ds4_tensor_device_idx(q_half) == 0) {
+        const uintptr_t q_half_addr = (uintptr_t)q_half->ptr;
+        const uintptr_t out_addr = (uintptr_t)out->ptr;
+        const uintptr_t x_addr = (uintptr_t)x->ptr;
+        const int overlaps_out = q_half_addr <= out_addr
+            ? (uint64_t)(out_addr - q_half_addr) < q_half_bytes
+            : (uint64_t)(q_half_addr - out_addr) < out_bytes;
+        const int overlaps_x = q_half_addr <= x_addr
+            ? (uint64_t)(x_addr - q_half_addr) < q_half_bytes
+            : (uint64_t)(q_half_addr - x_addr) < x_bytes;
+        use_graph_q_half = !overlaps_out && !overlaps_x;
+    }
+
+    const uint64_t blocks = in_dim / CUDA_QK_K;
+    const uint64_t row_bytes = blocks * sizeof(cuda_block_q4_K);
+    if (out_dim > UINT64_MAX / row_bytes || weight_offset > model_size) {
+        return fallback;
+    }
+    const uint64_t weight_bytes = out_dim * row_bytes;
+    if (weight_bytes > model_size - weight_offset) return fallback;
+
+    uint64_t transient_weight_bytes = 0;
+    uint64_t xh_offset = 0;
+    uint64_t qh_offset = 0;
+    uint64_t required_scratch_bytes = 0;
+    if (!cuda_q4_attn_q_b_transient_f16_scratch_size(
+            n_tok, use_f16_output,
+            &transient_weight_bytes, &xh_offset, &qh_offset,
+            &required_scratch_bytes)) {
+        return fallback;
+    }
+
+    /* Protect the shared W/X/output arena for the entire enqueue sequence.
+     * Stream ordering protects reuse after this function returns; host
+     * serialization protects the sequence itself from interleaving with
+     * another session. */
+    std::unique_lock<std::mutex> scratch_use_lock(
+        g_q4_attn_q_b_transient_f16_mutex);
+    if (!g_q4_attn_q_b_transient_f16_scratch ||
+        g_q4_attn_q_b_transient_f16_scratch_device != g_gpu[0].device_id ||
+        g_q4_attn_q_b_transient_f16_scratch_bytes <
+            required_scratch_bytes) {
+        return fallback;
+    }
+    __half *const scratch =
+        g_q4_attn_q_b_transient_f16_scratch;
+    __half *const xh = reinterpret_cast<__half *>(
+        reinterpret_cast<char *>(scratch) + xh_offset);
+    __half *const scratch_qh = use_f16_output
+        ? reinterpret_cast<__half *>(
+              reinterpret_cast<char *>(scratch) + qh_offset)
+        : NULL;
+    __half *const qh = use_graph_q_half
+        ? (__half *)q_half->ptr : scratch_qh;
+
+    /* Prefer an explicitly prepared persistent sidecar.  Keep its cache lock
+     * until the epilogue is enqueued so lifecycle release cannot free the
+     * matrix between lookup and its final consumer.  An optional cache miss
+     * may still use the default transient path. */
+    std::unique_lock<std::mutex> cache_use_lock(
+        g_q4_attn_q_b_f16_cache_mutex);
+    const __half *w_f16 = NULL;
+    int using_persistent = 0;
+    if (g_q4_attn_q_b_f16_multi_model_active) return fallback;
+    if (use_persistent) {
+        if (!g_q4_attn_q_b_f16_dispatch_disabled) {
+            w_f16 = cuda_q4_attn_q_b_f16_cache_lookup_locked(
+                model_map, model_size, weight_offset, weight_bytes,
+                in_dim, out_dim, g_gpu[0].device_id);
+        }
+        if (w_f16) {
+            using_persistent = 1;
+        } else {
+            if (strict) return -1;
+        }
+    }
+    if (!using_persistent &&
+        (!transient_candidate ||
+         g_q4_attn_q_b_transient_f16_runtime_disabled)) {
+        return fallback;
+    }
+
+    int previous_device = -1;
+    if (cudaGetDevice(&previous_device) != cudaSuccess) {
+        (void)cudaGetLastError();
+        return fallback;
+    }
+    if (cudaSetDevice(g_gpu[0].device_id) != cudaSuccess) {
+        (void)cudaGetLastError();
+        if (previous_device >= 0) (void)cudaSetDevice(previous_device);
+        return fallback;
+    }
+
+    const cudaStream_t stream = cuda_decode_stream();
+    const cublasHandle_t handle = cuda_cublas_for_tier(0);
+    cudaStream_t handle_stream = NULL;
+    if (cublasGetStream(handle, &handle_stream) != CUBLAS_STATUS_SUCCESS ||
+        handle_stream != stream) {
+        /* Never mutate the shared handle here.  A non-default stream means
+         * another subsystem owns its ordering; the native Q4 fallback is
+         * safer than racing the activation conversion. */
+        if (previous_device >= 0) (void)cudaSetDevice(previous_device);
+        return fallback;
+    }
+
+    const uint64_t xh_count = (uint64_t)n_tok * in_dim;
+    if (!using_persistent) {
+        const char *source =
+            model_map == g_model_host_base &&
+            g_model_device_owned && g_model_device_base
+                ? g_model_device_base + weight_offset : NULL;
+        if (!cuda_q4_attn_q_b_source_is_device_resident(
+                source, g_gpu[0].device_id)) {
+            if (previous_device >= 0) (void)cudaSetDevice(previous_device);
+            return fallback;
+        }
+        w_f16 = scratch;
+        const uint64_t chunks = transient_weight_bytes / (16u * sizeof(__half));
+        dequant_q4_K_to_f16_kernel<<<
+            (unsigned)((chunks + 255u) / 256u), 256, 0, stream>>>(
+                scratch,
+                reinterpret_cast<const cuda_block_q4_K *>(source),
+                in_dim, out_dim, blocks);
+        const cudaError_t dequant_err = cudaGetLastError();
+        if (dequant_err != cudaSuccess) {
+            fprintf(stderr,
+                    "ds4: CUDA Q4 attn_q_b transient dequantization failed: "
+                    "%s\n",
+                    cudaGetErrorString(dequant_err));
+            g_q4_attn_q_b_transient_f16_runtime_disabled = 1;
+            if (previous_device >= 0) (void)cudaSetDevice(previous_device);
+            return fallback;
+        }
+    }
+
+    /* Prefill is deliberately excluded from decode-graph capture above, so
+     * dequantization, activation conversion, cuBLAS, and the epilogue all ride
+     * one stream.  Avoid retargeting the shared handle on this hot path. */
+    f32_to_f16_kernel<<<
+        (unsigned)((xh_count + 255u) / 256u), 256, 0, stream>>>(
+            xh, (const float *)x->ptr, xh_count);
+    cudaError_t launch_err = cudaGetLastError();
+    if (launch_err != cudaSuccess) {
+        fprintf(stderr,
+                "ds4: CUDA Q4 attn_q_b activation conversion failed: %s\n",
+                cudaGetErrorString(launch_err));
+        if (using_persistent) {
+            cuda_q4_attn_q_b_f16_runtime_failure_evict(
+                &cache_use_lock, &scratch_use_lock);
+        } else {
+            g_q4_attn_q_b_transient_f16_runtime_disabled = 1;
+        }
+        if (previous_device >= 0) (void)cudaSetDevice(previous_device);
+        return fallback;
+    }
+
+    const float alpha = 1.0f;
+    const float beta = 0.0f;
+    const cublasStatus_t status = cublasGemmEx(
+        handle,
+        CUBLAS_OP_T, CUBLAS_OP_N,
+        (int)out_dim, (int)n_tok, (int)in_dim,
+        &alpha,
+        w_f16, CUDA_R_16F, (int)in_dim,
+        xh, CUDA_R_16F, (int)in_dim,
+        &beta,
+        use_f16_output ? (void *)qh : out->ptr,
+        use_f16_output ? CUDA_R_16F : CUDA_R_32F, (int)out_dim,
+        CUDA_R_32F,
+        CUBLAS_GEMM_DEFAULT);
+    if (status != CUBLAS_STATUS_SUCCESS) {
+        fprintf(stderr,
+                "ds4: CUDA Q4 attn_q_b F16 GEMM failed: status %d\n",
+                (int)status);
+        if (using_persistent) {
+            cuda_q4_attn_q_b_f16_runtime_failure_evict(
+                &cache_use_lock, &scratch_use_lock);
+        } else {
+            g_q4_attn_q_b_transient_f16_runtime_disabled = 1;
+        }
+        if (previous_device >= 0) (void)cudaSetDevice(previous_device);
+        return fallback;
+    }
+
+    /* cuBLAS has accepted the first output writer.  From this point onward a
+     * native-Q4 replay is unsafe even when the specialization was optional. */
+    if (use_f16_output) {
+        head_rms_norm_rope_tail_from_half_kernel<<<
+            n_tok * n_head, 256, 0, stream>>>(
+                (float *)out->ptr, qh,
+                n_tok, n_head, head_dim, n_rot, pos0, n_ctx_orig,
+                inverse ? 1 : 0, freq_base, freq_scale, ext_factor,
+                attn_factor, beta_fast, beta_slow, eps);
+    } else {
+        head_rms_norm_rope_tail_kernel<<<
+            n_tok * n_head, 256, 0, stream>>>(
+                (float *)out->ptr,
+                n_tok, n_head, head_dim, n_rot, pos0, n_ctx_orig,
+                inverse ? 1 : 0, freq_base, freq_scale, ext_factor,
+                attn_factor, beta_fast, beta_slow, eps);
+    }
+    launch_err = cudaGetLastError();
+    if (launch_err != cudaSuccess) {
+        fprintf(stderr,
+                "ds4: CUDA Q4 attn_q_b F16 RMS/RoPE epilogue failed: %s\n",
+                cudaGetErrorString(launch_err));
+        if (using_persistent) {
+            cuda_q4_attn_q_b_f16_runtime_failure_evict(
+                &cache_use_lock, &scratch_use_lock);
+        } else {
+            g_q4_attn_q_b_transient_f16_runtime_disabled = 1;
+        }
+        if (previous_device >= 0) (void)cudaSetDevice(previous_device);
+        return -1;
+    }
+
+    if (previous_device >= 0) (void)cudaSetDevice(previous_device);
+    return 1;
 }
 
 extern "C" int ds4_gpu_attention_prefill_raw_heads_range_tensor(
@@ -33150,6 +44056,22 @@ extern "C" int ds4_gpu_attention_output_q8_batch_f16_tensor(
     return 0;
 }
 
+static void cuda_q4_grouped_attn_a_oracle_register_report(void);
+static uint64_t g_q4_grouped_attn_a_oracle_calls;
+static uint64_t g_q4_grouped_attn_a_oracle_mismatches;
+static uint64_t g_q4_grouped_attn_a_oracle_skips;
+static uint64_t g_q4_grouped_attn_a_oracle_batch_candidates;
+static uint64_t g_q4_grouped_attn_a_oracle_batch_calls;
+static uint64_t g_q4_grouped_attn_a_oracle_batch_mismatches;
+static uint64_t g_q4_grouped_attn_a_oracle_batch_skips;
+static int g_q4_grouped_attn_a_oracle_report_registered;
+static int g_q4_grouped_attn_a_oracle_mismatch_reported;
+static int cuda_q4_grouped_attn_a_batch_oracle(
+        const char *out_a, const float *heads, float *low,
+        uint32_t n_tokens, uint32_t n_groups,
+        uint32_t group_dim, uint32_t rank,
+        int logical_tier, cudaStream_t stream);
+
 extern "C" int ds4_gpu_attention_output_q4_K_batch_tensor(
         ds4_gpu_tensor *out, ds4_gpu_tensor *low,
         ds4_gpu_tensor *group_tmp, ds4_gpu_tensor *low_tmp,
@@ -33157,22 +44079,547 @@ extern "C" int ds4_gpu_attention_output_q4_K_batch_tensor(
         uint64_t out_a_offset, uint64_t out_b_offset, uint32_t out_b_type,
         uint64_t group_dim, uint64_t rank, uint32_t n_groups,
         uint64_t out_dim, const ds4_gpu_tensor *heads, uint32_t n_tokens) {
-    (void)out; (void)low; (void)group_tmp; (void)low_tmp;
-    (void)model_map; (void)model_size; (void)out_a_offset;
-    (void)out_b_offset; (void)out_b_type; (void)group_dim; (void)rank;
-    (void)n_groups; (void)out_dim; (void)heads; (void)n_tokens;
-    return 0;
+    const int grouped_batch_require = cuda_env_flag_enabled(
+        "DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_BATCH", 0);
+    const int grouped_prefill_require = cuda_env_flag_enabled(
+        "DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_PREFILL", 0);
+    const int grouped_single_grid_require = cuda_env_flag_enabled(
+        "DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_SINGLE_GRID", 0);
+    const int grouped_q81_require = cuda_env_flag_enabled(
+        "DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_Q81", 0);
+    const int any_grouped_require = grouped_batch_require ||
+        grouped_prefill_require || grouped_single_grid_require ||
+        grouped_q81_require;
+    if (!out || !low || !group_tmp || !low_tmp || !heads || !model_map ||
+        group_dim == 0 || rank == 0 || n_groups == 0 || out_dim == 0 ||
+        n_tokens < 2u || group_dim > INT_MAX || rank > INT_MAX ||
+        out_dim > INT_MAX || n_tokens > INT_MAX || !cuda_use_mmq()) {
+        return any_grouped_require ? -1 : 0;
+    }
+    const uint64_t low_dim = (uint64_t)n_groups * rank;
+    if (low_dim > INT_MAX || (group_dim % CUDA_QK_K) != 0u ||
+        (low_dim % CUDA_QK_K) != 0u) {
+        return any_grouped_require ? -1 : 0;
+    }
+    const uint64_t row_a_bytes =
+        (group_dim / CUDA_QK_K) * sizeof(cuda_block_q4_K);
+    if (rank > UINT64_MAX / row_a_bytes ||
+        n_groups > UINT64_MAX / (rank * row_a_bytes)) {
+        return any_grouped_require ? -1 : 0;
+    }
+    const uint64_t out_a_bytes = (uint64_t)n_groups * rank * row_a_bytes;
+    if (out_a_offset > model_size ||
+        out_a_bytes > model_size - out_a_offset ||
+        heads->bytes < (uint64_t)n_tokens * n_groups * group_dim * sizeof(float) ||
+        low->bytes < (uint64_t)n_tokens * low_dim * sizeof(float) ||
+        out->bytes < (uint64_t)n_tokens * out_dim * sizeof(float) ||
+        group_tmp->bytes < (uint64_t)n_tokens * group_dim * sizeof(float) ||
+        low_tmp->bytes < (uint64_t)n_tokens * rank * sizeof(float)) {
+        return any_grouped_require ? -1 : 0;
+    }
+    const int logical_tier = ds4_tensor_device_idx(out);
+    const char *out_a = cuda_resolve_weight_ptr(
+        model_map, out_a_offset, out_a_bytes, logical_tier, "q4 attn_out_a");
+    if (!out_a) {
+        return any_grouped_require ? -1 : 0;
+    }
+    const int grouped_oracle = cuda_env_flag_enabled(
+        "DS4_CUDA_Q4_GROUPED_ATTN_A_ORACLE", 0);
+    const int grouped_batch_enable = cuda_env_flag_enabled(
+        "DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_BATCH", 0);
+    /* The direct-strided eight-grid path preserves the canonical MMQ
+     * reduction tree and is the GB10 prefill default. Exact ENABLE=0 is a
+     * compatibility opt-out; the presence-based NO switch remains the
+     * authoritative rollback. The separate grid.z candidate stays opt-in. */
+    const int grouped_prefill_enable = cuda_env_flag_enabled(
+        "DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_PREFILL", 1);
+    const int grouped_prefill_disable =
+        getenv("DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL") != NULL;
+    const int grouped_single_grid_enable = cuda_env_flag_enabled(
+        "DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_SINGLE_GRID", 0);
+    const int grouped_single_grid_disable = cuda_env_flag_enabled(
+        "DS4_CUDA_DISABLE_Q4_GROUPED_ATTN_A_SINGLE_GRID", 0);
+    const int grouped_q81_disable =
+        getenv("DS4_CUDA_NO_Q4_GROUPED_ATTN_A_Q81") != NULL;
+    const int grouped_single_grid_selected =
+        (grouped_single_grid_enable || grouped_single_grid_require) &&
+        !grouped_single_grid_disable;
+    const int grouped_prefill_selected = grouped_prefill_enable ||
+        grouped_prefill_require || grouped_single_grid_selected ||
+        grouped_q81_require;
+    if (grouped_oracle) cuda_q4_grouped_attn_a_oracle_register_report();
+
+    const int grouped_gb10 =
+        n_tokens <= 8u && n_groups <= INT_MAX &&
+        ds4_tensor_device_idx(low) == logical_tier &&
+        ds4_tensor_device_idx(heads) == logical_tier &&
+        cuda_q4_gb10_fast_path_enabled(
+            logical_tier, "DS4_CUDA_NO_Q4_GROUPED_ATTN_A");
+    const int grouped_prefill_gb10 =
+        n_tokens > 8u && n_groups <= INT_MAX &&
+        ds4_tensor_device_idx(low) == logical_tier &&
+        ds4_tensor_device_idx(heads) == logical_tier &&
+        !grouped_prefill_disable &&
+        cuda_q4_gb10_fast_path_enabled(
+            logical_tier, "DS4_CUDA_NO_Q4_GROUPED_ATTN_A");
+    if (grouped_batch_require &&
+        (!grouped_batch_enable || !grouped_gb10)) {
+        fprintf(stderr,
+                "ds4: required CUDA Q4 grouped attention-A batch path is "
+                "not eligible\n");
+        return -1;
+    }
+    if (grouped_prefill_require && !grouped_prefill_gb10) {
+        fprintf(stderr,
+                "ds4: required CUDA Q4 grouped attention-A prefill path "
+                "is not eligible\n");
+        return -1;
+    }
+    if (grouped_single_grid_require &&
+        (grouped_single_grid_disable || !grouped_prefill_gb10)) {
+        fprintf(stderr,
+                "ds4: required CUDA Q4 grouped attention-A single-grid "
+                "path is not eligible\n");
+        return -1;
+    }
+    if (grouped_q81_require &&
+        (grouped_q81_disable || !grouped_prefill_gb10 ||
+         rank != 1024u || group_dim != 4096u || n_groups != 8u ||
+         n_tokens > (uint32_t)(INT32_MAX / (8*4096)))) {
+        fprintf(stderr,
+                "ds4: required CUDA Q4 grouped attention-A K4096/G8 "
+                "Q8_1 quantizer is not eligible\n");
+        return -1;
+    }
+    if (grouped_prefill_selected && grouped_prefill_gb10) {
+        int rc = grouped_single_grid_selected
+            ? ds4_mmq_q4_K_grouped_dense_single_grid(
+                  out_a, (const float *)heads->ptr, (float *)low->ptr,
+                  (int)rank, (int)n_tokens, (int)group_dim, (int)n_groups,
+                  cuda_decode_stream())
+            : ds4_mmq_q4_K_grouped_dense(
+                  out_a, (const float *)heads->ptr, (float *)low->ptr,
+                  (int)rank, (int)n_tokens, (int)group_dim, (int)n_groups,
+                  cuda_decode_stream());
+        /* NOT_APPLICABLE is returned before allocation or enqueue, so the
+         * opt-in single-grid candidate can safely fall back to the default
+         * eight-grid grouped implementation. Every other failure may follow
+         * an enqueue and remains fail-closed. */
+        if (rc == DS4_MMQ_NOT_APPLICABLE && grouped_single_grid_selected) {
+            if (grouped_single_grid_require) {
+                fprintf(stderr,
+                        "ds4: required CUDA Q4 grouped attention-A "
+                        "single-grid dispatch was not applicable\n");
+                return -1;
+            }
+            rc = ds4_mmq_q4_K_grouped_dense(
+                out_a, (const float *)heads->ptr, (float *)low->ptr,
+                (int)rank, (int)n_tokens, (int)group_dim, (int)n_groups,
+                cuda_decode_stream());
+        }
+        if (rc != 0) {
+            fprintf(stderr,
+                    "ds4: CUDA Q4 grouped attention-A %s returned %d; "
+                    "failing closed after candidate dispatch\n",
+                    grouped_single_grid_selected ? "single-grid" : "prefill",
+                    rc);
+            return -1;
+        }
+    } else if (grouped_gb10) {
+        /* DSpark verification stores [token][group][K].  The opt-in batch
+         * entry flattens (token, group) into channels while keeping
+         * ncols_dst=1, preserving the canonical per-pair Q8_1 quantization
+         * and MMVQ reduction.  NOT_APPLICABLE is guaranteed pre-enqueue, so
+         * it safely falls back to the established per-token grouped loop.
+         * Any other error may follow an enqueue and therefore fails closed. */
+        const int batch_rc = ds4_mmq_q4_K_grouped_batch_vec(
+            out_a, (const float *)heads->ptr, (float *)low->ptr,
+            (int)rank, (int)group_dim, (int)n_tokens, (int)n_groups,
+            cuda_decode_stream());
+        if (batch_rc == DS4_MMQ_NOT_APPLICABLE) {
+            if (grouped_batch_require) {
+                fprintf(stderr,
+                        "ds4: required CUDA Q4 grouped attention-A batch "
+                        "dispatch was not applicable\n");
+                return -1;
+            }
+            for (uint32_t t = 0; t < n_tokens; t++) {
+                const int rc = ds4_mmq_q4_K_grouped_vec(
+                    out_a,
+                    (const float *)heads->ptr +
+                        (uint64_t)t * n_groups * group_dim,
+                    (float *)low->ptr + (uint64_t)t * low_dim,
+                    (int)rank, (int)group_dim, (int)n_groups,
+                    cuda_decode_stream());
+                if (rc == DS4_MMQ_NOT_APPLICABLE) return -1;
+                if (rc != 0) return -1;
+            }
+        } else if (batch_rc != 0) {
+            return -1;
+        } else if (grouped_oracle) {
+            g_q4_grouped_attn_a_oracle_batch_candidates++;
+        }
+        if (batch_rc == 0 && grouped_oracle &&
+                   !cuda_q4_grouped_attn_a_batch_oracle(
+                       out_a, (const float *)heads->ptr,
+                       (float *)low->ptr, n_tokens, n_groups,
+                       (uint32_t)group_dim, (uint32_t)rank,
+                       logical_tier, cuda_decode_stream())) {
+            return -1;
+        }
+    } else {
+        /* Existing cross-CUDA path: pack one group at a time, run a
+         * token-batched MMQ, then scatter rank rows back to token-major low.
+         * Keep this byte-for-byte outside the new GB10 dispatch. */
+        for (uint32_t g = 0; g < n_groups; g++) {
+            cudaError_t ce = cudaMemcpy2DAsync(
+                group_tmp->ptr, group_dim * sizeof(float),
+                (const float *)heads->ptr + (uint64_t)g * group_dim,
+                (uint64_t)n_groups * group_dim * sizeof(float),
+                group_dim * sizeof(float), n_tokens,
+                cudaMemcpyDeviceToDevice, cuda_decode_stream());
+            if (!cuda_ok(ce, "q4 attention output heads pack")) return -1;
+            const int rc = ds4_mmq_q4_K_dense(
+                out_a + (uint64_t)g * rank * row_a_bytes,
+                (const float *)group_tmp->ptr, (float *)low_tmp->ptr,
+                (int)rank, (int)n_tokens, (int)group_dim,
+                cuda_decode_stream());
+            if (rc != 0) return -1;
+            ce = cudaMemcpy2DAsync(
+                (float *)low->ptr + (uint64_t)g * rank,
+                low_dim * sizeof(float), low_tmp->ptr, rank * sizeof(float),
+                rank * sizeof(float), n_tokens,
+                cudaMemcpyDeviceToDevice, cuda_decode_stream());
+            if (!cuda_ok(ce, "q4 attention output low unpack")) return -1;
+        }
+    }
+    int b_rc = 0;
+    if (out_b_type == 12u) {
+        b_rc = cuda_matmul_q4_K_tensor(out, model_map, model_size,
+                                       out_b_offset, low_dim, out_dim,
+                                       low, n_tokens);
+    } else if (out_b_type == 8u) {
+        b_rc = cuda_matmul_q8_0_tensor_labeled(
+            out, model_map, model_size, out_b_offset, low_dim, out_dim,
+            low, n_tokens, "q4 attn_output_b");
+    }
+    /* Attention-A has already been enqueued.  A B rejection or launch error
+     * can no longer request whole-operation fallback without replaying A, so
+     * preserve the tri-state compound-operation contract and fail closed. */
+    return b_rc > 0 ? 1 : -1;
+}
+
+static void cuda_q4_grouped_attn_a_oracle_report(void) {
+    fprintf(stderr,
+            "ds4: CUDA Q4 grouped attention-A oracle: "
+            "calls=%llu mismatches=%llu skips=%llu "
+            "batch_candidates=%llu batch_calls=%llu "
+            "batch_mismatches=%llu batch_skips=%llu "
+            "(canonical MMVQ output retained)\n",
+            (unsigned long long)g_q4_grouped_attn_a_oracle_calls,
+            (unsigned long long)g_q4_grouped_attn_a_oracle_mismatches,
+            (unsigned long long)g_q4_grouped_attn_a_oracle_skips,
+            (unsigned long long)g_q4_grouped_attn_a_oracle_batch_candidates,
+            (unsigned long long)g_q4_grouped_attn_a_oracle_batch_calls,
+            (unsigned long long)g_q4_grouped_attn_a_oracle_batch_mismatches,
+            (unsigned long long)g_q4_grouped_attn_a_oracle_batch_skips);
+}
+
+static void cuda_q4_grouped_attn_a_oracle_register_report(void) {
+    if (!g_q4_grouped_attn_a_oracle_report_registered) {
+        g_q4_grouped_attn_a_oracle_report_registered = 1;
+        (void)atexit(cuda_q4_grouped_attn_a_oracle_report);
+    }
+}
+
+static int cuda_q4_grouped_attn_a_batch_reference(
+        const char *out_a, const float *heads, float *reference,
+        uint32_t n_tokens, uint32_t n_groups,
+        uint32_t group_dim, uint32_t rank,
+        cudaStream_t stream) {
+    const uint64_t heads_token_stride =
+        (uint64_t)n_groups * group_dim;
+    const uint64_t low_token_stride =
+        (uint64_t)n_groups * rank;
+    for (uint32_t t = 0; t < n_tokens; t++) {
+        const int rc = ds4_mmq_q4_K_grouped_vec(
+            out_a,
+            heads + (uint64_t)t * heads_token_stride,
+            reference + (uint64_t)t * low_token_stride,
+            (int)rank, (int)group_dim, (int)n_groups, stream);
+        if (rc != 0) return 0;
+    }
+    return 1;
+}
+
+/* Diagnostic oracle for the token-aware grouped dispatch.  The old
+ * per-token grouped loop is the canonical reference.  With the oracle on,
+ * that reference always replaces the candidate before the output-B consumer;
+ * the candidate is used only for a bitwise comparison.  Decode graphs are
+ * disabled globally for this env, but a capture check remains here so dynamic
+ * env changes and foreign captures still retain canonical output without a
+ * host synchronization. */
+static int cuda_q4_grouped_attn_a_batch_oracle(
+        const char *out_a, const float *heads, float *low,
+        uint32_t n_tokens, uint32_t n_groups,
+        uint32_t group_dim, uint32_t rank,
+        int logical_tier, cudaStream_t stream) {
+    cudaStreamCaptureStatus capture = cudaStreamCaptureStatusNone;
+    const cudaError_t capture_err = cudaStreamIsCapturing(stream, &capture);
+    if (capture_err != cudaSuccess ||
+        capture != cudaStreamCaptureStatusNone) {
+        (void)cudaGetLastError();
+        g_q4_grouped_attn_a_oracle_skips++;
+        g_q4_grouped_attn_a_oracle_batch_skips++;
+        return cuda_q4_grouped_attn_a_batch_reference(
+            out_a, heads, low, n_tokens, n_groups,
+            group_dim, rank, stream);
+    }
+
+    const uint64_t low_elems =
+        (uint64_t)n_tokens * n_groups * rank;
+    if (low_elems > UINT64_MAX / sizeof(float)) return 0;
+    const uint64_t reference_bytes = low_elems * sizeof(float);
+    if (reference_bytes > UINT64_MAX - 255u) return 0;
+    const uint64_t mismatch_offset =
+        (reference_bytes + 255u) & ~255ull;
+    if (mismatch_offset > UINT64_MAX - sizeof(uint32_t)) return 0;
+    unsigned char *scratch = (unsigned char *)cuda_tmp_alloc_on(
+        logical_tier, mismatch_offset + sizeof(uint32_t),
+        "q4 grouped attention-A batch oracle");
+    if (!scratch) {
+        g_q4_grouped_attn_a_oracle_skips++;
+        g_q4_grouped_attn_a_oracle_batch_skips++;
+        return cuda_q4_grouped_attn_a_batch_reference(
+            out_a, heads, low, n_tokens, n_groups,
+            group_dim, rank, stream);
+    }
+    float *reference = (float *)scratch;
+    uint32_t *mismatch_device =
+        (uint32_t *)(scratch + mismatch_offset);
+    if (!cuda_q4_grouped_attn_a_batch_reference(
+            out_a, heads, reference, n_tokens, n_groups,
+            group_dim, rank, stream)) {
+        return 0;
+    }
+
+    if (!cuda_ok(cudaMemsetAsync(mismatch_device, 0, sizeof(uint32_t),
+                                 stream),
+                 "clear q4 grouped attention-A batch oracle")) {
+        g_q4_grouped_attn_a_oracle_skips++;
+        g_q4_grouped_attn_a_oracle_batch_skips++;
+        return cuda_ok(cudaMemcpyAsync(
+                           low, reference, reference_bytes,
+                           cudaMemcpyDeviceToDevice, stream),
+                       "retain q4 grouped attention-A batch reference");
+    }
+    q4_K_attn_hc_bitwise_compare_kernel
+        <<<(unsigned)((low_elems + 255u) / 256u), 256, 0, stream>>>(
+            mismatch_device, reference, low, low_elems);
+    if (!cuda_ok(cudaGetLastError(),
+                 "q4 grouped attention-A batch oracle compare launch")) {
+        g_q4_grouped_attn_a_oracle_skips++;
+        g_q4_grouped_attn_a_oracle_batch_skips++;
+        return cuda_ok(cudaMemcpyAsync(
+                           low, reference, reference_bytes,
+                           cudaMemcpyDeviceToDevice, stream),
+                       "retain q4 grouped attention-A batch reference");
+    }
+
+    uint32_t mismatch_host = 0u;
+    const cudaError_t retain_err = cudaMemcpyAsync(
+        low, reference, reference_bytes, cudaMemcpyDeviceToDevice, stream);
+    if (retain_err != cudaSuccess) {
+        return cuda_ok(retain_err,
+                       "retain q4 grouped attention-A batch reference");
+    }
+    const cudaError_t read_err = cudaMemcpyAsync(
+        &mismatch_host, mismatch_device, sizeof(mismatch_host),
+        cudaMemcpyDeviceToHost, stream);
+    if (read_err != cudaSuccess) {
+        (void)cuda_ok(read_err,
+                      "read q4 grouped attention-A batch oracle");
+        g_q4_grouped_attn_a_oracle_skips++;
+        g_q4_grouped_attn_a_oracle_batch_skips++;
+        return cuda_ok(cudaStreamSynchronize(stream),
+                       "synchronize retained q4 grouped batch reference");
+    }
+    if (!cuda_ok(cudaStreamSynchronize(stream),
+                 "synchronize q4 grouped attention-A batch oracle")) {
+        return 0;
+    }
+
+    g_q4_grouped_attn_a_oracle_calls++;
+    g_q4_grouped_attn_a_oracle_batch_calls++;
+    if (mismatch_host != 0u) {
+        g_q4_grouped_attn_a_oracle_mismatches++;
+        g_q4_grouped_attn_a_oracle_batch_mismatches++;
+        if (!g_q4_grouped_attn_a_oracle_mismatch_reported) {
+            g_q4_grouped_attn_a_oracle_mismatch_reported = 1;
+            fprintf(stderr,
+                    "ds4: CUDA Q4 grouped attention-A oracle found a "
+                    "bitwise batch mismatch; retained per-token MMVQ "
+                    "output\n");
+        }
+    }
+    return 1;
 }
 
 extern "C" int ds4_gpu_attention_output_low_q4_K_slice_tensor(
         ds4_gpu_tensor *low, const void *model_map, uint64_t model_size,
         uint64_t out_a_offset, uint64_t group_dim, uint64_t rank,
         uint32_t group0, uint32_t group_cnt,
-        const ds4_gpu_tensor *heads) {
-    (void)low; (void)model_map; (void)model_size; (void)out_a_offset;
-    (void)group_dim; (void)rank; (void)group0; (void)group_cnt;
-    (void)heads;
-    return 0;
+        const ds4_gpu_tensor *heads, int resident_decode) {
+    (void)resident_decode;
+    const int oracle = cuda_env_flag_enabled(
+        "DS4_CUDA_Q4_GROUPED_ATTN_A_ORACLE", 0);
+    if (oracle) cuda_q4_grouped_attn_a_oracle_register_report();
+    if (!low || !heads || !model_map || group_dim == 0u || rank == 0u ||
+        group_cnt == 0u || group0 > UINT32_MAX - group_cnt ||
+        group_dim > INT_MAX || rank > INT_MAX || group_cnt > INT_MAX ||
+        (group_dim % CUDA_QK_K) != 0u) {
+        return 0;
+    }
+    const int logical_tier = ds4_tensor_device_idx(low);
+    if (ds4_tensor_device_idx(heads) != logical_tier ||
+        !cuda_q4_gb10_fast_path_enabled(
+            logical_tier, "DS4_CUDA_NO_Q4_GROUPED_ATTN_A")) {
+        return 0;
+    }
+
+    const uint64_t blocks = group_dim / CUDA_QK_K;
+    if (blocks == 0u ||
+        blocks > UINT64_MAX / sizeof(cuda_block_q4_K)) {
+        return 0;
+    }
+    const uint64_t row_bytes = blocks * sizeof(cuda_block_q4_K);
+    if (rank > UINT64_MAX / row_bytes ||
+        group_dim > UINT64_MAX / group_cnt ||
+        rank > UINT64_MAX / group_cnt) {
+        return 0;
+    }
+    const uint64_t group_weight_bytes = rank * row_bytes;
+    if ((uint64_t)group0 > UINT64_MAX / group_weight_bytes ||
+        (uint64_t)group_cnt > UINT64_MAX / group_weight_bytes) {
+        return 0;
+    }
+    const uint64_t group_weight_offset =
+        (uint64_t)group0 * group_weight_bytes;
+    const uint64_t selected_weight_bytes =
+        (uint64_t)group_cnt * group_weight_bytes;
+    if (out_a_offset > model_size ||
+        group_weight_offset > model_size - out_a_offset) {
+        return 0;
+    }
+    const uint64_t selected_offset = out_a_offset + group_weight_offset;
+    if (selected_weight_bytes > model_size - selected_offset) return 0;
+
+    const uint64_t heads_elems = (uint64_t)group_cnt * group_dim;
+    const uint64_t low_elems = (uint64_t)group_cnt * rank;
+    if (heads_elems > UINT64_MAX / sizeof(float) ||
+        low_elems > UINT64_MAX / sizeof(float) ||
+        heads->bytes < heads_elems * sizeof(float) ||
+        low->bytes < low_elems * sizeof(float)) {
+        return 0;
+    }
+
+    const char *out_a = cuda_resolve_weight_ptr(
+        model_map, selected_offset, selected_weight_bytes, logical_tier,
+        "q4 grouped attn_out_a");
+    if (!out_a) return 0;
+    cudaStream_t stream = cuda_decode_stream();
+    const int rc = ds4_mmq_q4_K_grouped_vec(
+        out_a, (const float *)heads->ptr, (float *)low->ptr,
+        (int)rank, (int)group_dim, (int)group_cnt, stream);
+    if (rc != 0) {
+        /* NOT_APPLICABLE is pre-enqueue and cleanly retries the established
+         * per-group loop. A negative result may follow a launch and must
+         * propagate so the graph cannot replay work over a partial result. */
+        return rc < 0 ? -1 : 0;
+    }
+
+    if (!oracle) return 1;
+
+    cudaStreamCaptureStatus capture = cudaStreamCaptureStatusNone;
+    const cudaError_t capture_err = cudaStreamIsCapturing(stream, &capture);
+    if (capture_err != cudaSuccess ||
+        capture != cudaStreamCaptureStatusNone) {
+        (void)cudaGetLastError();
+        g_q4_grouped_attn_a_oracle_skips++;
+        return 1;
+    }
+
+    const uint64_t reference_bytes = low_elems * sizeof(float);
+    const uint64_t mismatch_offset =
+        (reference_bytes + 255u) & ~255ull;
+    if (mismatch_offset < reference_bytes ||
+        mismatch_offset > UINT64_MAX - sizeof(uint32_t)) {
+        g_q4_grouped_attn_a_oracle_skips++;
+        return 1;
+    }
+    unsigned char *scratch = (unsigned char *)cuda_tmp_alloc_on(
+        logical_tier, mismatch_offset + sizeof(uint32_t),
+        "q4 grouped attention-A oracle");
+    if (!scratch) {
+        g_q4_grouped_attn_a_oracle_skips++;
+        return 1;
+    }
+    float *reference = (float *)scratch;
+    uint32_t *mismatch_device =
+        (uint32_t *)(scratch + mismatch_offset);
+
+    int reference_ok = 1;
+    for (uint32_t g = 0; g < group_cnt; g++) {
+        const int group_rc = ds4_mmq_q4_K_dense_vec(
+            out_a + (uint64_t)g * group_weight_bytes,
+            (const float *)heads->ptr + (uint64_t)g * group_dim,
+            reference + (uint64_t)g * rank,
+            (int)rank, 1, (int)group_dim, stream);
+        if (group_rc != 0) {
+            reference_ok = 0;
+            break;
+        }
+    }
+    if (!reference_ok ||
+        !cuda_ok(cudaMemsetAsync(mismatch_device, 0, sizeof(uint32_t),
+                                 stream),
+                 "clear q4 grouped attention-A oracle")) {
+        g_q4_grouped_attn_a_oracle_skips++;
+        return 1;
+    }
+    q4_K_attn_hc_bitwise_compare_kernel
+        <<<(low_elems + 255u) / 256u, 256, 0, stream>>>(
+            mismatch_device, reference, (const float *)low->ptr, low_elems);
+    if (!cuda_ok(cudaGetLastError(),
+                 "q4 grouped attention-A oracle compare launch")) {
+        return 0;
+    }
+
+    uint32_t mismatch_host = 0u;
+    if (!cuda_ok(cudaMemcpyAsync(&mismatch_host, mismatch_device,
+                                 sizeof(mismatch_host),
+                                 cudaMemcpyDeviceToHost, stream),
+                 "read q4 grouped attention-A oracle") ||
+        !cuda_ok(cudaStreamSynchronize(stream),
+                 "synchronize q4 grouped attention-A oracle")) {
+        return 0;
+    }
+    g_q4_grouped_attn_a_oracle_calls++;
+    if (mismatch_host != 0u) {
+        g_q4_grouped_attn_a_oracle_mismatches++;
+        if (!g_q4_grouped_attn_a_oracle_mismatch_reported) {
+            g_q4_grouped_attn_a_oracle_mismatch_reported = 1;
+            fprintf(stderr,
+                    "ds4: CUDA Q4 grouped attention-A oracle found a "
+                    "bitwise mismatch; retaining per-group MMVQ output\n");
+        }
+        if (!cuda_ok(cudaMemcpyAsync(low->ptr, reference, reference_bytes,
+                                     cudaMemcpyDeviceToDevice, stream),
+                     "restore q4 grouped attention-A oracle reference")) {
+            return 0;
+        }
+    }
+    return 1;
 }
 
 extern "C" int ds4_gpu_hc_expand_split_half_tensor(
@@ -33227,3 +44674,39 @@ extern "C" int ds4_gpu_tp_batch_gate_encode(uint32_t layer, uint32_t rows) {
 #define DS4_GLM53_VISION_STREAM cuda_decode_stream()
 #include "ds4_glm53_vision_gpu.cuh"
 #include "ds4_deepseek4_vision_gpu.cuh"
+
+/* The TP flag-fold and deferred kv-norm paths are Metal-only optimizations;
+ * non-Apple backends always take the plain fallback (add without the checked
+ * flag, kv norm always standalone). */
+extern "C" int ds4_gpu_add_tensor_tp_flag(
+        ds4_gpu_tensor       *out,
+        const ds4_gpu_tensor *a,
+        const ds4_gpu_tensor *b,
+        uint32_t              n,
+        uint32_t              layer,
+        uint32_t              gate) {
+    (void)layer; (void)gate;
+    return ds4_gpu_add_tensor(out, a, b, n);
+}
+
+extern "C" void ds4_gpu_tp_flag_fold_request(uint32_t layer, uint32_t gate) {
+    (void)layer; (void)gate;
+}
+
+extern "C" void ds4_gpu_dsv4_qkv_norm_defer_kv_next(void) {
+}
+
+extern "C" int ds4_gpu_kv_norm_task_pending(void) {
+    return 0;
+}
+
+extern "C" int ds4_gpu_kv_norm_task_flush(void) {
+    return 0;
+}
+
+extern "C" int ds4_gpu_kv_norm_task_begin_concurrent(void) {
+    return 0;
+}
+
+extern "C" void ds4_gpu_kv_norm_task_end_concurrent(void) {
+}
diff --git a/ds4_distributed.c b/ds4_distributed.c
index 3d797b5dd1..388748611d 100644
--- a/ds4_distributed.c
+++ b/ds4_distributed.c
@@ -65,7 +65,6 @@
 #define DS4_DIST_ACTIVATION_BITS_DEFAULT 32u
 #define DS4_DIST_ROUTE_F_OUTPUT_LOGITS 0x00000001u
 #define DS4_DIST_ROUTE_RETURN_UPSTREAM 1u
-#define DS4_DIST_RECV_TRANSPORT_ERROR 1
 #define DS4_DIST_RECV_REMOTE_ERROR 2
 #define DS4_DIST_SNAPSHOT_CHUNK_BYTES (8u * 1024u * 1024u)
 
@@ -8140,30 +8139,6 @@ void ds4_dist_options_free(ds4_dist_options *opt) {
     free(opt);
 }
 
-void ds4_dist_usage(FILE *fp) {
-    fprintf(fp,
-        "  --role ROLE\n"
-        "      Distributed role: coordinator or worker.\n"
-        "  --layers A:B\n"
-        "      Inclusive distributed layer slice, e.g. 10:20 or 21:output.\n"
-        "  --listen HOST PORT\n"
-        "      Coordinator TCP listen address. Workers may later use it to force their data listener.\n"
-        "  --coordinator HOST PORT\n"
-        "      Coordinator TCP address for --role worker.\n"
-        "  --dist-prefill-chunk N\n"
-        "      Coordinator prefill pipeline chunk size. Default: session cap, normally 4096.\n"
-        "      Non-default values are experimental and can change logits unless validated.\n"
-        "  --dist-prefill-window N\n"
-        "      Coordinator max end-to-end prefill chunks in flight. Default: workers+2, capped at 8.\n"
-        "  --dist-activation-bits N\n"
-        "      Coordinator hidden-state transport width: 32, 16, or 8. Default: 32.\n"
-        "  --dist-replay-check\n"
-        "      Coordinator diagnostic: reset and replay the prompt, then compare logits.\n"
-        "  --debug\n"
-        "      Print coordinator route/debug logs. Workers keep their normal logs without this.\n"
-    );
-}
-
 ds4_dist_cli_parse_result ds4_dist_parse_cli_arg(
         const char *arg,
         int *index,
diff --git a/ds4_distributed.h b/ds4_distributed.h
index 35c5343bd6..6493799f08 100644
--- a/ds4_distributed.h
+++ b/ds4_distributed.h
@@ -43,7 +43,6 @@ typedef enum {
 bool ds4_dist_enabled(const ds4_dist_options *opt);
 ds4_dist_options *ds4_dist_options_create(void);
 void ds4_dist_options_free(ds4_dist_options *opt);
-void ds4_dist_usage(FILE *fp);
 ds4_dist_cli_parse_result ds4_dist_parse_cli_arg(
         const char *arg,
         int *index,
diff --git a/ds4_gpu.h b/ds4_gpu.h
index 5b866dde2b..895912a72b 100644
--- a/ds4_gpu.h
+++ b/ds4_gpu.h
@@ -50,6 +50,12 @@ ds4_gpu_tensor *ds4_gpu_tensor_view(const ds4_gpu_tensor *base, uint64_t offset,
 void ds4_gpu_tensor_free(ds4_gpu_tensor *tensor);
 uint64_t ds4_gpu_tensor_bytes(const ds4_gpu_tensor *tensor);
 void *ds4_gpu_tensor_contents(ds4_gpu_tensor *tensor);
+#if !defined(__APPLE__) && !defined(DS4_ROCM_BUILD)
+/* Stable CUDA allocation identity, including a view's byte offset.  Unlike
+ * the wrapper handle, this stays unchanged when an equivalent tensor view is
+ * recreated and is therefore suitable for CUDA graph-cache keys. */
+uintptr_t ds4_gpu_tensor_storage_key(const ds4_gpu_tensor *tensor);
+#endif
 int ds4_gpu_tensor_fill_f32(ds4_gpu_tensor *tensor, float value, uint64_t count);
 int ds4_gpu_tensor_write(ds4_gpu_tensor *tensor, uint64_t offset, const void *data, uint64_t bytes);
 int ds4_gpu_tensor_read(const ds4_gpu_tensor *tensor, uint64_t offset, void *data, uint64_t bytes);
@@ -77,6 +83,12 @@ int ds4_gpu_pack_slot_rows_f32_tensor(
 int ds4_gpu_begin_commands(void);
 int ds4_gpu_flush_encoder(void);
 int ds4_gpu_flush_commands(void);
+#ifdef __APPLE__
+/* Commit the current Metal batch without draining.  The completion hook runs
+ * on a Metal-owned thread and must only publish thread-safe readiness state;
+ * the next full command drain joins it before returning. */
+int ds4_gpu_flush_commands_progress(void (*report)(void *ctx), void *ctx);
+#endif
 int ds4_gpu_commands_active(void);
 #ifdef __APPLE__
 int ds4_gpu_parallel_ffn_finish(void);
@@ -136,6 +148,7 @@ int ds4_gpu_parallel_ffn_start_split(
         uint32_t              n_expert,
         uint32_t              n_expert_used,
         uint32_t              shift_q16);
+#endif
 
 /* out = a + b into this rank's TP slab slot for (layer, gate), publishing the
  * gate's checked flag from the same kernel; falls back to ds4_gpu_add_tensor
@@ -161,11 +174,11 @@ int ds4_gpu_kv_norm_task_pending(void);
 int ds4_gpu_kv_norm_task_flush(void);
 int ds4_gpu_kv_norm_task_begin_concurrent(void);
 void ds4_gpu_kv_norm_task_end_concurrent(void);
-#endif
 int ds4_gpu_signal_selected_readback_ready(uint64_t *event_value);
 int ds4_gpu_commit_and_wait_selected_readback(uint64_t event_value, const char *label);
 int ds4_gpu_wait_selected_readback_ready(uint64_t event_value, const char *label);
-#ifdef DS4_ROCM_BUILD
+#if defined(DS4_ROCM_BUILD) || \
+    (!defined(__APPLE__) && !defined(DS4_NO_GPU))
 int ds4_gpu_tensor_read_after_selected_event(const ds4_gpu_tensor *tensor,
                                              uint64_t offset,
                                              void *data,
@@ -174,11 +187,45 @@ int ds4_gpu_tensor_read_after_selected_event(const ds4_gpu_tensor *tensor,
                                              const char *label);
 #endif
 int ds4_gpu_end_commands(void);
+#ifdef __APPLE__
+/* Metal-only asynchronous command streams. Command encoding remains
+ * serialized; at most eight committed streams may execute concurrently. */
+void ds4_gpu_set_stream(int idx);
+int ds4_gpu_current_stream(void);
+int ds4_gpu_end_commands_async(void);
+int ds4_gpu_wait_stream(int idx);
+#endif
 int ds4_gpu_synchronize(void);
 
 int ds4_gpu_set_model_map(const void *model_map, uint64_t model_size);
 int ds4_gpu_set_model_fd(int fd);
 int ds4_gpu_set_model_fd_for_map(int fd, const void *model_map);
+#if defined(DS4_BENCH_CUDA) || \
+    (!defined(__APPLE__) && !defined(DS4_ROCM_BUILD) && \
+     !defined(DS4_NO_GPU))
+/* CUDA benchmark/test controls.  These are deliberately explicit API hooks
+ * rather than environment knobs: production callers must not depend on
+ * strict dispatch or backend-internal model provenance. */
+int ds4_cuda_test_model_range_is_device_resident(
+        const void *model_map,
+        uint64_t model_size,
+        uint64_t offset,
+        uint64_t bytes,
+        int logical_tier);
+int ds4_cuda_test_model_range_device_ptr(
+        const void *model_map,
+        uint64_t model_size,
+        uint64_t offset,
+        uint64_t bytes,
+        int logical_tier,
+        const void **device_ptr);
+void ds4_cuda_test_set_q4_mmq_strict(int required);
+#endif
+/* Prepare a second, fully resident support GGUF without replacing the active
+ * target-model mapping used by SSD streaming. */
+int ds4_gpu_prepare_support_model(const void *model_map, uint64_t model_size,
+                                  uint64_t map_offset, uint64_t map_size,
+                                  uint64_t max_tensor_bytes);
 int ds4_gpu_build_derived_artifacts(const void *model_map, uint64_t model_size,
                                     const char *model_path);
 int ds4_gpu_model_range_replaced(const void *model_map, uint64_t offset,
@@ -233,6 +280,36 @@ void ds4_gpu_set_quality(bool quality);
 void ds4_gpu_set_glm_model(bool enabled);
 void ds4_gpu_set_ssd_streaming(bool enabled);
 void ds4_gpu_set_glm_streaming_prefill_full_layer(bool enabled);
+
+typedef struct ds4_gpu_q4_attn_q_b_f16_sidecar_desc {
+    uint64_t weight_offset;
+    uint64_t weight_bytes;
+    uint64_t in_dim;
+    uint64_t out_dim;
+    uint32_t weight_type;
+    uint32_t layer;
+} ds4_gpu_q4_attn_q_b_f16_sidecar_desc;
+
+/* Backend-neutral prefill preflight for optional Q4_K attn_q_b F16
+ * acceleration.  A backend may prepare resident sidecars or reusable
+ * transient scratch/pipelines according to its policy.  Prepare returns 1
+ * when the selected path is ready, 0 for a policy/safety skip, and -1 when
+ * strict mode requires an unavailable specialization. */
+int ds4_gpu_prepare_q4_attn_q_b_f16_sidecars(
+        const void *model_map,
+        uint64_t model_size,
+        const ds4_gpu_q4_attn_q_b_f16_sidecar_desc *descs,
+        uint32_t count,
+        uint32_t max_prefill_rows,
+        uint64_t working_set_reserve_bytes,
+        uint64_t *prepared_bytes);
+/* Release sidecars at a quiescent backend lifecycle point.  Returns zero
+ * only when pending GPU work could not be synchronized safely. */
+int ds4_gpu_release_q4_attn_q_b_f16_sidecars(void);
+uint64_t ds4_gpu_q4_attn_q_b_f16_cache_generation(void);
+/* Evict resident sidecars before adding a graph to a live-session set. */
+int ds4_gpu_make_room_for_q4_attn_q_b_f16_session(void);
+
 #ifdef __APPLE__
 int ds4_gpu_device_is_pre_m5_apple_silicon(void);
 int ds4_gpu_device_is_m5_apple_silicon(void);
@@ -244,6 +321,139 @@ int ds4_gpu_test_decode_pipeline_fast_lookup_ext(void);
 /* Strict test oracle for the generated resident-prefill MXFP4 half LUT. */
 int ds4_gpu_test_mxfp4_down_half_lut(uint16_t *legacy_bits,
                                      uint16_t *lut_bits);
+typedef struct ds4_gpu_iq2_mid_only_oracle_report {
+    uint64_t mid_words;
+    uint64_t mid_mismatches;
+    uint64_t canonical_gate_unwritten;
+    uint64_t canonical_up_unwritten;
+    uint64_t candidate_gate_writes;
+    uint64_t candidate_up_writes;
+    uint64_t masked_mid_mismatches;
+    uint64_t masked_inactive_writes;
+    uint64_t masked_canonical_gate_unwritten;
+    uint64_t masked_canonical_up_unwritten;
+    uint64_t masked_gate_writes;
+    uint64_t masked_up_writes;
+    uint64_t guard_byte_mismatches;
+} ds4_gpu_iq2_mid_only_oracle_report;
+/* Full-shape address-table oracle for the experimental M1 IQ2 mid-only
+ * producer.  Return zero means setup/execution failure; numerical and
+ * sentinel failures are reported explicitly in `report`. */
+int ds4_gpu_test_iq2_addr_mid_only_oracle(
+        ds4_gpu_iq2_mid_only_oracle_report *report);
+typedef struct ds4_gpu_stream_expert_live_index_report {
+    uint64_t scans;
+    uint64_t entries;
+    uint64_t fallbacks;
+    uint64_t inserts;
+    uint64_t removes;
+    uint64_t reuse_scan_calls;
+    uint64_t reuse_scan_entries;
+    uint64_t resident_hash;
+    uint32_t live_count;
+    uint32_t cache_entries;
+    uint32_t eligible;
+    uint32_t active;
+    uint32_t broken;
+} ds4_gpu_stream_expert_live_index_report;
+/* Test-only policy/state hooks for the IQ2 production-size SSD cache index. */
+int ds4_gpu_test_stream_expert_live_index_policy(
+        int ssd_streaming,
+        uint64_t gate_expert_bytes,
+        uint64_t down_expert_bytes,
+        int enable,
+        int disable);
+void ds4_gpu_test_stream_expert_live_index_report(
+        ds4_gpu_stream_expert_live_index_report *report);
+typedef struct ds4_gpu_exact_rows_persistent_report {
+    uint64_t persistent_calls;
+    uint64_t transient_calls;
+    uint64_t persistent_fallbacks;
+    uint64_t persistent_failures;
+    uint64_t mapped_view_calls;
+    uint32_t max_unique;
+} ds4_gpu_exact_rows_persistent_report;
+/* Test-only policy and counters for exact-row private cache snapshots. */
+int ds4_gpu_test_exact_rows_persistent_policy(
+        uint32_t configured_count,
+        uint32_t unique_count,
+        int      size_class_ok);
+void ds4_gpu_test_exact_rows_persistent_report(
+        ds4_gpu_exact_rows_persistent_report *report);
+typedef struct ds4_gpu_q4_attn_q_b_f16_cache_report {
+    uint64_t entries;
+    uint64_t bytes;
+    uint64_t lookups;
+    uint64_t hits;
+    uint64_t misses;
+    uint64_t builds;
+    uint64_t build_failures;
+    uint64_t candidate_calls;
+    uint64_t fallbacks;
+    uint64_t rejects;
+    uint64_t build_circuit_open;
+    uint64_t transient_exact_views_created;
+    uint64_t transient_exact_views_live;
+    uint64_t model_exact_cache_entries;
+    uint64_t model_exact_cache_bytes;
+} ds4_gpu_q4_attn_q_b_f16_cache_report;
+/* Test observability for the resident Metal Q4_K attn_q_b F16 sidecar. */
+void ds4_gpu_test_q4_attn_q_b_f16_cache_report(
+        ds4_gpu_q4_attn_q_b_f16_cache_report *report);
+void ds4_gpu_test_q4_attn_q_b_f16_cache_reset(void);
+int ds4_gpu_test_q4_attn_q_b_f16_projection_tensor(
+        ds4_gpu_tensor       *out,
+        const void           *model_map,
+        uint64_t              model_size,
+        uint64_t              weight_offset,
+        uint64_t              in_dim,
+        uint64_t              out_dim,
+        const ds4_gpu_tensor *x,
+        uint32_t              n_tok);
+typedef enum ds4_gpu_test_q4_qb_mm_arm {
+    DS4_GPU_TEST_Q4_QB_MM_Q4_F32 = 0,
+    DS4_GPU_TEST_Q4_QB_MM_Q4_F16 = 1,
+    DS4_GPU_TEST_Q4_QB_MM_F16_F32 = 2,
+    DS4_GPU_TEST_Q4_QB_MM_F16_F16 = 3,
+    DS4_GPU_TEST_Q4_QB_MM_Q4_TRANSIENT_F16_F16 = 4,
+    DS4_GPU_TEST_Q4_QB_MM_ARM_COUNT = 5,
+} ds4_gpu_test_q4_qb_mm_arm;
+/* Runtime capability probe for test-only matmul arms. */
+int ds4_gpu_test_q4_attn_q_b_mm_arm_supported(
+        ds4_gpu_test_q4_qb_mm_arm arm);
+/* Strict projection-only resident benchmark hook.  F16-weight arms require
+ * a READY sidecar; F16-RHS arms optionally include the production copy.  The
+ * transient arm rebuilds its F16 weight matrix for every benchmark/oracle
+ * projection and may consume either a prepacked or freshly copied F16 RHS. */
+int ds4_gpu_test_q4_attn_q_b_mm_variant_tensor(
+        ds4_gpu_tensor              *out_f32,
+        ds4_gpu_tensor              *rhs_f16,
+        const void                  *model_map,
+        uint64_t                     model_size,
+        uint64_t                     weight_offset,
+        uint64_t                     in_dim,
+        uint64_t                     out_dim,
+        const ds4_gpu_tensor        *x_f32,
+        uint32_t                     n_tok,
+        ds4_gpu_test_q4_qb_mm_arm    arm,
+        bool                         materialize_rhs);
+int ds4_gpu_test_q4_attn_q_b_f16_working_set_policy(
+        uint64_t recommended,
+        uint64_t allocated,
+        uint64_t additional);
+typedef struct ds4_gpu_stream_test_stats {
+    uint64_t tensor_live_bytes;
+    uint64_t transient_references;
+    uint32_t tensor_live_count;
+    uint32_t pending_command_buffers;
+    uint32_t last_command_buffers;
+    uint32_t active_queue_mask;
+    uint32_t model_residency_queue_mask;
+    uint32_t q4_residency_queue_mask;
+} ds4_gpu_stream_test_stats;
+/* Test-only observability and lifetime injection for Metal stream oracles. */
+void ds4_gpu_test_stream_stats(ds4_gpu_stream_test_stats *stats);
+int ds4_gpu_test_hold_stream_transient(uint64_t bytes);
 enum {
     DS4_GPU_TEST_MXFP4_PAIR_TAIL_CULL = 1u << 0,
     DS4_GPU_TEST_MXFP4_PAIR_COMPACT_TILE = 1u << 1,
@@ -252,13 +462,205 @@ enum {
     DS4_GPU_TEST_MXFP4_DOWN_HALF_LUT = 1u << 4,
     DS4_GPU_TEST_OUTPUT_HC_WEIGHTS4 = 1u << 5,
     DS4_GPU_TEST_HC_RMS_SCALE_PROJ = 1u << 6,
+    DS4_GPU_TEST_STREAMING_LIVE_INDEX_FAILURE = 1u << 7,
+    DS4_GPU_TEST_IQ2_SSD_GROUPED_PIPELINE_FAILURE = 1u << 8,
+    DS4_GPU_TEST_ATTN_OUT_LOW_Q8_STATIC = 1u << 9,
+    DS4_GPU_TEST_BATCH_ATTN_OUT_Q8_HC_FUSION = 1u << 10,
+    DS4_GPU_TEST_BATCH_ATTN_OUT_Q4_HC_FUSION = 1u << 11,
+    DS4_GPU_TEST_FLASH_ATTN_SMALL_PREFILL_NWG32 = 1u << 12,
+    DS4_GPU_TEST_FLASH_ATTN_SMALL_PREFILL_NWG1_FAILURE = 1u << 13,
+    DS4_GPU_TEST_REQUIRE_IQ2_TOP8_PAIR_SWIGLU = 1u << 14,
 };
 void ds4_gpu_test_set_flags(uint32_t flags);
+double ds4_gpu_test_last_completed_gpu_ms(void);
+uint32_t ds4_gpu_test_last_flash_attn_prefill_nwg(void);
+int ds4_gpu_test_reset_flash_attn_tmp(void);
+uint64_t ds4_gpu_test_flash_attn_tmp_bytes(void);
 void ds4_gpu_release_zero_prefix_prefill_mask_cache(void);
 #else
 static inline int ds4_gpu_device_is_pre_m5_apple_silicon(void) { return 0; }
 static inline int ds4_gpu_device_is_m5_apple_silicon(void) { return 0; }
 #endif
+#if !defined(__APPLE__) && !defined(DS4_ROCM_BUILD) && !defined(DS4_NO_GPU)
+typedef struct ds4_cuda_stream_selected_batch_io_report {
+    uint64_t candidates;
+    uint64_t attempts;
+    uint64_t completed;
+    uint64_t legacy_batches;
+    uint64_t safe_fallbacks;
+    uint64_t failures;
+    uint64_t required_failures;
+    uint64_t oracle_runs;
+    uint64_t oracle_failures;
+    uint64_t tasks;
+    uint64_t segments;
+    uint64_t reads;
+    uint64_t bytes;
+    int enabled;
+    int required;
+    int oracle;
+} ds4_cuda_stream_selected_batch_io_report;
+typedef struct ds4_cuda_stream_selected_event_pipeline_report {
+    uint64_t candidates;
+    uint64_t signals;
+    uint64_t readbacks;
+    uint64_t uploads;
+    uint64_t compute_waits;
+    uint64_t safe_fallbacks;
+    uint64_t failures;
+    uint64_t required_failures;
+    uint64_t oracle_runs;
+    uint64_t oracle_failures;
+    int enabled;
+    int required;
+    int oracle;
+} ds4_cuda_stream_selected_event_pipeline_report;
+typedef struct ds4_cuda_iq2_ssd_grouped_report {
+    uint64_t candidates;
+    uint64_t eligible;
+    uint64_t attempts;
+    uint64_t completed;
+    uint64_t not_applicable;
+    uint64_t safe_fallbacks;
+    uint64_t failures;
+    uint64_t required_failures;
+    uint64_t upload_waits;
+    uint64_t lease_waits;
+    uint64_t lease_records;
+    uint64_t lease_drains;
+    int enabled;
+    int required;
+    int stats;
+} ds4_cuda_iq2_ssd_grouped_report;
+typedef struct ds4_cuda_stream_expert_persistent_report {
+    uint64_t plan_attempts;
+    uint64_t plans_built;
+    uint64_t commits;
+    uint64_t rollbacks;
+    uint64_t hits;
+    uint64_t misses;
+    uint64_t duplicates;
+    uint64_t free_assignments;
+    uint64_t evictions;
+    uint64_t rejects;
+    uint64_t budget_rejects;
+    uint64_t class_rejects;
+    uint64_t protected_rejects;
+    uint64_t key_misses;
+    uint64_t overflow_rejects;
+    uint64_t oracle_runs;
+    uint64_t oracle_failures;
+    uint64_t arena_allocations;
+    uint64_t arena_reuses;
+    uint64_t arena_releases;
+    uint64_t arena_failures;
+    uint64_t arena_oracle_runs;
+    uint64_t arena_oracle_failures;
+    uint64_t epochs_attempted;
+    uint64_t epochs_published;
+    uint64_t all_hit_epochs;
+    uint64_t miss_epochs;
+    uint64_t miss_experts;
+    uint64_t weight_bytes_uploaded;
+    uint64_t remap_bytes_uploaded;
+    uint64_t upload_failures;
+    uint64_t fallbacks;
+    uint64_t slot_invalidations;
+    uint64_t poisons;
+    uint64_t persistent_dispatches;
+    uint64_t transient_dispatches;
+    uint64_t runtime_oracle_runs;
+    uint64_t runtime_oracle_failures;
+    int enabled;
+    int required;
+    int stats;
+    int oracle;
+} ds4_cuda_stream_expert_persistent_report;
+typedef struct ds4_cuda_q8_hc_expand_report {
+    uint64_t candidates;
+    uint64_t fused_attempts;
+    uint64_t fused_completed;
+    uint64_t split_attempts;
+    uint64_t split_completed;
+    uint64_t failures;
+    uint64_t capture_candidates;
+    uint64_t owned_forced_fused;
+    uint64_t multi_gpu_forced_fused;
+    uint64_t oracle_runs;
+    uint64_t oracle_failures;
+    int force_fused;
+    int split_requested;
+    int stats;
+} ds4_cuda_q8_hc_expand_report;
+/* CUDA-only policy/planner/scatter hooks.  The policy hook takes explicit
+ * values so tests do not need to mutate process environment around once_flag
+ * initialization. */
+int ds4_cuda_test_q8_hc_expand_policy(
+        int force_fused, int disable_fused, int n_gpus, int owned,
+        int capture, int *fused_out, int *owned_forced_out,
+        int *multi_gpu_forced_out, int *capture_out);
+int ds4_cuda_test_q8_hc_expand_env_value(const char *value);
+int ds4_cuda_test_q8_hc_expand_oracle(void);
+void ds4_cuda_q8_hc_expand_get_report(
+        ds4_cuda_q8_hc_expand_report *report);
+int ds4_cuda_test_stream_selected_batch_policy(
+        int enable, int disable, int require, int oracle,
+        int *enabled_out, int *required_out, int *oracle_out);
+int ds4_cuda_test_stream_selected_batch_env_value(const char *value);
+int ds4_cuda_test_stream_selected_batch_plan(void);
+int ds4_cuda_test_stream_selected_batch_copy(void);
+void ds4_cuda_stream_selected_batch_io_get_report(
+        ds4_cuda_stream_selected_batch_io_report *report);
+int ds4_cuda_test_stream_selected_event_pipeline_policy(
+        int enable, int disable, int require, int oracle,
+        int *enabled_out, int *required_out, int *oracle_out);
+int ds4_cuda_test_stream_selected_event_env_value(const char *value);
+int ds4_cuda_test_stream_selected_event_pipeline(void);
+int ds4_cuda_test_stream_selected_owner_device(void);
+void ds4_cuda_stream_selected_event_pipeline_get_report(
+        ds4_cuda_stream_selected_event_pipeline_report *report);
+int ds4_cuda_test_iq2_ssd_grouped_policy(
+        int enable, int disable, int require, int stats,
+        int *enabled_out, int *required_out, int *stats_out);
+int ds4_cuda_test_iq2_ssd_grouped_eligibility(
+        int enabled, int ssd_streaming, int single_gpu, int gb10,
+        int quality, int owned_filtered, int capture, int mmq,
+        uint32_t n_tokens, uint32_t n_expert, int top6_unique,
+        int raw_layout, int binding_valid);
+int ds4_cuda_test_iq2_ssd_grouped_candidate(
+        int iq2_path, int ssd_streaming, int allow_streaming,
+        int owned_filtered, uint32_t n_tokens, uint32_t n_expert,
+        int top6_unique, int raw_layout, int binding_valid);
+int ds4_cuda_test_iq2_ssd_grouped_raw_layout(
+        uint32_t gate_type, uint32_t down_type,
+        uint64_t gate_expert_bytes, uint64_t gate_row_bytes,
+        uint64_t down_expert_bytes, uint64_t down_row_bytes,
+        uint32_t expert_in_dim, uint32_t expert_mid_dim,
+        uint32_t out_dim);
+int ds4_cuda_test_iq2_ssd_grouped_lease(void);
+void ds4_cuda_iq2_ssd_grouped_get_report(
+        ds4_cuda_iq2_ssd_grouped_report *report);
+int ds4_cuda_test_stream_expert_persistent_policy(
+        int enable, int disable, int require, int stats, int oracle,
+        int *enabled_out, int *required_out, int *stats_out,
+        int *oracle_out);
+int ds4_cuda_test_stream_expert_persistent_env_value(const char *value);
+int ds4_cuda_test_stream_expert_persistent_planner(void);
+int ds4_cuda_test_stream_expert_persistent_arena(void);
+int ds4_cuda_test_stream_expert_persistent_runtime(void);
+void ds4_cuda_stream_expert_persistent_get_report(
+        ds4_cuda_stream_expert_persistent_report *report);
+int ds4_gpu_cuda_stream_selected_event_pipeline_enabled(void);
+int ds4_gpu_cuda_stream_selected_event_pipeline_required(void);
+int ds4_gpu_cuda_stream_selected_set_owner_device(void);
+void ds4_gpu_cuda_stream_selected_event_note_candidate(void);
+void ds4_gpu_cuda_stream_selected_event_note_fallback(void);
+void ds4_gpu_cuda_stream_selected_event_note_failure(int required);
+int ds4_gpu_signal_selected_readback_ready_async(uint64_t *event_value);
+int ds4_gpu_stream_expert_cache_wait_selected_upload(
+        uint64_t event_value, const char *label);
+int ds4_gpu_cuda_stream_selected_event_abort(void);
+#endif
 void ds4_gpu_set_streaming_expert_cache_budget(uint32_t experts);
 void ds4_gpu_set_streaming_expert_cache_expert_bytes(uint64_t bytes);
 uint64_t ds4_gpu_recommended_working_set_size(void);
@@ -290,6 +692,15 @@ int ds4_gpu_stream_expert_cache_begin_selected_load(
         const ds4_gpu_stream_expert_table *table,
         const int32_t                     *selected_ids,
         uint32_t                           n_selected);
+#if !defined(__APPLE__) && !defined(DS4_ROCM_BUILD) && !defined(DS4_NO_GPU)
+/* Returns 1 on publication, 0 on a safe pre-enqueue rejection, and -1 on
+ * a post-enqueue failure which callers must not retry. */
+int ds4_gpu_stream_expert_cache_begin_selected_load_async(
+        const ds4_gpu_stream_expert_table *table,
+        const int32_t                     *selected_ids,
+        uint32_t                           n_selected,
+        uint64_t                          *upload_event_value);
+#endif
 int ds4_gpu_glm_stream_expert_cache_begin_selected_load_tensor(
         const ds4_gpu_stream_expert_table *table,
         const ds4_gpu_tensor              *selected,
@@ -331,6 +742,29 @@ int ds4_gpu_stream_expert_cache_seed_experts_gpu_copy(
         const int32_t                     *expert_ids,
         const uint32_t                    *expert_priorities,
         uint32_t                           n_experts);
+/* Exact speculative decode may compute up to five independent router rows
+ * before executing their routed MoE tails.  begin_collect() first isolates
+ * this layer from ordinary decode's pending selected-expert load and global
+ * selected-id override.  prepare() then builds one immutable SSD address table
+ * for the union of those rows, and set_row() arms exactly one routed-MoE call
+ * at a time in increasing row order.  prepare() is a Metal command-stream
+ * boundary: it waits for the router rows to become CPU-visible and reopens the
+ * command batch.  release() is required on every success/error exit after
+ * begin_collect(), and before collecting the next layer. */
+int ds4_gpu_stream_expert_exact_rows_begin_collect(void);
+int ds4_gpu_stream_expert_exact_rows_prepare(
+        const ds4_gpu_stream_expert_table *table,
+        const ds4_gpu_tensor              *selected_rows,
+        uint32_t                           n_rows,
+        uint32_t                           n_selected);
+int ds4_gpu_stream_expert_exact_rows_set_row(uint32_t row);
+/* Commit the completed routed-tail command buffer without waiting.  The
+ * backend retains every private address/overflow/cache buffer owned by the
+ * exact-row scope until that command buffer completes, so release() may be
+ * called immediately after a successful return.  This is an experimental
+ * boundary used only when the caller explicitly enables asynchronous tails. */
+int ds4_gpu_stream_expert_exact_rows_end_async(void);
+void ds4_gpu_stream_expert_exact_rows_release(void);
 #endif
 void ds4_gpu_print_memory_report(const char *label);
 
@@ -408,9 +842,9 @@ int ds4_gpu_tp_failed(void);
  *
  * ds4_gpu_matmul_q8_0_kslice_tensor computes a k-range partial matvec:
  * out[out_dim] = W[:, k_off : k_off + k_cnt] @ x[x_elem_off : +k_cnt] where
- * W rows span full_in_dim quantized Q8_0 elements.  k offsets/counts must be
- * multiples of 32 (Q8_0 block).  Partial results from both ranks sum to the
- * full projection.
+ * W rows span full_in_dim quantized elements. Q8_0 slices use multiples of 32;
+ * the generic dispatch also accepts Q4_K slices in multiples of 256. Partial
+ * results from both ranks sum to the full projection.
  *
  * ds4_gpu_attention_output_q8_tp_tensor is the group-sliced attention output
  * pair: low projection for groups [group0, group0+group_cnt) plus the
@@ -452,21 +886,6 @@ int ds4_gpu_matmul_quant_kslice_tensor(
         uint64_t                out_dim,
         const ds4_gpu_tensor *x,
         uint64_t                x_elem_off);
-int ds4_gpu_attention_output_q8_tp_tensor(
-        ds4_gpu_tensor       *out,
-        ds4_gpu_tensor       *low,
-        const void             *model_map,
-        uint64_t                model_size,
-        uint64_t                out_a_offset,
-        uint64_t                out_b_offset,
-        uint64_t                group_dim,
-        uint64_t                rank,
-        uint32_t                n_groups_total,
-        uint32_t                group0,
-        uint32_t                group_cnt,
-        uint64_t                out_dim,
-        const ds4_gpu_tensor *heads);
-
 /* =========================================================================
  * Embeddings and Indexer Helpers.
  * =========================================================================
@@ -580,6 +999,51 @@ int ds4_gpu_dspark_markov_argmax_tensor(ds4_gpu_tensor *out_idx,
                                         uint32_t prev_token,
                                         uint32_t vocab,
                                         uint32_t rank);
+
+/* Optional GPU-resident DSpark proposer tail.  The backend keeps the Markov
+ * token chain on-device across all draft rows, stops it at the first rejected
+ * confidence row, and returns proposals plus the evaluated confidence logits
+ * in one result/readback.  Callers must re-evaluate the returned prefix with
+ * the established CPU sigmoid policy and fall back if the device stopped a
+ * row that policy accepts.
+ *
+ * This acceleration hook is deliberately fail-closed: it returns zero for
+ * disabled or unsupported inputs and callers must use the established
+ * per-row path.  CUDA requires DS4_CUDA_DSPARK_DEVICE_PROPOSER=1; Metal uses
+ * DS4_METAL_DSPARK_DEVICE_PROPOSER=1.  The corresponding NO_DEVICE_PROPOSER
+ * variables are unconditional kill switches.  The first implementation
+ * supports Q8_0 w1, w2, and confidence weights with 32-aligned hidden/rank
+ * dimensions.  The caller-owned result tensor must provide
+ * DS4_GPU_DSPARK_DEVICE_PROPOSAL_BYTES; the first sizeof(result) bytes are the
+ * public payload and the remainder is private per-call backend state. */
+#define DS4_GPU_DSPARK_MAX_DRAFTS 6u
+#define DS4_GPU_DSPARK_DEVICE_PROPOSAL_BYTES 2048u
+typedef struct {
+    int32_t  tokens[DS4_GPU_DSPARK_MAX_DRAFTS];
+    float    confidence_logits[DS4_GPU_DSPARK_MAX_DRAFTS];
+    uint32_t proposal_len;
+    uint32_t confidence_len;
+    uint32_t status;       /* 1: complete; 0: device-side failure */
+    uint32_t reserved;
+} ds4_gpu_dspark_device_proposal;
+
+int ds4_gpu_dspark_markov_confidence_q8_tensor(
+        ds4_gpu_tensor       *out_result,
+        const ds4_gpu_tensor *logits_rows,
+        const ds4_gpu_tensor *hidden_rows,
+        const void           *model_map,
+        uint64_t              model_size,
+        uint64_t              w1_offset,
+        uint64_t              w2_offset,
+        uint64_t              confidence_offset,
+        uint32_t              first_prev_token,
+        uint32_t              vocab,
+        uint32_t              rank,
+        uint32_t              hidden_dim,
+        uint32_t              n_drafts,
+        float                 confidence_threshold,
+        int                   reuse_confidence0,
+        float                 confidence0);
 int ds4_gpu_indexer_topk_tensor(
         ds4_gpu_tensor       *selected,
         const ds4_gpu_tensor *scores,
@@ -736,6 +1200,57 @@ int ds4_gpu_matmul_q4_K_pair_decode_tensor(
         uint64_t              out_dim,
         const ds4_gpu_tensor *x);
 
+/* Optional dense Q4_K pair for decode, microbatch, and prefill. Backends
+ * return 1 when the pair was encoded, 0 to request separate fallback
+ * matmuls from the graph, and -1 after a required/attempted-path error. */
+int ds4_gpu_matmul_q4_K_pair_tensor(
+        ds4_gpu_tensor       *out0,
+        ds4_gpu_tensor       *out1,
+        const void           *model_map,
+        uint64_t              model_size,
+        uint64_t              weight0_offset,
+        uint64_t              weight1_offset,
+        uint64_t              in_dim,
+        uint64_t              out0_dim,
+        uint64_t              out1_dim,
+        const ds4_gpu_tensor *x,
+        uint64_t              n_tok);
+
+/* Metal decode compound for AProjQ4: Q-A/KV Q4_K pair plus the attention
+ * and indexer F16 compressor pairs/state stores. Returns 1 when encoded,
+ * 0 to use the separate fallback, and -1 on an attempted-path error. */
+int ds4_gpu_q4_K_pair_quad_compressor_store_tensor(
+        ds4_gpu_tensor       *qr,
+        ds4_gpu_tensor       *kv_raw,
+        ds4_gpu_tensor       *out0_kv,
+        ds4_gpu_tensor       *out0_score,
+        ds4_gpu_tensor       *out1_kv,
+        ds4_gpu_tensor       *out1_score,
+        ds4_gpu_tensor       *state0_kv,
+        ds4_gpu_tensor       *state0_score,
+        ds4_gpu_tensor       *state1_kv,
+        ds4_gpu_tensor       *state1_score,
+        const void           *model_map,
+        uint64_t              model_size,
+        uint64_t              q_a_offset,
+        uint64_t              kv_offset,
+        uint64_t              weight0_kv_offset,
+        uint64_t              weight0_score_offset,
+        uint64_t              weight1_kv_offset,
+        uint64_t              weight1_score_offset,
+        uint64_t              ape0_offset,
+        uint32_t              ape0_type,
+        uint64_t              ape1_offset,
+        uint32_t              ape1_type,
+        uint32_t              in_dim,
+        uint32_t              q_rank,
+        uint32_t              kv_dim,
+        uint32_t              width0,
+        uint32_t              width1,
+        const ds4_gpu_tensor *x,
+        uint32_t              ratio,
+        uint32_t              pos);
+
 /* Multi-row decode projections that preserve the one-row reduction order. */
 int ds4_gpu_matmul_q8_0_decode_rows_exact_tensor(
         ds4_gpu_tensor       *out,
@@ -943,6 +1458,11 @@ int ds4_gpu_matmul_f16_pair_compressor_store_tensor(
         uint32_t                ratio,
         uint32_t                pos);
 
+/* Optional Metal ratio-4 decode fusion.  Returns 1 when both paired
+ * compressor projections and their recurrent-state stores were encoded in a
+ * single dispatch, 0 when the caller should keep the established separate
+ * paths, and -1 on an attempted-path error. */
+int ds4_gpu_f16_quad_compressor_store_auto_available(void);
 int ds4_gpu_matmul_f16_quad_compressor_store_tensor(
         ds4_gpu_tensor       *out0_kv,
         ds4_gpu_tensor       *out0_score,
@@ -1187,12 +1707,20 @@ int ds4_gpu_head_rms_norm_rope_tail_tensor(
         float             beta_slow,
         float             eps);
 
+/* Returns 1 when the backend-specific fused projection and its consumers were
+ * accepted/encoded successfully, 0 only while the caller may safely use its
+ * generic projection path, and -1 when a required specialization could not be
+ * honored or work failed after an output writer was encoded (so replay is
+ * unsafe). Completion is established by the enclosing backend synchronization.
+ * q_half is optional: backends whose graph owns F16 projection staging may use
+ * it, while CUDA/ROCm can emit F32 directly into out. */
 int ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor(
         ds4_gpu_tensor       *out,
         ds4_gpu_tensor       *q_half,
         const void           *model_map,
         uint64_t              model_size,
         uint64_t              weight_offset,
+        uint32_t              weight_type,
         uint64_t              in_dim,
         uint64_t              out_dim,
         const ds4_gpu_tensor *x,
@@ -2318,6 +2846,33 @@ int ds4_gpu_attention_output_q8_batch_tensor(
         uint64_t                out_dim,
         const ds4_gpu_tensor *heads,
         uint32_t                n_tokens);
+#ifdef __APPLE__
+/* Optional resident Metal output-B + HC4 epilogues.  Return 1 when fused work
+ * was encoded, 0 before writing anything when ineligible, and -1 after an
+ * attempted-path failure (the caller must not replay the fallback then). */
+int ds4_gpu_attention_output_q8_batch_hc_tensor(
+        ds4_gpu_tensor       *out,
+        ds4_gpu_tensor       *out_hc,
+        const ds4_gpu_tensor *residual_hc,
+        const ds4_gpu_tensor *split,
+        ds4_gpu_tensor       *low,
+        ds4_gpu_tensor       *group_tmp,
+        ds4_gpu_tensor       *low_tmp,
+        const void           *model_map,
+        uint64_t              model_size,
+        uint64_t              out_a_offset,
+        uint64_t              out_b_offset,
+        uint64_t              group_dim,
+        uint64_t              rank,
+        uint32_t              n_groups,
+        uint64_t              out_dim,
+        const ds4_gpu_tensor *heads,
+        uint32_t              n_tokens,
+        uint32_t              n_hc);
+#endif
+/* Returns 1 when the batch path ran, 0 for the ordinary row fallback, and -1
+ * for a post-enqueue failure or a backend REQUIRE diagnostic.  The caller
+ * must not retry the row fallback after -1. */
 int ds4_gpu_attention_output_q4_K_batch_tensor(
         ds4_gpu_tensor       *out,
         ds4_gpu_tensor       *low,
@@ -2334,6 +2889,28 @@ int ds4_gpu_attention_output_q4_K_batch_tensor(
         uint64_t                out_dim,
         const ds4_gpu_tensor *heads,
         uint32_t                n_tokens);
+#ifdef __APPLE__
+int ds4_gpu_attention_output_q4_K_batch_hc_tensor(
+        ds4_gpu_tensor       *out,
+        ds4_gpu_tensor       *out_hc,
+        const ds4_gpu_tensor *residual_hc,
+        const ds4_gpu_tensor *split,
+        ds4_gpu_tensor       *low,
+        ds4_gpu_tensor       *group_tmp,
+        ds4_gpu_tensor       *low_tmp,
+        const void           *model_map,
+        uint64_t              model_size,
+        uint64_t              out_a_offset,
+        uint64_t              out_b_offset,
+        uint32_t              out_b_type,
+        uint64_t              group_dim,
+        uint64_t              rank,
+        uint32_t              n_groups,
+        uint64_t              out_dim,
+        const ds4_gpu_tensor *heads,
+        uint32_t              n_tokens,
+        uint32_t              n_hc);
+#endif
 
 int ds4_gpu_attention_output_q8_batch_f16_tensor(
         ds4_gpu_tensor       *out_h,
@@ -2358,6 +2935,8 @@ int ds4_gpu_attention_output_low_q8_tensor(
         uint64_t                rank,
         uint32_t                n_groups,
         const ds4_gpu_tensor *heads);
+/* Q4_K grouped low projection: positive is success, zero is a clean
+ * pre-enqueue fallback, and negative is a required/possibly-enqueued failure. */
 int ds4_gpu_attention_output_low_q4_K_slice_tensor(
         ds4_gpu_tensor       *low,
         const void             *model_map,
@@ -2367,7 +2946,8 @@ int ds4_gpu_attention_output_low_q4_K_slice_tensor(
         uint64_t                rank,
         uint32_t                group0,
         uint32_t                group_cnt,
-        const ds4_gpu_tensor *heads);
+        const ds4_gpu_tensor *heads,
+        int                    resident_decode);
 
 int ds4_gpu_attention_output_low_q8_rows_exact_tensor(
         ds4_gpu_tensor       *low,
@@ -2814,6 +3394,19 @@ int ds4_gpu_hc_weighted_sum_tensor(
         uint32_t                n_embd,
         uint32_t                n_hc);
 
+#ifdef __APPLE__
+/* Metal DSpark prefill capture: materialize every reduced HC row and mirror
+ * the final row in the same compute dispatch. `out` and `last_out` must refer
+ * to non-overlapping storage; production uses separate persistent tensors. */
+int ds4_gpu_hc_weighted_sum_capture_last_tensor(
+        ds4_gpu_tensor       *out,
+        ds4_gpu_tensor       *last_out,
+        const ds4_gpu_tensor *residual_hc,
+        const ds4_gpu_tensor *weights,
+        uint32_t                n_embd,
+        uint32_t                n_hc);
+#endif
+
 int ds4_gpu_hc_weighted_sum_split_tensor(
         ds4_gpu_tensor       *out,
         const ds4_gpu_tensor *residual_hc,
@@ -2854,6 +3447,10 @@ int ds4_gpu_hc_split_weighted_sum_norm_tensor(
         float                   eps,
         float                   norm_eps);
 
+/* Exact one-row HC decode fusion: unweighted RMSNorm followed by the narrow
+ * F16 HC-mix projection. The Metal and CUDA implementations are specialized
+ * for the 16384 -> 24 DS4 Flash shape and preserve their standalone reduction
+ * trees. */
 int ds4_gpu_hc_rms_norm_mix_f16_available(void);
 int ds4_gpu_hc_rms_norm_mix_f16_tensor(
         ds4_gpu_tensor       *out,
@@ -2865,22 +3462,10 @@ int ds4_gpu_hc_rms_norm_mix_f16_tensor(
         uint32_t              out_dim,
         float                 eps);
 
-/* Batched HC RMSNorm followed by its narrow F16 mixer projection. On the
- * tuned Metal path, scale_scratch stores one float per row instead of the
- * full normalized HC tensor; other shapes retain the established fallback. */
-int ds4_gpu_hc_rms_scale_project_f16_tensor(
-        ds4_gpu_tensor       *out,
-        ds4_gpu_tensor       *scale_scratch,
-        const void             *model_map,
-        uint64_t                model_size,
-        uint64_t                weight_offset,
-        uint32_t                in_dim,
-        uint32_t                out_dim,
-        const ds4_gpu_tensor *x,
-        uint32_t                n_rows,
-        float                   eps);
-
 #ifdef __APPLE__
+/* Exact one-row continuation of HC RMSNorm+mix: split/Sinkhorn, HC collapse,
+ * and the following weighted RMSNorm are encoded in the producer dispatch. */
+int ds4_gpu_hc_rms_norm_mix_split_norm_f16_available(void);
 int ds4_gpu_hc_rms_norm_mix_split_norm_f16_tensor(
         ds4_gpu_tensor       *mix,
         ds4_gpu_tensor       *out,
@@ -2928,6 +3513,22 @@ int ds4_gpu_hc_expand_add_rms_norm_mix_split_norm_f16_tensor(
         float                 norm_eps);
 
 #endif
+
+/* Batched HC RMSNorm followed by its narrow F16 mixer projection. On the
+ * tuned Metal path, scale_scratch stores one float per row instead of the
+ * full normalized HC tensor; other shapes retain the established fallback. */
+int ds4_gpu_hc_rms_scale_project_f16_tensor(
+        ds4_gpu_tensor       *out,
+        ds4_gpu_tensor       *scale_scratch,
+        const void             *model_map,
+        uint64_t                model_size,
+        uint64_t                weight_offset,
+        uint32_t                in_dim,
+        uint32_t                out_dim,
+        const ds4_gpu_tensor *x,
+        uint32_t                n_rows,
+        float                   eps);
+
 int ds4_gpu_output_hc_weights_tensor(
         ds4_gpu_tensor       *out,
         const ds4_gpu_tensor *pre,
@@ -2955,18 +3556,6 @@ int ds4_gpu_hc_expand_add_tensor(
         const ds4_gpu_tensor *comb,
         uint32_t                n_embd,
         uint32_t                n_hc);
-
-
-int ds4_gpu_hc_expand_add_tensor(
-        ds4_gpu_tensor       *out_hc,
-        const ds4_gpu_tensor *block_out,
-        const ds4_gpu_tensor *block_add,
-        const ds4_gpu_tensor *residual_hc,
-        const ds4_gpu_tensor *post,
-        const ds4_gpu_tensor *comb,
-        uint32_t                n_embd,
-        uint32_t                n_hc);
-
 int ds4_gpu_hc_expand_split_tensor(
         ds4_gpu_tensor       *out_hc,
         const ds4_gpu_tensor *block_out,
@@ -3252,11 +3841,28 @@ int ds4_gpu_glm53_kda_prefill(
         float                 gate_lower_bound,
         float                 norm_eps);
 
+/* Q4_K sibling of the decode attention-output/HC compound. */
+int ds4_gpu_matmul_q4_K_hc_expand_available(void);
+int ds4_gpu_matmul_q4_K_hc_expand_tensor(
+        ds4_gpu_tensor       *out_hc,
+        ds4_gpu_tensor       *block_out,
+        const void           *model_map,
+        uint64_t              model_size,
+        uint64_t              weight_offset,
+        uint64_t              in_dim,
+        uint64_t              out_dim,
+        const ds4_gpu_tensor *x,
+        const ds4_gpu_tensor *residual_hc,
+        const ds4_gpu_tensor *split,
+        uint32_t              n_embd,
+        uint32_t              n_hc);
+
 /* Decode-island CUDA graph capture (CUDA backend; Metal/ROCm/CPU stub it
  * out and stay eager).  Design ported from the Entrpi/ds4 batched-serving
  * fork's per-layer decode graph capture.  The key identifies a captured
- * island: layer, island index, and the activation buffers whose addresses
- * the captured kernels bake in.  ds4_cuda.cu mirrors this struct
+ * island: layer, island index, and the stable device-storage addresses that
+ * the captured kernels bake in (never short-lived view-wrapper addresses).
+ * ds4_cuda.cu mirrors this struct
  * byte-for-byte (it does not include this header); keep both in sync. */
 typedef struct ds4_decode_graph_key {
     uint32_t il;
@@ -3269,6 +3875,10 @@ typedef struct ds4_decode_graph_key {
     void    *attn_norm;
 } ds4_decode_graph_key;
 
+/* Exact-N uses a disjoint CUDA graph-cache domain so its five batch-row
+ * activation addresses cannot evict the ordinary decode variants. */
+#define DS4_DECODE_GRAPH_VARIANT_EXACTN 0x80000000u
+
 int  ds4_gpu_decode_graphs_supported(void);
 /* 1: replayed (island already executed; skip encoding it)
  * 0: capturing (encode the island, then call _end)
@@ -3279,6 +3889,14 @@ int  ds4_gpu_decode_graph_begin(const ds4_decode_graph_key *key);
 int  ds4_gpu_decode_graph_end(const ds4_decode_graph_key *key);
 void ds4_gpu_decode_graph_abort(const ds4_decode_graph_key *key);
 void ds4_gpu_decode_graphs_invalidate(void);
+#if !defined(__APPLE__) && !defined(DS4_ROCM_BUILD)
+void ds4_gpu_decode_graph_counters(
+        uint64_t *captures,
+        uint64_t *replays,
+        uint64_t *warms,
+        uint64_t *no_slots,
+        uint64_t *failures);
+#endif
 
 #ifdef __cplusplus
 }
diff --git a/ds4_help.c b/ds4_help.c
index 961ca8cb05..f716ed9c05 100644
--- a/ds4_help.c
+++ b/ds4_help.c
@@ -172,7 +172,7 @@ static void print_model_runtime(FILE *fp, const help_colors *c,
     opt(fp, c, "--power N", "GPU duty-cycle target, 1..100. Default: 100");
     opt(fp, c, "--ssd-streaming", "Metal/CUDA/ROCm: opt in to SSD-backed model streaming instead of full residency.");
     opt(fp, c, "--ssd-streaming-cold", "SSD streaming: skip default popularity-based expert-cache preload.");
-    opt(fp, c, "--ssd-streaming-cache-experts N|NGB", "SSD streaming cache target. N requests dynamic expert slots; NGB also reserves two full prefill layers. Either may be reduced to fit the model, graph, context, and backend working set.");
+    opt(fp, c, "--ssd-streaming-cache-experts N|NGB", "SSD streaming cache target. N requests dynamic expert slots without the two-layer reserve; NGB is an accounted routed budget that includes it. Either may be reduced by the final memory check. Metal SSD+DSpark can A/B a support-aware pre-cap with DS4_METAL_DSPARK_SAFE_EXPERT_COUNT=1. Auto: 80% working set minus fixed weights.");
     opt(fp, c, "--ssd-streaming-full-layers N", "GLM Metal streaming: keep the first N routed layers fully resident. Default: auto from NGB expert budget; use 0 to disable.");
     opt(fp, c, "--ssd-streaming-preload-experts N", "SSD streaming: upfront popularity preload count. DeepSeek auto-seeds by default; GLM demand-fills unless N is explicit.");
     opt(fp, c, "--simulate-used-memory NGB", "Diagnostic: lock N GiB before model load to simulate a smaller-memory machine.");
diff --git a/ds4_kvstore.c b/ds4_kvstore.c
index 5b73de7b65..40b0e2b5a9 100644
--- a/ds4_kvstore.c
+++ b/ds4_kvstore.c
@@ -1168,25 +1168,6 @@ bool ds4_kvstore_store_live_prefix(ds4_kvstore *kc,
                                               hooks, err, err_len);
 }
 
-bool ds4_kvstore_maybe_store_continued(ds4_kvstore *kc,
-                                       ds4_engine *engine,
-                                       ds4_session *session,
-                                       const ds4_kvstore_trailer_hooks *hooks,
-                                       char *err,
-                                       size_t err_len) {
-    const ds4_tokens *tokens = ds4_session_tokens(session);
-    if (!tokens) return false;
-    const int target = ds4_kvstore_continued_store_target(kc, tokens->len);
-    if (target == 0) return false;
-    if (ds4_kvstore_store_live_prefix(kc, engine, session, tokens, target,
-                                      "continued", hooks, err, err_len))
-    {
-        ds4_kvstore_note_store(kc, target);
-        return true;
-    }
-    return false;
-}
-
 int ds4_kvstore_find_text_prefix(ds4_kvstore *kc, const char *prompt_text,
                                  int model_id, int quant_bits, int ctx_size) {
     if (!prompt_text) return -1;
diff --git a/ds4_kvstore.h b/ds4_kvstore.h
index 28ccdb7eaf..47e826cc53 100644
--- a/ds4_kvstore.h
+++ b/ds4_kvstore.h
@@ -17,7 +17,7 @@
 #define DS4_KVSTORE_EXT_THINKING_VISIBLE  (1u << 2)
 #define DS4_KVSTORE_EXT_SESSION_TITLE     (1u << 3)
 
-typedef enum {
+enum {
     DS4_KVSTORE_REASON_UNKNOWN   = 0,
     DS4_KVSTORE_REASON_COLD      = 1,
     DS4_KVSTORE_REASON_CONTINUED = 2,
@@ -25,7 +25,7 @@ typedef enum {
     DS4_KVSTORE_REASON_SHUTDOWN  = 4,
     DS4_KVSTORE_REASON_AGENT_SYSTEM  = 5,
     DS4_KVSTORE_REASON_AGENT_SESSION = 6,
-} ds4_kvstore_reason;
+};
 
 typedef enum {
     DS4_KVSTORE_LOG_DEFAULT,
@@ -180,12 +180,6 @@ bool ds4_kvstore_store_live_prefix(ds4_kvstore *kc,
                                    const ds4_kvstore_trailer_hooks *hooks,
                                    char *err,
                                    size_t err_len);
-bool ds4_kvstore_maybe_store_continued(ds4_kvstore *kc,
-                                       ds4_engine *engine,
-                                       ds4_session *session,
-                                       const ds4_kvstore_trailer_hooks *hooks,
-                                       char *err,
-                                       size_t err_len);
 int ds4_kvstore_try_load_text(ds4_kvstore *kc,
                               ds4_engine *engine,
                               ds4_session *session,
diff --git a/ds4_metal.m b/ds4_metal.m
index 47bea08216..8278b34e3c 100644
--- a/ds4_metal.m
+++ b/ds4_metal.m
@@ -1,5 +1,6 @@
 #import <Foundation/Foundation.h>
 #import <Metal/Metal.h>
+#import <dispatch/dispatch.h>
 #include <execinfo.h>
 
 #include <stdint.h>
@@ -17,6 +18,7 @@
 #include <pthread.h>
 #include <unistd.h>
 #include <sys/mman.h>
+#include <sys/stat.h>
 #include <sys/sysctl.h>
 #include <mach/mach.h>
 #include <mach-o/dyld.h>
@@ -49,10 +51,28 @@
     DS4_METAL_TENSOR_MXFP4   = 39,
 };
 
+/* kernel_mul_mm_mpp_direct_rhs double-buffers two 64x32 half tiles. */
+enum {
+    DS4_METAL_MPP_DIRECT_RHS_SMEM = 2u * 64u * 32u * sizeof(uint16_t),
+    /* One fixed-capacity, stream-local sidecar avoids replacing a buffer that
+     * an unretained/in-flight command buffer may still reference.  8192 is
+     * above every currently supported q_a/KV input width; 128 is the largest
+     * exact token tile admitted by the experimental pair path. */
+    DS4_METAL_Q4_PAIR_RHS_MAX_IN = 8192u,
+    DS4_METAL_Q4_PAIR_RHS_MAX_TOKENS = 128u,
+    DS4_METAL_Q4_PAIR_RHS_BYTES =
+        DS4_METAL_Q4_PAIR_RHS_MAX_IN *
+        DS4_METAL_Q4_PAIR_RHS_MAX_TOKENS * sizeof(uint16_t),
+};
+
 @class DS4MetalQ4ExpertTable;
 
 static id<MTLDevice> g_device;
+#define DS4_GPU_MAX_STREAMS 8
+static _Thread_local int g_ds4_stream = 0;
 static id<MTLCommandQueue> g_queue;
+static id<MTLCommandQueue> g_stream_queues[DS4_GPU_MAX_STREAMS];
+static id<MTLCommandBuffer> g_stream_last_cb[DS4_GPU_MAX_STREAMS];
 static id<MTLLibrary> g_library;
 static id<MTLCommandBuffer> g_batch_cb;
 static id<MTLComputeCommandEncoder> g_batch_enc;
@@ -375,7 +395,13 @@ static void ds4_gpu_timeline_attach(id<MTLCommandBuffer> cb) {
 }
 
 static void ds4_gpu_parallel_ffn_reset_state(BOOL close_encoder);
-static NSMutableArray<id<MTLCommandBuffer>> *g_pending_cbs;
+static NSMutableArray<id<MTLCommandBuffer>>
+    *g_pending_cbs_by_stream[DS4_GPU_MAX_STREAMS];
+#define g_pending_cbs (g_pending_cbs_by_stream[g_ds4_stream])
+/* Completion handlers may still be running after waitUntilCompleted returns.
+ * Track the small progress hooks explicitly so stack-backed hook contexts can
+ * be joined before a command drain returns to the graph driver. */
+static dispatch_group_t g_progress_completion_group;
 static id<MTLSharedEvent> g_selected_readback_event;
 static uint64_t g_selected_readback_event_value;
 static id<MTLComputePipelineState> g_set_rows_f32_i32_pipeline;
@@ -395,7 +421,6 @@ static void ds4_gpu_timeline_attach(id<MTLCommandBuffer> cb) {
 static id<MTLComputePipelineState> g_cpy_contig_f16_f32_pipeline;
 static id<MTLComputePipelineState> g_cpy_contig_f16_f16_pipeline;
 static id<MTLComputePipelineState> g_flash_kv_stage_f16_pipeline;
-static id<MTLComputePipelineState> g_swiglu_pipeline;
 static id<MTLComputePipelineState> g_swiglu_flat_pipeline;
 static id<MTLComputePipelineState> g_add_pipeline;
 static id<MTLComputePipelineState> g_add2_pipeline;
@@ -418,8 +443,10 @@ static void ds4_gpu_timeline_attach(id<MTLCommandBuffer> cb) {
 static NSMutableDictionary<NSString *, id<MTLBuffer>> *g_dsv4_hc_barrier_cache;
 static NSMutableDictionary<NSString *, NSNumber *> *g_dsv4_hc_barrier_gen;
 static id<MTLComputePipelineState> g_hc_weighted_sum_pipeline;
+static id<MTLComputePipelineState> g_hc_weighted_sum_capture_last_pipeline;
 static id<MTLComputePipelineState> g_output_hc_weights4_pipeline;
 static uint32_t g_test_flags;
+static _Thread_local uint32_t g_test_last_flash_attn_prefill_nwg;
 static id<MTLComputePipelineState> g_hc_expand_pipeline;
 static id<MTLComputePipelineState> g_unary_sigmoid_pipeline;
 static id<MTLComputePipelineState> g_unary_silu_pipeline;
@@ -471,6 +498,8 @@ static void ds4_gpu_timeline_attach(id<MTLCommandBuffer> cb) {
 static id<MTLComputePipelineState> g_moe_mul_mv_slots6_mxfp4_pair_swiglu_pipeline;
 static id<MTLComputePipelineState> g_moe_mul_mv_slots6_mxfp4_sum6_pipeline;
 static id<MTLComputePipelineState> g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_pipeline;
+static id<MTLComputePipelineState> g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_mid_only_4096x2048_pipeline;
+static id<MTLComputePipelineState> g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_mid_only_4096x2048_masked_pipeline;
 static id<MTLComputePipelineState> g_moe_mul_mv_addr_iq2_xxs_pipeline;
 static id<MTLComputePipelineState> g_moe_mul_mv_addr_q2_k_sum6_pipeline;
 static id<MTLComputePipelineState> g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_masked_pipeline;
@@ -503,6 +532,8 @@ static void ds4_gpu_timeline_attach(id<MTLCommandBuffer> cb) {
 static id<MTLComputePipelineState> g_soft_max_f32_4_pipeline;
 static id<MTLComputePipelineState> g_argsort_f32_i32_desc_pipeline;
 static id<MTLComputePipelineState> g_argsort_merge_f32_i32_desc_pipeline;
+static id<MTLComputePipelineState> g_dsv4_argmax_top1_stage1_pipeline;
+static id<MTLComputePipelineState> g_dsv4_argmax_top1_stage2_pipeline;
 static id<MTLComputePipelineState> g_sum_rows_f32_f32_pipeline;
 static id<MTLComputePipelineState> g_dsv4_topk_mask_pipeline;
 static id<MTLComputePipelineState> g_dsv4_topk_mask_scatter_pipeline;
@@ -515,6 +546,7 @@ static void ds4_gpu_timeline_attach(id<MTLCommandBuffer> cb) {
 static id<MTLComputePipelineState> g_dsv4_indexed_attention_heads16_dual_pipeline;
 static id<MTLComputePipelineState> g_dsv4_indexed_attention_heads8_split_pipeline;
 static id<MTLComputePipelineState> g_dsv4_indexed_attention_heads8_split_reduce_pipeline;
+static bool g_attn_out_low_q8_static_unavailable;
 static id<MTLComputePipelineState> g_dsv4_softplus_sqrt_pipeline;
 static id<MTLComputePipelineState> g_dsv4_router_finalize_one_pipeline;
 static id<MTLComputePipelineState> g_dsv4_router_finalize_one_simd_pipeline;
@@ -545,13 +577,11 @@ static void ds4_gpu_timeline_attach(id<MTLCommandBuffer> cb) {
 static id<MTLComputePipelineState> g_glm_indexer_score_one_direct_pipeline;
 static id<MTLComputePipelineState> g_glm_indexer_scores_batch_pipeline;
 static id<MTLComputePipelineState> g_glm_indexer_scores_tiled_pipeline;
-static id<MTLComputePipelineState> g_glm_indexer_scores_tiled_f32_pipeline;
 static id<MTLComputePipelineState> g_glm_qk_lowrank_pipeline;
 static id<MTLComputePipelineState> g_glm_qk_lowrank_glm52_pipeline;
 static id<MTLComputePipelineState> g_glm_qk_lowrank_glm52_sg_pipeline;
 static id<MTLComputePipelineState> g_glm_qk_lowrank_batch_pipeline;
 static id<MTLComputePipelineState> g_glm_qk_lowrank_batch_glm52_t4_pipeline;
-static id<MTLComputePipelineState> g_glm_value_project_q8_0_pipeline;
 static id<MTLComputePipelineState> g_glm_value_project_q8_0_batch_heads_pipeline;
 static id<MTLComputePipelineState> g_glm_value_project_q8_0_batch_heads_mma_pipeline;
 static id<MTLComputePipelineState> g_glm_attention_indexed_decode_pipeline;
@@ -568,11 +598,8 @@ static void ds4_gpu_timeline_attach(id<MTLCommandBuffer> cb) {
 static id<MTLComputePipelineState> g_glm_attention_indexed_batch_lora_group8_vec_valid_fullheads_pipeline;
 static id<MTLComputePipelineState> g_glm_attention_indexed_batch_lora_group8_vec_causal_pipeline;
 static id<MTLComputePipelineState> g_glm_attention_indexed_batch_lora_group8_vec_causal_fullheads_pipeline;
-static id<MTLComputePipelineState> g_glm_q4_k_pair_swiglu_f32_pipeline;
 static id<MTLComputePipelineState> g_glm_q4_k_pair_swiglu2_f32_pipeline;
 static id<MTLComputePipelineState> g_glm_q4_k_pair_swiglu4_f32_pipeline;
-static id<MTLComputePipelineState> g_glm_q4_k_pair_swiglu2_mapped_f32_pipeline;
-static id<MTLComputePipelineState> g_glm_q4_k_pair_swiglu2_mapped_row_f32_pipeline;
 static id<MTLComputePipelineState> g_glm_q2_k_pair_swiglu_f32_pipeline;
 static id<MTLComputePipelineState> g_glm_q2_k_addr_pair_swiglu2_f32_pipeline;
 static id<MTLComputePipelineState> g_glm_q2_k_addr_pair_swiglu2_masked_f32_pipeline;
@@ -583,13 +610,14 @@ static void ds4_gpu_timeline_attach(id<MTLCommandBuffer> cb) {
 static id<MTLComputePipelineState> g_glm_q2_k_addr_down_f32_pipeline;
 static id<MTLComputePipelineState> g_glm_q4_k_addr_down_f32_pipeline;
 static id<MTLComputePipelineState> g_glm_q5_k_pair_swiglu_f32_pipeline;
-static id<MTLComputePipelineState> g_glm_q5_k_pair_swiglu_mapped_f32_pipeline;
-static id<MTLComputePipelineState> g_glm_q5_k_pair_swiglu_mapped_row_f32_pipeline;
 static id<MTLComputePipelineState> g_glm_q5_k_down_f32_pipeline;
 static id<MTLComputePipelineState> g_glm_q6_k_down_f32_pipeline;
 static id<MTLComputePipelineState> g_dsv4_router_weights_batch_pipeline;
 static id<MTLComputePipelineState> g_dsv4_hc_expand4_pipeline;
+static id<MTLComputePipelineState> g_dsv4_attn_out_q8_mm_hc_expand4_pipeline;
+static id<MTLComputePipelineState> g_dsv4_attn_out_q4_mm_hc_expand4_pipeline;
 static NSMutableDictionary<NSString *, id<MTLComputePipelineState>> *g_pipeline_cache;
+static uint64_t g_pipeline_cache_generation;
 
 enum {
     DS4_METAL_DECODE_PIPELINE_FAST_CACHE_SLOTS = 64,
@@ -626,7 +654,9 @@ static void ds4_gpu_timeline_attach(id<MTLCommandBuffer> cb) {
 static NSMutableDictionary<NSString *, id<MTLBuffer>> *g_model_buffer_cache;
 static NSMutableDictionary<NSString *, DS4MetalQ4ExpertTable *> *g_q4_expert_table_cache;
 static NSMutableDictionary<NSString *, id> *g_q4_expert_layer_residency_cache;
-static NSMutableArray<id<MTLBuffer>> *g_transient_buffers;
+static NSMutableArray<id<MTLBuffer>>
+    *g_transient_buffers_by_stream[DS4_GPU_MAX_STREAMS];
+#define g_transient_buffers (g_transient_buffers_by_stream[g_ds4_stream])
 static id g_model_residency_set;
 
 typedef struct {
@@ -658,42 +688,154 @@ static void ds4_gpu_timeline_attach(id<MTLCommandBuffer> cb) {
 static ds4_gpu_zero_prefix_prefill_mask_cache_entry
     g_zero_prefix_prefill_mask_cache[DS4_GPU_PREFILL_MASK_CACHE_SLOTS];
 static void ds4_gpu_invalidate_zero_prefix_prefill_block_maps(void);
-static id<MTLBuffer> g_flash_attn_mask_buffer;
-static id<MTLBuffer> g_flash_attn_zero_mask_buffer;
-static id<MTLBuffer> g_flash_attn_pad_buffer;
-static id<MTLBuffer> g_flash_attn_tmp_buffer;
-static id<MTLBuffer> g_flash_attn_blk_buffer;
-static id<MTLBuffer> g_flash_attn_ring_buffer;
-static id<MTLBuffer> g_flash_attn_kv_buffer;
-static id<MTLBuffer> g_glm_flash_attn_mask_buffer;
-static id<MTLBuffer> g_compressor_pool_kv_buffer;
-static id<MTLBuffer> g_compressor_pool_score_buffer;
-static id<MTLBuffer> g_compressor_pool_score_cont_buffer;
-static id<MTLBuffer> g_compressor_pool_softmax_buffer;
-static id<MTLBuffer> g_compressor_pool_product_buffer;
-static id<MTLBuffer> g_compressor_store_ape_buffer;
-static id<MTLBuffer> g_compressor_store_score_buffer;
-static id<MTLBuffer> g_embed_rows_buffer;
-static id<MTLBuffer> g_router_selection_buffer;
-static id<MTLBuffer> g_router_weight_sum_buffer;
-static id<MTLBuffer> g_indexer_head_scores_buffer;
-static id<MTLBuffer> g_indexer_topk_buffer;
-static id<MTLBuffer> g_indexed_topk_buffer;
-static id<MTLBuffer> g_f16_round_scratch_buffer;
-static id<MTLBuffer> g_raw_store_round_buffer;
-static id<MTLBuffer> g_moe_gate_scratch_buffer;
-static id<MTLBuffer> g_moe_down_scratch_buffer;
-static id<MTLBuffer> g_moe_id_map_buffer;
-static id<MTLBuffer> g_moe_q4_gate_slots_buffer;
-static id<MTLBuffer> g_moe_q4_up_slots_buffer;
-static id<MTLBuffer> g_moe_q4_down_slots_buffer;
-static id<MTLBuffer> g_attn_out_group_ids_buffer;
+typedef struct {
+    __strong id<MTLBuffer> flash_attn_mask_buffer;
+    __strong id<MTLBuffer> flash_attn_zero_mask_buffer;
+    __strong id<MTLBuffer> flash_attn_pad_buffer;
+    __strong id<MTLBuffer> flash_attn_tmp_buffer;
+    __strong id<MTLBuffer> flash_attn_blk_buffer;
+    __strong id<MTLBuffer> flash_attn_kv_buffer;
+    __strong id<MTLBuffer> glm_flash_attn_mask_buffer;
+    __strong id<MTLBuffer> compressor_pool_kv_buffer;
+    __strong id<MTLBuffer> compressor_pool_score_buffer;
+    __strong id<MTLBuffer> compressor_pool_score_cont_buffer;
+    __strong id<MTLBuffer> compressor_pool_softmax_buffer;
+    __strong id<MTLBuffer> compressor_pool_product_buffer;
+    __strong id<MTLBuffer> compressor_store_ape_buffer;
+    __strong id<MTLBuffer> compressor_store_score_buffer;
+    __strong id<MTLBuffer> embed_rows_buffer;
+    __strong id<MTLBuffer> router_selection_buffer;
+    __strong id<MTLBuffer> router_weight_sum_buffer;
+    __strong id<MTLBuffer> indexer_head_scores_buffer;
+    __strong id<MTLBuffer> indexer_topk_buffer;
+    __strong id<MTLBuffer> argmax_top1_scratch_v;
+    __strong id<MTLBuffer> argmax_top1_scratch_i;
+    uint32_t argmax_top1_seq;
+    __strong id<MTLBuffer> indexed_topk_buffer;
+    __strong id<MTLBuffer> q4_pair_rhs_f16_buffer;
+    __strong id<MTLBuffer> f16_round_scratch_buffer;
+    __strong id<MTLBuffer> raw_store_round_buffer;
+    __strong id<MTLBuffer> moe_gate_scratch_buffer;
+    __strong id<MTLBuffer> moe_down_scratch_buffer;
+    __strong id<MTLBuffer> moe_id_map_buffer;
+    __strong id<MTLBuffer> moe_q4_gate_slots_buffer;
+    __strong id<MTLBuffer> moe_q4_up_slots_buffer;
+    __strong id<MTLBuffer> moe_q4_down_slots_buffer;
+    __strong id<MTLBuffer> attn_out_group_ids_buffer;
+    NSUInteger flash_attn_mask_bytes;
+    NSUInteger flash_attn_zero_mask_bytes;
+    NSUInteger flash_attn_pad_bytes;
+    NSUInteger flash_attn_tmp_bytes;
+    NSUInteger flash_attn_blk_bytes;
+    NSUInteger flash_attn_kv_bytes;
+    NSUInteger glm_flash_attn_mask_bytes;
+    NSUInteger compressor_pool_kv_bytes;
+    NSUInteger compressor_pool_score_bytes;
+    NSUInteger compressor_pool_score_cont_bytes;
+    NSUInteger compressor_pool_softmax_bytes;
+    NSUInteger compressor_pool_product_bytes;
+    NSUInteger compressor_store_ape_bytes;
+    NSUInteger compressor_store_score_bytes;
+    NSUInteger embed_rows_bytes;
+    NSUInteger router_selection_bytes;
+    NSUInteger router_weight_sum_bytes;
+    NSUInteger indexer_head_scores_bytes;
+    NSUInteger indexer_topk_bytes;
+    NSUInteger indexed_topk_bytes;
+    NSUInteger q4_pair_rhs_f16_bytes;
+    NSUInteger f16_round_scratch_bytes;
+    NSUInteger raw_store_round_bytes;
+    NSUInteger moe_gate_scratch_bytes;
+    NSUInteger moe_down_scratch_bytes;
+    NSUInteger moe_id_map_bytes;
+    NSUInteger moe_q4_gate_slots_bytes;
+    NSUInteger moe_q4_up_slots_bytes;
+    NSUInteger moe_q4_down_slots_bytes;
+    NSUInteger attn_out_group_ids_bytes;
+} ds4_gpu_stream_scratch_state;
+
+static ds4_gpu_stream_scratch_state
+    g_stream_scratch[DS4_GPU_MAX_STREAMS];
+
+#define DS4_STREAM_SCRATCH(field_) (g_stream_scratch[g_ds4_stream].field_)
+#define g_flash_attn_mask_buffer DS4_STREAM_SCRATCH(flash_attn_mask_buffer)
+#define g_flash_attn_zero_mask_buffer DS4_STREAM_SCRATCH(flash_attn_zero_mask_buffer)
+#define g_flash_attn_pad_buffer DS4_STREAM_SCRATCH(flash_attn_pad_buffer)
+#define g_flash_attn_tmp_buffer DS4_STREAM_SCRATCH(flash_attn_tmp_buffer)
+#define g_flash_attn_blk_buffer DS4_STREAM_SCRATCH(flash_attn_blk_buffer)
+#define g_flash_attn_kv_buffer DS4_STREAM_SCRATCH(flash_attn_kv_buffer)
+#define g_glm_flash_attn_mask_buffer DS4_STREAM_SCRATCH(glm_flash_attn_mask_buffer)
+#define g_compressor_pool_kv_buffer DS4_STREAM_SCRATCH(compressor_pool_kv_buffer)
+#define g_compressor_pool_score_buffer DS4_STREAM_SCRATCH(compressor_pool_score_buffer)
+#define g_compressor_pool_score_cont_buffer DS4_STREAM_SCRATCH(compressor_pool_score_cont_buffer)
+#define g_compressor_pool_softmax_buffer DS4_STREAM_SCRATCH(compressor_pool_softmax_buffer)
+#define g_compressor_pool_product_buffer DS4_STREAM_SCRATCH(compressor_pool_product_buffer)
+#define g_compressor_store_ape_buffer DS4_STREAM_SCRATCH(compressor_store_ape_buffer)
+#define g_compressor_store_score_buffer DS4_STREAM_SCRATCH(compressor_store_score_buffer)
+#define g_embed_rows_buffer DS4_STREAM_SCRATCH(embed_rows_buffer)
+#define g_router_selection_buffer DS4_STREAM_SCRATCH(router_selection_buffer)
+#define g_router_weight_sum_buffer DS4_STREAM_SCRATCH(router_weight_sum_buffer)
+#define g_indexer_head_scores_buffer DS4_STREAM_SCRATCH(indexer_head_scores_buffer)
+#define g_indexer_topk_buffer DS4_STREAM_SCRATCH(indexer_topk_buffer)
+#define g_indexed_topk_buffer DS4_STREAM_SCRATCH(indexed_topk_buffer)
+#define g_q4_pair_rhs_f16_buffer DS4_STREAM_SCRATCH(q4_pair_rhs_f16_buffer)
+#define g_f16_round_scratch_buffer DS4_STREAM_SCRATCH(f16_round_scratch_buffer)
+#define g_raw_store_round_buffer DS4_STREAM_SCRATCH(raw_store_round_buffer)
+#define g_moe_gate_scratch_buffer DS4_STREAM_SCRATCH(moe_gate_scratch_buffer)
+#define g_moe_down_scratch_buffer DS4_STREAM_SCRATCH(moe_down_scratch_buffer)
+#define g_moe_id_map_buffer DS4_STREAM_SCRATCH(moe_id_map_buffer)
+#define g_moe_q4_gate_slots_buffer DS4_STREAM_SCRATCH(moe_q4_gate_slots_buffer)
+#define g_moe_q4_up_slots_buffer DS4_STREAM_SCRATCH(moe_q4_up_slots_buffer)
+#define g_moe_q4_down_slots_buffer DS4_STREAM_SCRATCH(moe_q4_down_slots_buffer)
+#define g_attn_out_group_ids_buffer DS4_STREAM_SCRATCH(attn_out_group_ids_buffer)
+#define g_flash_attn_mask_bytes DS4_STREAM_SCRATCH(flash_attn_mask_bytes)
+#define g_flash_attn_zero_mask_bytes DS4_STREAM_SCRATCH(flash_attn_zero_mask_bytes)
+#define g_flash_attn_pad_bytes DS4_STREAM_SCRATCH(flash_attn_pad_bytes)
+#define g_flash_attn_tmp_bytes DS4_STREAM_SCRATCH(flash_attn_tmp_bytes)
+#define g_flash_attn_blk_bytes DS4_STREAM_SCRATCH(flash_attn_blk_bytes)
+#define g_flash_attn_kv_bytes DS4_STREAM_SCRATCH(flash_attn_kv_bytes)
+#define g_glm_flash_attn_mask_bytes DS4_STREAM_SCRATCH(glm_flash_attn_mask_bytes)
+#define g_compressor_pool_kv_bytes DS4_STREAM_SCRATCH(compressor_pool_kv_bytes)
+#define g_compressor_pool_score_bytes DS4_STREAM_SCRATCH(compressor_pool_score_bytes)
+#define g_compressor_pool_score_cont_bytes DS4_STREAM_SCRATCH(compressor_pool_score_cont_bytes)
+#define g_compressor_pool_softmax_bytes DS4_STREAM_SCRATCH(compressor_pool_softmax_bytes)
+#define g_compressor_pool_product_bytes DS4_STREAM_SCRATCH(compressor_pool_product_bytes)
+#define g_compressor_store_ape_bytes DS4_STREAM_SCRATCH(compressor_store_ape_bytes)
+#define g_compressor_store_score_bytes DS4_STREAM_SCRATCH(compressor_store_score_bytes)
+#define g_embed_rows_bytes DS4_STREAM_SCRATCH(embed_rows_bytes)
+#define g_router_selection_bytes DS4_STREAM_SCRATCH(router_selection_bytes)
+#define g_router_weight_sum_bytes DS4_STREAM_SCRATCH(router_weight_sum_bytes)
+#define g_indexer_head_scores_bytes DS4_STREAM_SCRATCH(indexer_head_scores_bytes)
+#define g_indexer_topk_bytes DS4_STREAM_SCRATCH(indexer_topk_bytes)
+#define g_indexed_topk_bytes DS4_STREAM_SCRATCH(indexed_topk_bytes)
+#define g_q4_pair_rhs_f16_bytes DS4_STREAM_SCRATCH(q4_pair_rhs_f16_bytes)
+#define g_f16_round_scratch_bytes DS4_STREAM_SCRATCH(f16_round_scratch_bytes)
+#define g_raw_store_round_bytes DS4_STREAM_SCRATCH(raw_store_round_bytes)
+#define g_moe_gate_scratch_bytes DS4_STREAM_SCRATCH(moe_gate_scratch_bytes)
+#define g_moe_down_scratch_bytes DS4_STREAM_SCRATCH(moe_down_scratch_bytes)
+#define g_moe_id_map_bytes DS4_STREAM_SCRATCH(moe_id_map_bytes)
+#define g_moe_q4_gate_slots_bytes DS4_STREAM_SCRATCH(moe_q4_gate_slots_bytes)
+#define g_moe_q4_up_slots_bytes DS4_STREAM_SCRATCH(moe_q4_up_slots_bytes)
+#define g_moe_q4_down_slots_bytes DS4_STREAM_SCRATCH(moe_q4_down_slots_bytes)
+#define g_attn_out_group_ids_bytes DS4_STREAM_SCRATCH(attn_out_group_ids_bytes)
 static int g_model_fd = -1;
+/* Second model descriptor with F_NOCACHE for all streaming expert preads or
+ * only the batched-prefill requests.  Individual pread tasks carry the phase
+ * preference so decode never depends on mutable process-global phase state. */
+static int g_model_fd_nocache = -1;
+static int g_model_fd_nocache_all;
+static int g_model_fd_nocache_prefill;
+static int g_model_fd_nocache_prefill_auto;
 static const void *g_model_map_ptr;
 static uint64_t g_model_map_size;
 static uint64_t g_model_mapped_offset;
 static uint64_t g_model_mapped_size;
 static uint64_t g_model_mapped_max_tensor_bytes;
+/* Keep the support GGUF identity separate from the most recently installed
+ * model view.  The target and DSpark mappings coexist, and g_model_map_ptr is
+ * intentionally updated whenever either mapping is prepared. */
+static const void *g_support_model_map_ptr;
+static uint64_t g_support_model_map_size;
 static uint64_t g_tensor_alloc_live_bytes;
 static uint64_t g_tensor_alloc_peak_bytes;
 static pthread_mutex_t g_tensor_mu = PTHREAD_MUTEX_INITIALIZER;
@@ -707,8 +849,274 @@ static void ds4_gpu_timeline_attach(id<MTLCommandBuffer> cb) {
 static uint64_t g_model_buffer_cache_bytes;
 static uint64_t g_model_buffer_cache_evictions;
 static int g_model_buffer_cache_over_limit;
+#define DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES 80u
+typedef struct {
+    const void                 *model_map;
+    uint64_t                    model_size;
+    uint64_t                    weight_offset;
+    uint64_t                    weight_bytes;
+    uint64_t                    in_dim;
+    uint64_t                    out_dim;
+    uint64_t                    f16_bytes;
+    id<MTLBuffer> __strong      buffer;
+    int                         valid;
+} ds4_gpu_q4_attn_q_b_f16_cache_entry;
+static ds4_gpu_q4_attn_q_b_f16_cache_entry
+    g_q4_attn_q_b_f16_cache[DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES];
+static pthread_mutex_t g_q4_attn_q_b_f16_cache_mu = PTHREAD_MUTEX_INITIALIZER;
+static pthread_mutex_t g_q4_attn_q_b_f16_build_mu = PTHREAD_MUTEX_INITIALIZER;
+static uint64_t g_q4_attn_q_b_f16_cache_bytes;
+static uint64_t g_q4_attn_q_b_f16_cache_lookups;
+static uint64_t g_q4_attn_q_b_f16_cache_hits;
+static uint64_t g_q4_attn_q_b_f16_cache_misses;
+static uint64_t g_q4_attn_q_b_f16_cache_builds;
+static uint64_t g_q4_attn_q_b_f16_cache_build_failures;
+static uint64_t g_q4_attn_q_b_f16_cache_candidate_calls;
+static uint64_t g_q4_attn_q_b_f16_cache_fallbacks;
+static uint64_t g_q4_attn_q_b_f16_cache_rejects;
+static uint32_t g_q4_attn_q_b_f16_cache_entries;
+static uint64_t g_q4_attn_q_b_f16_cache_generation = 1u;
+/* A failed resident->SSD release must not leave an old resident sidecar
+ * reachable through the SSD opt-in path.  The lifecycle setter closes this
+ * latch only after a successful synchronized release. */
+static int g_q4_attn_q_b_f16_ssd_admission_blocked;
+typedef enum {
+    DS4_Q4_ATTN_Q_B_F16_CIRCUIT_CLOSED = 0,
+    DS4_Q4_ATTN_Q_B_F16_CIRCUIT_HARD = 1,
+    DS4_Q4_ATTN_Q_B_F16_CIRCUIT_PRESSURE = 2,
+} ds4_gpu_q4_attn_q_b_f16_circuit_state;
+static ds4_gpu_q4_attn_q_b_f16_circuit_state
+    g_q4_attn_q_b_f16_build_circuit_state;
+/* Layer-agnostic, stream-local F16 expansion scratch.  Each serial Metal
+ * stream rewrites its own buffer from the current Q4_K source immediately
+ * before consuming it.  Independent streams therefore never alias this
+ * transient state, while consecutive layers reuse only 64 MiB per stream. */
+static id<MTLBuffer> __strong
+    g_q4_qb_transient_f16_scratch[DS4_GPU_MAX_STREAMS];
+static NSUInteger
+    g_q4_qb_transient_f16_scratch_capacity[DS4_GPU_MAX_STREAMS];
+typedef struct {
+    const void *model_map;
+    uint64_t model_size;
+    uint64_t generation;
+    uint64_t reserve_bytes;
+    bool admitted;
+} ds4_gpu_q4_qb_transient_f16_admission;
+static ds4_gpu_q4_qb_transient_f16_admission
+    g_q4_qb_transient_f16_admission[DS4_GPU_MAX_STREAMS];
+static uint64_t g_q4_qb_transient_f16_exact_views_created;
+static uint64_t g_q4_qb_transient_f16_exact_views_live;
+/* Protect allocation/publication, SSD admission metadata, and cleanup.
+ * Encoding never retains this mutex; command ordering and the
+ * one-in-flight-batch-per-stream rule protect reuse after a buffer has been
+ * captured strongly by the caller. */
+static pthread_mutex_t g_q4_qb_transient_f16_scratch_mu =
+    PTHREAD_MUTEX_INITIALIZER;
+static int g_initialized;
+
+static void ds4_gpu_q4_qb_transient_f16_admission_clear(void) {
+    pthread_mutex_lock(&g_q4_qb_transient_f16_scratch_mu);
+    memset(g_q4_qb_transient_f16_admission, 0,
+           sizeof(g_q4_qb_transient_f16_admission));
+    pthread_mutex_unlock(&g_q4_qb_transient_f16_scratch_mu);
+}
+
+static bool ds4_gpu_q4_qb_transient_f16_admission_begin(
+        int stream,
+        const void *model_map,
+        uint64_t model_size,
+        uint64_t reserve_bytes,
+        bool *same_contract) {
+    if (stream < 0 || stream >= DS4_GPU_MAX_STREAMS) return false;
+    pthread_mutex_lock(&g_q4_qb_transient_f16_scratch_mu);
+    const bool invalidated =
+        g_q4_qb_transient_f16_admission[stream].admitted;
+    if (same_contract) {
+        *same_contract = invalidated &&
+            g_q4_qb_transient_f16_admission[stream].model_map == model_map &&
+            g_q4_qb_transient_f16_admission[stream].model_size == model_size &&
+            g_q4_qb_transient_f16_admission[stream].reserve_bytes ==
+                reserve_bytes;
+    }
+    g_q4_qb_transient_f16_admission[stream] =
+        (ds4_gpu_q4_qb_transient_f16_admission) {
+            .model_map = model_map,
+            .model_size = model_size,
+            .generation = 0u,
+            .reserve_bytes = reserve_bytes,
+            .admitted = false,
+        };
+    pthread_mutex_unlock(&g_q4_qb_transient_f16_scratch_mu);
+    return invalidated;
+}
+
+static void ds4_gpu_q4_qb_transient_f16_admission_commit(
+        int stream,
+        const void *model_map,
+        uint64_t model_size,
+        uint64_t generation,
+        uint64_t reserve_bytes) {
+    if (stream < 0 || stream >= DS4_GPU_MAX_STREAMS) return;
+    pthread_mutex_lock(&g_q4_qb_transient_f16_scratch_mu);
+    ds4_gpu_q4_qb_transient_f16_admission *state =
+        &g_q4_qb_transient_f16_admission[stream];
+    if (state->model_map == model_map &&
+        state->model_size == model_size) {
+        state->generation = generation;
+        state->reserve_bytes = reserve_bytes;
+        state->admitted = true;
+    }
+    pthread_mutex_unlock(&g_q4_qb_transient_f16_scratch_mu);
+}
+
+static bool ds4_gpu_q4_qb_transient_f16_admission_allows(
+        int stream,
+        const void *model_map,
+        uint64_t model_size,
+        uint64_t generation) {
+    if (stream < 0 || stream >= DS4_GPU_MAX_STREAMS) return false;
+    pthread_mutex_lock(&g_q4_qb_transient_f16_scratch_mu);
+    const ds4_gpu_q4_qb_transient_f16_admission *state =
+        &g_q4_qb_transient_f16_admission[stream];
+    const bool allowed = state->admitted &&
+        state->model_map == model_map &&
+        state->model_size == model_size &&
+        state->generation == generation;
+    pthread_mutex_unlock(&g_q4_qb_transient_f16_scratch_mu);
+    return allowed;
+}
+
+static void ds4_gpu_q4_attn_q_b_f16_cache_clear(int reset_stats) {
+    pthread_mutex_lock(&g_q4_attn_q_b_f16_build_mu);
+    pthread_mutex_lock(&g_q4_attn_q_b_f16_cache_mu);
+    for (uint32_t i = 0;
+         i < DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES;
+         i++) {
+        ds4_gpu_q4_attn_q_b_f16_cache_entry *entry =
+            &g_q4_attn_q_b_f16_cache[i];
+        entry->buffer = nil;
+        entry->model_map = NULL;
+        entry->model_size = 0;
+        entry->weight_offset = 0;
+        entry->weight_bytes = 0;
+        entry->in_dim = 0;
+        entry->out_dim = 0;
+        entry->f16_bytes = 0;
+        entry->valid = 0;
+    }
+    g_q4_attn_q_b_f16_cache_bytes = 0;
+    g_q4_attn_q_b_f16_cache_entries = 0;
+    g_q4_attn_q_b_f16_cache_generation++;
+    if (g_q4_attn_q_b_f16_cache_generation == 0u) {
+        g_q4_attn_q_b_f16_cache_generation = 1u;
+    }
+    g_q4_attn_q_b_f16_build_circuit_state =
+        DS4_Q4_ATTN_Q_B_F16_CIRCUIT_CLOSED;
+    if (reset_stats) {
+        g_q4_attn_q_b_f16_cache_lookups = 0;
+        g_q4_attn_q_b_f16_cache_hits = 0;
+        g_q4_attn_q_b_f16_cache_misses = 0;
+        g_q4_attn_q_b_f16_cache_builds = 0;
+        g_q4_attn_q_b_f16_cache_build_failures = 0;
+        g_q4_attn_q_b_f16_cache_candidate_calls = 0;
+        g_q4_attn_q_b_f16_cache_fallbacks = 0;
+        g_q4_attn_q_b_f16_cache_rejects = 0;
+    }
+    pthread_mutex_unlock(&g_q4_attn_q_b_f16_cache_mu);
+    pthread_mutex_unlock(&g_q4_attn_q_b_f16_build_mu);
+}
+
+static void ds4_gpu_q4_attn_q_b_f16_advance_generation(void) {
+    pthread_mutex_lock(&g_q4_attn_q_b_f16_cache_mu);
+    g_q4_attn_q_b_f16_cache_generation++;
+    if (g_q4_attn_q_b_f16_cache_generation == 0u) {
+        g_q4_attn_q_b_f16_cache_generation = 1u;
+    }
+    pthread_mutex_unlock(&g_q4_attn_q_b_f16_cache_mu);
+}
+
+void ds4_gpu_test_q4_attn_q_b_f16_cache_report(
+        ds4_gpu_q4_attn_q_b_f16_cache_report *report) {
+    if (!report) return;
+    pthread_mutex_lock(&g_q4_attn_q_b_f16_cache_mu);
+    *report = (ds4_gpu_q4_attn_q_b_f16_cache_report) {
+        .entries = g_q4_attn_q_b_f16_cache_entries,
+        .bytes = g_q4_attn_q_b_f16_cache_bytes,
+        .lookups = g_q4_attn_q_b_f16_cache_lookups,
+        .hits = g_q4_attn_q_b_f16_cache_hits,
+        .misses = g_q4_attn_q_b_f16_cache_misses,
+        .builds = g_q4_attn_q_b_f16_cache_builds,
+        .build_failures = g_q4_attn_q_b_f16_cache_build_failures,
+        .candidate_calls = g_q4_attn_q_b_f16_cache_candidate_calls,
+        .fallbacks = g_q4_attn_q_b_f16_cache_fallbacks,
+        .rejects = g_q4_attn_q_b_f16_cache_rejects,
+        .build_circuit_open = (uint64_t)(
+            g_q4_attn_q_b_f16_build_circuit_state !=
+            DS4_Q4_ATTN_Q_B_F16_CIRCUIT_CLOSED),
+        .transient_exact_views_created = __atomic_load_n(
+            &g_q4_qb_transient_f16_exact_views_created,
+            __ATOMIC_ACQUIRE),
+        .transient_exact_views_live = __atomic_load_n(
+            &g_q4_qb_transient_f16_exact_views_live,
+            __ATOMIC_ACQUIRE),
+        .model_exact_cache_entries = g_model_buffer_cache
+            ? (uint64_t)[g_model_buffer_cache count] : 0u,
+        .model_exact_cache_bytes = g_model_buffer_cache_bytes,
+    };
+    pthread_mutex_unlock(&g_q4_attn_q_b_f16_cache_mu);
+}
+
+void ds4_gpu_test_q4_attn_q_b_f16_cache_reset(void) {
+    if (g_initialized && !ds4_gpu_synchronize()) {
+        __atomic_store_n(&g_q4_attn_q_b_f16_ssd_admission_blocked,
+                         1,
+                         __ATOMIC_RELEASE);
+        return;
+    }
+    ds4_gpu_q4_attn_q_b_f16_cache_clear(1);
+    ds4_gpu_q4_qb_transient_f16_admission_clear();
+    __atomic_store_n(&g_q4_qb_transient_f16_exact_views_created,
+                     0u,
+                     __ATOMIC_RELEASE);
+    __atomic_store_n(&g_q4_qb_transient_f16_exact_views_live,
+                     0u,
+                     __ATOMIC_RELEASE);
+    __atomic_store_n(&g_q4_attn_q_b_f16_ssd_admission_blocked,
+                     0,
+                     __ATOMIC_RELEASE);
+}
+
+uint64_t ds4_gpu_q4_attn_q_b_f16_cache_generation(void) {
+    pthread_mutex_lock(&g_q4_attn_q_b_f16_cache_mu);
+    const uint64_t generation = g_q4_attn_q_b_f16_cache_generation;
+    pthread_mutex_unlock(&g_q4_attn_q_b_f16_cache_mu);
+    return generation;
+}
+
+int ds4_gpu_release_q4_attn_q_b_f16_sidecars(void) {
+    int has_entries = 0;
+    pthread_mutex_lock(&g_q4_attn_q_b_f16_cache_mu);
+    has_entries = g_q4_attn_q_b_f16_cache_entries != 0u;
+    pthread_mutex_unlock(&g_q4_attn_q_b_f16_cache_mu);
+
+    /* Callers use this only at session/mode lifecycle boundaries.  Finish
+     * every command that may still reference an unretained Metal buffer
+     * before dropping the cache's final strong references.  Always clear,
+     * even with zero entries: a rejected build may have left the circuit
+     * breaker open and must not poison the next resident session. */
+    if (has_entries && g_initialized && !ds4_gpu_synchronize()) return 0;
+    ds4_gpu_q4_attn_q_b_f16_cache_clear(0);
+    ds4_gpu_q4_qb_transient_f16_admission_clear();
+    __atomic_store_n(&g_q4_attn_q_b_f16_ssd_admission_blocked,
+                     0,
+                     __ATOMIC_RELEASE);
+    return 1;
+}
 static uint64_t g_stream_expert_cache_bytes;
 static uint64_t g_stream_expert_cache_expert_bytes;
+static uint64_t g_stream_expert_cache_gate_class_bytes;
+static uint64_t g_stream_expert_cache_down_class_bytes;
+static int g_stream_expert_cache_class_conflict;
 static uint32_t g_stream_expert_cache_entry_count;
 static uint32_t g_stream_expert_cache_budget_override;
 static uint64_t g_stream_expert_cache_hits;
@@ -720,6 +1128,10 @@ static void ds4_gpu_timeline_attach(id<MTLCommandBuffer> cb) {
 static uint64_t g_stream_expert_cache_willneed_advise_bytes;
 static uint64_t g_stream_expert_cache_pread_bytes;
 static double g_stream_expert_cache_pread_ms;
+static uint64_t g_stream_expert_pread_cached_calls;
+static uint64_t g_stream_expert_pread_cached_bytes;
+static uint64_t g_stream_expert_pread_nocache_calls;
+static uint64_t g_stream_expert_pread_nocache_bytes;
 static uint64_t g_stream_expert_cache_buffer_allocs;
 static uint64_t g_stream_expert_cache_buffer_reuses;
 static uint64_t g_stream_expert_cache_decode_tokens;
@@ -754,6 +1166,17 @@ static void ds4_gpu_timeline_attach(id<MTLCommandBuffer> cb) {
 static uint64_t g_stream_expert_timing_reuse_scan_entries;
 static double g_stream_expert_timing_reuse_scan_ms;
 static double g_stream_expert_timing_reuse_clear_ms;
+static uint64_t g_stream_expert_live_index_scans;
+static uint64_t g_stream_expert_live_index_entries;
+static uint64_t g_stream_expert_live_index_fallbacks;
+static uint64_t g_stream_expert_live_index_inserts;
+static uint64_t g_stream_expert_live_index_removes;
+static uint64_t g_stream_expert_exact_persistent_calls;
+static uint64_t g_stream_expert_exact_transient_calls;
+static uint64_t g_stream_expert_exact_persistent_fallbacks;
+static uint64_t g_stream_expert_exact_persistent_failures;
+static uint64_t g_stream_expert_exact_mapped_view_calls;
+static uint32_t g_stream_expert_exact_max_unique;
 static uint64_t g_stream_expert_timing_readahead_calls;
 static uint64_t g_stream_expert_timing_readahead_bytes;
 static double g_stream_expert_timing_readahead_ms;
@@ -805,8 +1228,20 @@ static void ds4_gpu_timeline_attach(id<MTLCommandBuffer> cb) {
 static ds4_gpu_stream_expert_timing_snapshot g_stream_expert_timing_last_report;
 static int g_stream_prefill_batch_selected_addr_building;
 static int g_glm_stream_expert_addr_table_building;
+/* Coverage-only counters for the IQ2_XXS/Q2_K SSD grouped address-MM path.
+ * They are updated on the host after an otherwise successful routed batch
+ * and never add a command-buffer boundary or GPU synchronization. */
+static int g_iq2_stream_addr_mm_stats_enabled;
+static uint64_t g_iq2_stream_addr_mm_candidate_calls;
+static uint64_t g_iq2_stream_addr_mm_calls;
+static uint64_t g_iq2_stream_addr_mm_tokens;
+static uint64_t g_iq2_stream_addr_mm_rows;
+static uint64_t g_iq2_stream_addr_mm_require_failures;
+static uint32_t g_iq2_stream_addr_mm_min_tokens;
+static uint32_t g_iq2_stream_addr_mm_max_tokens;
 static uint64_t g_model_residency_count;
 static int g_model_residency_added_to_queue;
+static uint32_t g_model_residency_queue_mask;
 static int g_glm_model_mode;
 static int g_ssd_streaming_mode;
 static int g_glm_streaming_prefill_full_layer_runtime;
@@ -825,6 +1260,7 @@ static int ds4_gpu_stream_expert_cache_note_expert_size(
 static void ds4_gpu_stream_expert_cache_clear_all(int reset_stats);
 static void ds4_gpu_stream_expert_pending_load_clear(void);
 static void ds4_gpu_stream_expert_pread_pool_shutdown(void);
+static void ds4_gpu_stream_expert_cache_live_release(void);
 static int ds4_gpu_stream_expert_timing_summary_enabled(void);
 static int ds4_gpu_stream_expert_cache_entry_protected(
         uint32_t       layer,
@@ -848,41 +1284,10 @@ static int ds4_gpu_stream_expert_cache_on_service_thread(void) {
     return g_stream_expert_service_thread_set &&
            pthread_equal(pthread_self(), g_stream_expert_service_thread);
 }
-static NSUInteger g_flash_attn_mask_bytes;
-static NSUInteger g_flash_attn_zero_mask_bytes;
-static NSUInteger g_flash_attn_pad_bytes;
-static NSUInteger g_flash_attn_tmp_bytes;
-static NSUInteger g_flash_attn_blk_bytes;
-static NSUInteger g_flash_attn_ring_bytes;
-static NSUInteger g_flash_attn_kv_bytes;
-static NSUInteger g_glm_flash_attn_mask_bytes;
 static uint32_t g_glm_flash_attn_mask_pos0;
 static uint32_t g_glm_flash_attn_mask_tokens;
 static uint32_t g_glm_flash_attn_mask_cache_len;
 static int g_glm_flash_attn_mask_valid;
-static NSUInteger g_compressor_pool_kv_bytes;
-static NSUInteger g_compressor_pool_score_bytes;
-static NSUInteger g_compressor_pool_score_cont_bytes;
-static NSUInteger g_compressor_pool_softmax_bytes;
-static NSUInteger g_compressor_pool_product_bytes;
-static NSUInteger g_compressor_store_ape_bytes;
-static NSUInteger g_compressor_store_score_bytes;
-static NSUInteger g_embed_rows_bytes;
-static NSUInteger g_router_selection_bytes;
-static NSUInteger g_router_weight_sum_bytes;
-static NSUInteger g_indexer_head_scores_bytes;
-static NSUInteger g_indexer_topk_bytes;
-static NSUInteger g_indexed_topk_bytes;
-static NSUInteger g_f16_round_scratch_bytes;
-static NSUInteger g_raw_store_round_bytes;
-static NSUInteger g_moe_gate_scratch_bytes;
-static NSUInteger g_moe_down_scratch_bytes;
-static NSUInteger g_moe_id_map_bytes;
-static NSUInteger g_moe_q4_gate_slots_bytes;
-static NSUInteger g_moe_q4_up_slots_bytes;
-static NSUInteger g_moe_q4_down_slots_bytes;
-static NSUInteger g_attn_out_group_ids_bytes;
-static int g_initialized;
 static int g_quality_mode;
 static int g_mpp_invalid_env_reported;
 #define DS4_METAL_MAX_ROUTED_EXPERT_USED 8
@@ -942,6 +1347,10 @@ static void ds4_gpu_print_device_summary(void) {
     DS4_METAL_STREAM_EXPERT_CACHE_MAX_SLABS = 256,
     DS4_METAL_STREAM_EXPERT_HOTNESS_DECAY_TOKENS = 16,
     DS4_METAL_STREAM_EXPERT_VALIDATE_WORDS = 16,
+    DS4_METAL_STREAM_EXPERT_PREAD_MAX_SPLIT = 8,
+    DS4_METAL_STREAM_EXPERT_PENDING_MAX_TASKS =
+        DS4_METAL_STREAM_EXPERT_CACHE_MAX_SELECTED * 3u *
+        DS4_METAL_STREAM_EXPERT_PREAD_MAX_SPLIT,
 };
 
 typedef struct {
@@ -988,6 +1397,79 @@ static void ds4_gpu_print_device_summary(void) {
     NSUInteger down_inner;
 } ds4_gpu_stream_expert_reusable_buffers;
 
+enum {
+    DS4_METAL_EXACT_ROWS_MAX = 5,
+    DS4_METAL_EXACT_ROWS_MAX_RESOURCES =
+        DS4_METAL_EXACT_ROWS_MAX * DS4_METAL_STREAM_EXPERT_CACHE_MAX_SELECTED,
+};
+
+/* A layer-local immutable view of all experts selected by an exact speculative
+ * microbatch.  The selected matrix and address tables are private to the
+ * scope: ordinary decode's mutable per-layer table can therefore be updated by
+ * neither a later row nor cache pruning while earlier rows remain in flight. */
+typedef struct {
+    int collecting;
+    int active;
+    int row_armed;
+    uint32_t row;
+    uint32_t next_row;
+    uint32_t n_rows;
+    uint32_t n_selected;
+    ds4_gpu_stream_expert_table table;
+    int32_t selected_ids[DS4_METAL_EXACT_ROWS_MAX_RESOURCES];
+    __strong id<MTLBuffer> selected_buffer;
+    NSUInteger selected_offset;
+    __strong id<MTLBuffer> gate_addrs;
+    __strong id<MTLBuffer> up_addrs;
+    __strong id<MTLBuffer> down_addrs;
+    __strong id<MTLBuffer> overflow_gate;
+    __strong id<MTLBuffer> overflow_up;
+    __strong id<MTLBuffer> overflow_down;
+    ds4_gpu_stream_expert_cache_entry
+        *resources[DS4_METAL_EXACT_ROWS_MAX_RESOURCES];
+    __strong id<MTLBuffer>
+        resource_gate[DS4_METAL_EXACT_ROWS_MAX_RESOURCES];
+    __strong id<MTLBuffer>
+        resource_up[DS4_METAL_EXACT_ROWS_MAX_RESOURCES];
+    __strong id<MTLBuffer>
+        resource_down[DS4_METAL_EXACT_ROWS_MAX_RESOURCES];
+    uint32_t n_resources;
+    uint32_t unique_count;
+} ds4_gpu_stream_expert_exact_rows_scope;
+
+static ds4_gpu_stream_expert_exact_rows_scope
+    g_stream_expert_exact_rows_scope;
+
+static void ds4_gpu_stream_expert_exact_rows_clear(void) {
+    ds4_gpu_stream_expert_exact_rows_scope *scope =
+        &g_stream_expert_exact_rows_scope;
+    scope->selected_buffer = nil;
+    scope->gate_addrs = nil;
+    scope->up_addrs = nil;
+    scope->down_addrs = nil;
+    scope->overflow_gate = nil;
+    scope->overflow_up = nil;
+    scope->overflow_down = nil;
+    for (uint32_t i = 0; i < DS4_METAL_EXACT_ROWS_MAX_RESOURCES; i++) {
+        scope->resources[i] = NULL;
+        scope->resource_gate[i] = nil;
+        scope->resource_up[i] = nil;
+        scope->resource_down[i] = nil;
+        scope->selected_ids[i] = -1;
+    }
+    memset(&scope->table, 0, sizeof(scope->table));
+    scope->collecting = 0;
+    scope->active = 0;
+    scope->row_armed = 0;
+    scope->row = 0;
+    scope->next_row = 0;
+    scope->n_rows = 0;
+    scope->n_selected = 0;
+    scope->selected_offset = 0;
+    scope->n_resources = 0;
+    scope->unique_count = 0;
+}
+
 static ds4_gpu_stream_expert_cache_entry
     g_stream_expert_cache[DS4_METAL_STREAM_EXPERT_CACHE_MAX_LAYER][DS4_METAL_STREAM_EXPERT_CACHE_MAX_EXPERT];
 static ds4_gpu_stream_expert_cache_entry
@@ -1025,6 +1507,7 @@ static void ds4_gpu_print_device_summary(void) {
 static double g_stream_expert_cache_mlock_ms;
 static int g_stream_expert_cache_mlock_warned;
 static uint64_t g_stream_expert_cache_slab_slot_bytes;
+static uint64_t g_stream_expert_cache_slab_allocated_bytes;
 static uint64_t g_stream_expert_cache_cb_seq;
 static uint64_t g_stream_expert_cache_done_seq;
 static uint64_t g_stream_expert_cache_batch_seq;
@@ -1037,6 +1520,19 @@ static void ds4_gpu_print_device_summary(void) {
 static id<MTLBuffer> g_stream_selected_id_buffers[DS4_METAL_STREAM_EXPERT_CACHE_MAX_LAYER];
 static id<MTLBuffer> g_stream_expert_validate_status_buffer;
 
+static void ds4_gpu_atomic_u64_add_sat(uint64_t *value, uint64_t add) {
+    uint64_t current = __atomic_load_n(value, __ATOMIC_RELAXED);
+    for (;;) {
+        const uint64_t next =
+            add > UINT64_MAX - current ? UINT64_MAX : current + add;
+        if (__atomic_compare_exchange_n(value, &current, next, 0,
+                                        __ATOMIC_RELEASE,
+                                        __ATOMIC_RELAXED)) {
+            return;
+        }
+    }
+}
+
 @interface DS4MetalTensor : NSObject
 @property(nonatomic, strong) id<MTLBuffer> buffer;
 @property(nonatomic, assign) uint64_t offset;
@@ -1052,7 +1548,7 @@ @interface DS4MetalQ4ExpertTable : NSObject
 @property(nonatomic, strong) id<MTLBuffer> addressBuffer;
 @property(nonatomic, strong) NSMutableArray<id<MTLBuffer>> *expertBuffers;
 @property(nonatomic, strong) id residencySet;
-@property(nonatomic, assign) BOOL residencySetAddedToQueue;
+@property(nonatomic, assign) uint32_t residencySetQueueMask;
 @property(nonatomic, assign) uint32_t nExpert;
 @property(nonatomic, assign) uint64_t expertBytes;
 @end
@@ -1062,10 +1558,14 @@ - (void)dealloc {
 #if TARGET_OS_OSX
     if (@available(macOS 15.0, *)) {
         if (_residencySet) {
-            if (_residencySetAddedToQueue &&
-                g_queue &&
-                [g_queue respondsToSelector:@selector(removeResidencySet:)]) {
-                [g_queue removeResidencySet:_residencySet];
+            for (int i = 0; i < DS4_GPU_MAX_STREAMS; i++) {
+                if ((_residencySetQueueMask & (1u << i)) == 0) continue;
+                id<MTLCommandQueue> queue =
+                    i == 0 ? g_queue : g_stream_queues[i];
+                if (queue &&
+                    [queue respondsToSelector:@selector(removeResidencySet:)]) {
+                    [queue removeResidencySet:_residencySet];
+                }
             }
             [_residencySet endResidency];
         }
@@ -1076,7 +1576,8 @@ - (void)dealloc {
 
 @interface DS4MetalQ4LayerResidency : NSObject
 @property(nonatomic, strong) id residencySet;
-@property(nonatomic, assign) BOOL addedToQueue;
+@property(nonatomic, assign) BOOL queueResident;
+@property(nonatomic, assign) uint32_t queueMask;
 @end
 
 @implementation DS4MetalQ4LayerResidency
@@ -1084,10 +1585,14 @@ - (void)dealloc {
 #if TARGET_OS_OSX
     if (@available(macOS 15.0, *)) {
         if (_residencySet) {
-            if (_addedToQueue &&
-                g_queue &&
-                [g_queue respondsToSelector:@selector(removeResidencySet:)]) {
-                [g_queue removeResidencySet:_residencySet];
+            for (int i = 0; i < DS4_GPU_MAX_STREAMS; i++) {
+                if ((_queueMask & (1u << i)) == 0) continue;
+                id<MTLCommandQueue> queue =
+                    i == 0 ? g_queue : g_stream_queues[i];
+                if (queue &&
+                    [queue respondsToSelector:@selector(removeResidencySet:)]) {
+                    [queue removeResidencySet:_residencySet];
+                }
             }
             [_residencySet endResidency];
         }
@@ -1275,6 +1780,86 @@ static NSUInteger ds4_gpu_tensor_offset(const ds4_gpu_tensor *tensor) {
 }
 
 static id<MTLCommandBuffer> ds4_gpu_new_command_buffer(void);
+
+/* Command encoding remains serialized through the process-global batch, but
+ * committed command buffers can execute concurrently on independent queues.
+ * The TLS selector also selects the stream-local scratch/transient state. */
+void ds4_gpu_set_stream(int idx) {
+    if (g_batch_cb) return;
+    if (idx < 0 || idx >= DS4_GPU_MAX_STREAMS) idx = 0;
+    g_ds4_stream = idx;
+}
+
+int ds4_gpu_current_stream(void) { return g_ds4_stream; }
+
+static uint32_t ds4_gpu_active_queue_mask(void) {
+    uint32_t mask = g_queue ? 1u : 0u;
+    for (int i = 1; i < DS4_GPU_MAX_STREAMS; i++) {
+        if (g_stream_queues[i]) mask |= 1u << i;
+    }
+    return mask;
+}
+
+static void ds4_gpu_q4_attach_layer_residency_to_queue(
+        DS4MetalQ4LayerResidency *entry,
+        id<MTLCommandQueue>       queue,
+        int                       stream) {
+#if TARGET_OS_OSX
+    if (!entry || !entry.queueResident || !entry.residencySet || !queue ||
+        stream < 0 || stream >= DS4_GPU_MAX_STREAMS) {
+        return;
+    }
+    if (@available(macOS 15.0, *)) {
+        const uint32_t bit = 1u << stream;
+        if ((entry.queueMask & bit) == 0 &&
+            [queue respondsToSelector:@selector(addResidencySet:)]) {
+            [queue addResidencySet:entry.residencySet];
+            entry.queueMask |= bit;
+        }
+    }
+#else
+    (void)entry;
+    (void)queue;
+    (void)stream;
+#endif
+}
+
+static void ds4_gpu_q4_attach_cached_residency_to_queue(
+        id<MTLCommandQueue> queue,
+        int                 stream) {
+#if TARGET_OS_OSX
+    if (!queue || stream < 0 || stream >= DS4_GPU_MAX_STREAMS ||
+        !g_q4_expert_layer_residency_cache) {
+        return;
+    }
+    if (@available(macOS 15.0, *)) {
+        for (DS4MetalQ4LayerResidency *entry in
+                 [g_q4_expert_layer_residency_cache allValues]) {
+            ds4_gpu_q4_attach_layer_residency_to_queue(entry, queue, stream);
+        }
+    }
+#else
+    (void)queue;
+    (void)stream;
+#endif
+}
+
+static id<MTLCommandQueue> ds4_gpu_active_queue(void) {
+    if (g_ds4_stream == 0) return g_queue;
+    if (!g_stream_queues[g_ds4_stream]) {
+        id<MTLCommandQueue> queue = [g_device newCommandQueue];
+        if (queue && g_model_residency_set &&
+            g_model_residency_added_to_queue &&
+            [queue respondsToSelector:@selector(addResidencySet:)]) {
+            [queue addResidencySet:g_model_residency_set];
+            g_model_residency_queue_mask |= 1u << g_ds4_stream;
+        }
+        g_stream_queues[g_ds4_stream] = queue;
+        ds4_gpu_q4_attach_cached_residency_to_queue(queue, g_ds4_stream);
+    }
+    return g_stream_queues[g_ds4_stream];
+}
+
 static void ds4_gpu_stream_expert_cache_note_owned_created(void);
 
 static id<MTLCommandBuffer> ds4_gpu_command_buffer(int *owned) {
@@ -1329,6 +1914,11 @@ static void ds4_gpu_close_batch_encoder(void) {
 
 static double g_gpu_busy_accum;
 static uint64_t g_gpu_busy_cbs;
+/* Stage profilers split a batch into owned command buffers.  Retain the GPU
+ * interval from the last synchronous completion on the calling thread so
+ * their reports can exclude CPU encoding, submit, and wait overhead. */
+static _Thread_local double g_last_completed_gpu_seconds;
+static _Thread_local int g_last_completed_gpu_time_valid;
 
 /* A failed command buffer can leave a cross-threadgroup arrival counter at an
  * arbitrary partial value.  Drop cached ownership instead of CPU-resetting
@@ -1342,7 +1932,15 @@ static void ds4_gpu_invalidate_completion_counters(void) {
 }
 
 static int ds4_gpu_wait_command_buffer(id<MTLCommandBuffer> cb, const char *label) {
+    g_last_completed_gpu_time_valid = 0;
     [cb waitUntilCompleted];
+    const double gpu_start = cb.GPUStartTime;
+    const double gpu_end = cb.GPUEndTime;
+    const double busy = gpu_end - gpu_start;
+    if (isfinite(gpu_start) && isfinite(gpu_end) && busy > 0.0) {
+        g_last_completed_gpu_seconds = busy;
+        g_last_completed_gpu_time_valid = 1;
+    }
     if (getenv("DS4_METAL_CB_TIMES")) {
         static double prev_gpu_end;
         static uint64_t n_printed;
@@ -1359,7 +1957,6 @@ static int ds4_gpu_wait_command_buffer(id<MTLCommandBuffer> cb, const char *labe
         prev_gpu_end = cb.GPUEndTime;
     }
     if (getenv("DS4_METAL_GPU_BUSY_PROFILE")) {
-        const double busy = cb.GPUEndTime - cb.GPUStartTime;
         if (busy > 0) g_gpu_busy_accum += busy;
         if ((++g_gpu_busy_cbs % 64u) == 0u) {
             fprintf(stderr, "ds4: gpu busy accum %.1f ms over %llu cbs\n",
@@ -1384,9 +1981,9 @@ static int ds4_gpu_wait_command_buffer(id<MTLCommandBuffer> cb, const char *labe
         initialized = 1;
     }
     if (use_unretained) {
-        return [g_queue commandBufferWithUnretainedReferences];
+        return [ds4_gpu_active_queue() commandBufferWithUnretainedReferences];
     }
-    return [g_queue commandBuffer];
+    return [ds4_gpu_active_queue() commandBuffer];
 }
 
 static uint64_t ds4_gpu_exact_view_cache_limit_bytes(void) {
@@ -1458,9 +2055,16 @@ static void ds4_gpu_model_buffer_cache_clear(const char *reason) {
 }
 
 static void ds4_gpu_model_buffer_cache_maybe_evict(const char *reason) {
-    if (g_model_buffer_cache_over_limit) {
-        ds4_gpu_model_buffer_cache_clear(reason);
+    if (!g_model_buffer_cache_over_limit || g_batch_cb) return;
+    /* Exact model views can be the only strong references when command
+     * buffers are unretained. Defer eviction until every stream is drained. */
+    for (int i = 0; i < DS4_GPU_MAX_STREAMS; i++) {
+        if (g_stream_last_cb[i] ||
+            [g_pending_cbs_by_stream[i] count] != 0) {
+            return;
+        }
     }
+    ds4_gpu_model_buffer_cache_clear(reason);
 }
 
 static uint64_t ds4_gpu_stream_expert_cache_next_cb_seq(void) {
@@ -1545,17 +2149,35 @@ static int ds4_gpu_stream_expert_cache_mark_entries_inflight(
 
 static int ds4_gpu_stream_expert_cache_wait_inflight(const char *label);
 
-static int ds4_gpu_wait_pending_command_buffers(const char *label) {
+static int ds4_gpu_wait_pending_command_buffers_for_stream(
+        int idx, const char *label) {
+    if (idx < 0 || idx >= DS4_GPU_MAX_STREAMS) return 0;
     int ok = 1;
-    for (id<MTLCommandBuffer> pending in g_pending_cbs) {
+    for (id<MTLCommandBuffer> pending in g_pending_cbs_by_stream[idx]) {
         if (!ds4_gpu_wait_command_buffer(pending, label)) ok = 0;
     }
-    [g_pending_cbs removeAllObjects];
+    [g_pending_cbs_by_stream[idx] removeAllObjects];
+    if (g_progress_completion_group) {
+        dispatch_group_wait(g_progress_completion_group,
+                            DISPATCH_TIME_FOREVER);
+    }
     ds4_gpu_stream_expert_cache_note_pending_completed();
     if (!ok) ds4_gpu_invalidate_zero_prefix_prefill_block_maps();
     return ok;
 }
 
+static int ds4_gpu_wait_pending_command_buffers(const char *label) {
+    return ds4_gpu_wait_pending_command_buffers_for_stream(
+            g_ds4_stream, label);
+}
+
+static int ds4_gpu_pending_command_buffers_any(void) {
+    for (int i = 0; i < DS4_GPU_MAX_STREAMS; i++) {
+        if ([g_pending_cbs_by_stream[i] count] != 0) return 1;
+    }
+    return 0;
+}
+
 static double g_batch_cb_created_ms;
 static double ds4_gpu_now_ms(void);
 static void ds4_gpu_queue_keepalive_start(void);
@@ -2065,25 +2687,82 @@ static void ds4_gpu_progress_failed(void) {
     fflush(stderr);
 }
 
+static void ds4_gpu_model_view_clear(uint32_t i) {
+    if (i >= DS4_METAL_MAX_MODEL_VIEWS) return;
+    g_model_views[i].buffer = nil;
+    g_model_views[i].model_map = NULL;
+    g_model_views[i].model_size = 0;
+    g_model_views[i].model_offset = 0;
+    g_model_views[i].bytes = 0;
+}
+
+static void ds4_gpu_model_views_truncate(uint32_t count) {
+    if (count > g_model_view_count) return;
+    for (uint32_t i = count; i < g_model_view_count; i++) {
+        ds4_gpu_model_view_clear(i);
+    }
+    g_model_view_count = count;
+}
+
 static void ds4_gpu_model_views_clear(void) {
-    for (uint32_t i = 0; i < g_model_view_count; i++) {
-        g_model_views[i].buffer = nil;
-        g_model_views[i].model_map = NULL;
-        g_model_views[i].model_size = 0;
-        g_model_views[i].model_offset = 0;
-        g_model_views[i].bytes = 0;
+    ds4_gpu_model_views_truncate(0);
+}
+
+static void ds4_gpu_model_map_state_invalidate(void) {
+    g_model_map_ptr = NULL;
+    g_model_map_size = 0;
+    g_model_mapped_offset = 0;
+    g_model_mapped_size = 0;
+    g_model_mapped_max_tensor_bytes = 0;
+}
+
+/* Replace only the old views for one mmap while preserving views belonging to
+ * another simultaneously active model (for example a DSpark support GGUF).
+ * New views are appended first; prefix_count marks the registry boundary from
+ * before the append, so a failed build can be truncated without disturbing
+ * the previous mapping. */
+static void ds4_gpu_model_views_remove_map_prefix(
+        const void *model_map,
+        uint64_t    model_size,
+        uint32_t    prefix_count) {
+    if (!model_map || g_model_view_count == 0) return;
+    if (prefix_count > g_model_view_count) prefix_count = g_model_view_count;
+
+    uint32_t write = 0;
+    for (uint32_t read = 0; read < g_model_view_count; read++) {
+        const bool remove =
+            read < prefix_count &&
+            g_model_views[read].model_map == model_map &&
+            g_model_views[read].model_size == model_size;
+        if (remove) {
+            ds4_gpu_model_view_clear(read);
+            continue;
+        }
+        if (write != read) {
+            g_model_views[write].buffer = g_model_views[read].buffer;
+            g_model_views[write].model_map = g_model_views[read].model_map;
+            g_model_views[write].model_size = g_model_views[read].model_size;
+            g_model_views[write].model_offset = g_model_views[read].model_offset;
+            g_model_views[write].bytes = g_model_views[read].bytes;
+            ds4_gpu_model_view_clear(read);
+        }
+        write++;
     }
-    g_model_view_count = 0;
+    g_model_view_count = write;
 }
 
 static void ds4_gpu_model_residency_clear(void) {
 #if TARGET_OS_OSX
     if (@available(macOS 15.0, *)) {
         if (g_model_residency_set) {
-            if (g_model_residency_added_to_queue &&
-                g_queue &&
-                [g_queue respondsToSelector:@selector(removeResidencySet:)]) {
-                [g_queue removeResidencySet:g_model_residency_set];
+            for (int i = 0; i < DS4_GPU_MAX_STREAMS; i++) {
+                if ((g_model_residency_queue_mask & (1u << i)) == 0) continue;
+                id<MTLCommandQueue> queue =
+                    i == 0 ? g_queue : g_stream_queues[i];
+                if (queue &&
+                    [queue respondsToSelector:@selector(removeResidencySet:)]) {
+                    [queue removeResidencySet:g_model_residency_set];
+                }
             }
             [g_model_residency_set endResidency];
             [g_model_residency_set removeAllAllocations];
@@ -2093,6 +2772,7 @@ static void ds4_gpu_model_residency_clear(void) {
 #endif
     g_model_residency_count = 0;
     g_model_residency_added_to_queue = 0;
+    g_model_residency_queue_mask = 0;
 }
 
 /* TP sharding keeps only this rank's expert ranges warm,
@@ -2140,11 +2820,20 @@ static int ds4_gpu_model_residency_request_views(void) {
         }
         [g_model_residency_set commit];
         [g_model_residency_set requestResidency];
-        if (getenv("DS4_METAL_DISABLE_QUEUE_RESIDENCY_SET") == NULL &&
-            g_queue &&
-            [g_queue respondsToSelector:@selector(addResidencySet:)]) {
-            [g_queue addResidencySet:g_model_residency_set];
-            g_model_residency_added_to_queue = 1;
+        g_model_residency_queue_mask = 0;
+        if (getenv("DS4_METAL_DISABLE_QUEUE_RESIDENCY_SET") == NULL) {
+            for (int i = 0; i < DS4_GPU_MAX_STREAMS; i++) {
+                id<MTLCommandQueue> queue =
+                    i == 0 ? g_queue : g_stream_queues[i];
+                if (!queue ||
+                    ![queue respondsToSelector:@selector(addResidencySet:)]) {
+                    continue;
+                }
+                [queue addResidencySet:g_model_residency_set];
+                g_model_residency_queue_mask |= 1u << i;
+            }
+            g_model_residency_added_to_queue =
+                g_model_residency_queue_mask != 0;
         }
         g_model_residency_count = g_model_view_count;
     }
@@ -2355,26 +3044,6 @@ static int ds4_gpu_finish_model_views(
     return 1;
 }
 
-static int ds4_gpu_map_model_views(
-        const void *model_map,
-        uint64_t    model_size,
-        uint64_t    map_offset,
-        uint64_t    map_size,
-        uint64_t    max_tensor_bytes) {
-    const double t0 = ds4_gpu_now_ms();
-    uint64_t mapped_model_size = 0;
-    if (!ds4_gpu_add_model_view_range(model_map,
-                                      model_size,
-                                      map_offset,
-                                      map_size,
-                                      max_tensor_bytes,
-                                      false,
-                                      &mapped_model_size)) {
-        return 0;
-    }
-    return ds4_gpu_finish_model_views(t0, mapped_model_size, map_offset);
-}
-
 static id<MTLBuffer> ds4_gpu_new_transient_buffer(NSUInteger bytes, const char *label) {
     if (bytes == 0) bytes = 1;
 
@@ -2396,6 +3065,42 @@ static int ds4_gpu_map_model_views(
     return buffer;
 }
 
+int ds4_gpu_test_hold_stream_transient(uint64_t bytes) {
+    if (!g_initialized && !ds4_gpu_init()) return 0;
+    if (bytes > (uint64_t)NSUIntegerMax) return 0;
+    return ds4_gpu_new_transient_buffer(
+            (NSUInteger)bytes, "ds4_stream_test_transient") != nil;
+}
+
+void ds4_gpu_test_stream_stats(ds4_gpu_stream_test_stats *stats) {
+    if (!stats) return;
+    memset(stats, 0, sizeof(*stats));
+    pthread_mutex_lock(&g_tensor_mu);
+    stats->tensor_live_bytes = g_tensor_alloc_live_bytes;
+    stats->tensor_live_count =
+        g_tensor_live_count > UINT32_MAX ?
+            UINT32_MAX : (uint32_t)g_tensor_live_count;
+    pthread_mutex_unlock(&g_tensor_mu);
+
+    uint64_t pending = 0;
+    for (int i = 0; i < DS4_GPU_MAX_STREAMS; i++) {
+        stats->transient_references +=
+            (uint64_t)[g_transient_buffers_by_stream[i] count];
+        pending += (uint64_t)[g_pending_cbs_by_stream[i] count];
+        if (g_stream_last_cb[i]) stats->last_command_buffers++;
+    }
+    stats->pending_command_buffers =
+        pending > UINT32_MAX ? UINT32_MAX : (uint32_t)pending;
+    stats->active_queue_mask = ds4_gpu_active_queue_mask();
+    stats->model_residency_queue_mask = g_model_residency_queue_mask;
+    if (g_q4_expert_layer_residency_cache) {
+        for (DS4MetalQ4LayerResidency *entry in
+                 [g_q4_expert_layer_residency_cache allValues]) {
+            stats->q4_residency_queue_mask |= entry.queueMask;
+        }
+    }
+}
+
 static int ds4_gpu_zero_prefix_prefill_mask_cache_enabled(void) {
     if (getenv("DS4_METAL_DISABLE_ZERO_PREFIX_PREFILL_MASK_CACHE") != NULL ||
         getenv("DS4_METAL_FLASH_ATTN_STAGE_PROFILE") != NULL) {
@@ -2519,9 +3224,12 @@ void ds4_gpu_release_zero_prefix_prefill_mask_cache(void) {
      * release point has no outstanding cache users. Keep the guard here for
      * diagnostic callers that may have an open or asynchronously flushed CB. */
     if (!g_initialized || g_batch_cb ||
-        (g_pending_cbs && [g_pending_cbs count] != 0)) {
+        ds4_gpu_pending_command_buffers_any()) {
         return;
     }
+    for (int i = 0; i < DS4_GPU_MAX_STREAMS; i++) {
+        if (g_stream_last_cb[i]) return;
+    }
     ds4_gpu_clear_zero_prefix_prefill_mask_cache();
 }
 
@@ -2689,6 +3397,129 @@ static int ds4_gpu_env_bool(const char *name) {
     return 1;
 }
 
+/* IQ2_XXS/Q2_K SSD grouped address-MM is the production default.  Its
+ * implicit fail-closed arm is deliberately narrower than an explicit
+ * REQUIRE: it applies only after the complete selected-address domain is
+ * materially available.  This keeps byte/automatic cache budgets that cannot
+ * retain every expert on the established sparse-MV fallback.  ENABLE=0 and
+ * DISABLE=1 are rollback controls; REQUIRE=0 retains automatic selection but
+ * permits fallback, while explicit REQUIRE=1 remains strong. */
+static int ds4_gpu_iq2_stream_addr_mm_resolve_policy(
+        int  enable,
+        int  require,
+        int  disable,
+        int  material_ready,
+        int *requested_out,
+        int *required_out) {
+    if (!requested_out || !required_out) return 0;
+
+    const int explicitly_required = require == 1;
+    const int disabled = disable == 1;
+    if (disabled) {
+        *requested_out = 0;
+        *required_out = explicitly_required;
+        return 1;
+    }
+
+    const int requested = explicitly_required || enable != 0;
+    const int implicitly_required =
+        requested && require < 0 && material_ready != 0;
+    *requested_out = requested;
+    *required_out = explicitly_required || implicitly_required;
+    return 1;
+}
+
+/* Standalone policy hook: values use ds4_gpu_env_bool's -1/0/1 convention. */
+int ds4_gpu_test_iq2_stream_addr_mm_policy(
+        int  enable,
+        int  require,
+        int  disable,
+        int  material_ready,
+        int *requested_out,
+        int *required_out) {
+    return ds4_gpu_iq2_stream_addr_mm_resolve_policy(enable,
+                                                      require,
+                                                      disable,
+                                                      material_ready,
+                                                      requested_out,
+                                                      required_out);
+}
+
+static void ds4_gpu_iq2_stream_addr_mm_stats_reset(void) {
+    g_iq2_stream_addr_mm_candidate_calls = 0;
+    g_iq2_stream_addr_mm_calls = 0;
+    g_iq2_stream_addr_mm_tokens = 0;
+    g_iq2_stream_addr_mm_rows = 0;
+    g_iq2_stream_addr_mm_require_failures = 0;
+    g_iq2_stream_addr_mm_min_tokens = 0;
+    g_iq2_stream_addr_mm_max_tokens = 0;
+}
+
+static void ds4_gpu_iq2_stream_addr_mm_stats_note_candidate(void) {
+    if (g_iq2_stream_addr_mm_stats_enabled) {
+        g_iq2_stream_addr_mm_candidate_calls++;
+    }
+}
+
+static void ds4_gpu_iq2_stream_addr_mm_stats_note_selected(
+        uint32_t n_tokens,
+        uint32_t n_expert) {
+    if (!g_iq2_stream_addr_mm_stats_enabled) return;
+
+    if (g_iq2_stream_addr_mm_calls == 0 ||
+        n_tokens < g_iq2_stream_addr_mm_min_tokens) {
+        g_iq2_stream_addr_mm_min_tokens = n_tokens;
+    }
+    if (n_tokens > g_iq2_stream_addr_mm_max_tokens) {
+        g_iq2_stream_addr_mm_max_tokens = n_tokens;
+    }
+    g_iq2_stream_addr_mm_calls++;
+    g_iq2_stream_addr_mm_tokens += n_tokens;
+    g_iq2_stream_addr_mm_rows += (uint64_t)n_tokens * n_expert;
+}
+
+static void ds4_gpu_iq2_stream_addr_mm_stats_note_require_failure(void) {
+    if (g_iq2_stream_addr_mm_stats_enabled) {
+        g_iq2_stream_addr_mm_require_failures++;
+    }
+}
+
+static void ds4_gpu_iq2_stream_addr_mm_stats_print(void) {
+    if (!g_iq2_stream_addr_mm_stats_enabled) return;
+
+    fprintf(stderr,
+            "ds4: Metal IQ2_XXS SSD prefill MM stats candidates=%llu "
+            "calls=%llu tokens=%llu rows=%llu min_tokens=%u max_tokens=%u "
+            "require_failures=%llu\n",
+            (unsigned long long)g_iq2_stream_addr_mm_candidate_calls,
+            (unsigned long long)g_iq2_stream_addr_mm_calls,
+            (unsigned long long)g_iq2_stream_addr_mm_tokens,
+            (unsigned long long)g_iq2_stream_addr_mm_rows,
+            g_iq2_stream_addr_mm_min_tokens,
+            g_iq2_stream_addr_mm_max_tokens,
+            (unsigned long long)g_iq2_stream_addr_mm_require_failures);
+}
+
+/* Standalone Metal oracle hook. Production code only updates the counters;
+ * the dedicated test reads them without changing path selection. */
+int ds4_gpu_test_iq2_stream_addr_mm_stats(
+        uint64_t *candidate_calls,
+        uint64_t *calls,
+        uint64_t *tokens,
+        uint64_t *rows,
+        uint64_t *require_failures,
+        uint32_t *min_tokens,
+        uint32_t *max_tokens) {
+    if (candidate_calls) *candidate_calls = g_iq2_stream_addr_mm_candidate_calls;
+    if (calls) *calls = g_iq2_stream_addr_mm_calls;
+    if (tokens) *tokens = g_iq2_stream_addr_mm_tokens;
+    if (rows) *rows = g_iq2_stream_addr_mm_rows;
+    if (require_failures) *require_failures = g_iq2_stream_addr_mm_require_failures;
+    if (min_tokens) *min_tokens = g_iq2_stream_addr_mm_min_tokens;
+    if (max_tokens) *max_tokens = g_iq2_stream_addr_mm_max_tokens;
+    return 1;
+}
+
 static uint64_t ds4_gpu_env_u64(const char *name,
                                 uint64_t    fallback,
                                 uint64_t    min_value,
@@ -2769,6 +3600,12 @@ int ds4_gpu_device_is_pre_m5_apple_silicon(void) {
             g_metal_device_name[8] == ' ');
 }
 
+static int ds4_gpu_device_is_m1_apple_silicon(void) {
+    return strncmp(g_metal_device_name, "Apple M1", 8) == 0 &&
+           (g_metal_device_name[8] == '\0' ||
+            g_metal_device_name[8] == ' ');
+}
+
 int ds4_gpu_device_is_m5_apple_silicon(void) {
     return strncmp(g_metal_device_name, "Apple M5", 8) == 0 &&
            (g_metal_device_name[8] == '\0' ||
@@ -3467,11 +4304,14 @@ int ds4_gpu_test_decode_pipeline_fast_lookup_ext(void) {
     static struct {
         bool m;
         int32_t nc;
+        uint64_t generation;
         id<MTLComputePipelineState> pipeline;
     } memo;
     const bool memo_disabled =
         getenv("DS4_METAL_DISABLE_PRE_M5_FLASH_ATTN_PAD_BLK_MEMO") != NULL;
-    if (!memo_disabled && memo.pipeline && memo.m == has_mask && memo.nc == ncpsg) {
+    if (!memo_disabled && memo.pipeline &&
+        memo.generation == g_pipeline_cache_generation &&
+        memo.m == has_mask && memo.nc == ncpsg) {
         return memo.pipeline;
     }
 
@@ -3480,7 +4320,9 @@ int ds4_gpu_test_decode_pipeline_fast_lookup_ext(void) {
     id<MTLComputePipelineState> cached = [g_pipeline_cache objectForKey:key];
     if (cached) {
         if (!memo_disabled) {
-            memo = (typeof(memo)){ has_mask, ncpsg, cached };
+            memo = (typeof(memo)){
+                has_mask, ncpsg, g_pipeline_cache_generation, cached
+            };
         }
         return cached;
     }
@@ -3509,7 +4351,9 @@ int ds4_gpu_test_decode_pipeline_fast_lookup_ext(void) {
 
     [g_pipeline_cache setObject:pipeline forKey:key];
     if (!memo_disabled) {
-        memo = (typeof(memo)){ has_mask, ncpsg, pipeline };
+        memo = (typeof(memo)){
+            has_mask, ncpsg, g_pipeline_cache_generation, pipeline
+        };
     }
     return pipeline;
 }
@@ -3526,11 +4370,14 @@ int ds4_gpu_test_decode_pipeline_fast_lookup_ext(void) {
     static struct {
         int32_t nq;
         int32_t nc;
+        uint64_t generation;
         id<MTLComputePipelineState> pipeline;
     } memo;
     const bool memo_disabled =
         getenv("DS4_METAL_DISABLE_PRE_M5_FLASH_ATTN_PAD_BLK_MEMO") != NULL;
-    if (!memo_disabled && memo.pipeline && memo.nq == nqptg && memo.nc == ncpsg) {
+    if (!memo_disabled && memo.pipeline &&
+        memo.generation == g_pipeline_cache_generation &&
+        memo.nq == nqptg && memo.nc == ncpsg) {
         return memo.pipeline;
     }
 
@@ -3539,7 +4386,9 @@ int ds4_gpu_test_decode_pipeline_fast_lookup_ext(void) {
     id<MTLComputePipelineState> cached = [g_pipeline_cache objectForKey:key];
     if (cached) {
         if (!memo_disabled) {
-            memo = (typeof(memo)){ nqptg, ncpsg, cached };
+            memo = (typeof(memo)){
+                nqptg, ncpsg, g_pipeline_cache_generation, cached
+            };
         }
         return cached;
     }
@@ -3568,7 +4417,9 @@ int ds4_gpu_test_decode_pipeline_fast_lookup_ext(void) {
 
     [g_pipeline_cache setObject:pipeline forKey:key];
     if (!memo_disabled) {
-        memo = (typeof(memo)){ nqptg, ncpsg, pipeline };
+        memo = (typeof(memo)){
+            nqptg, ncpsg, g_pipeline_cache_generation, pipeline
+        };
     }
     return pipeline;
 }
@@ -3595,11 +4446,13 @@ int ds4_gpu_test_decode_pipeline_fast_lookup_ext(void) {
         const char *fn;
         bool m, s, b, c, k, bc;
         int32_t n10, n20, sg;
+        uint64_t generation;
         id<MTLComputePipelineState> pipeline;
     } memo;
     const bool memo_disabled =
         getenv("DS4_METAL_DISABLE_PRE_M5_FLASH_ATTN_BATCHED_MEMO") != NULL;
-    if (!memo_disabled && memo.pipeline && memo.fn != NULL &&
+    if (!memo_disabled && memo.pipeline &&
+        memo.generation == g_pipeline_cache_generation && memo.fn != NULL &&
         strcmp(memo.fn, function_name) == 0 &&
         memo.m == has_mask && memo.s == has_sinks && memo.b == has_bias &&
         memo.c == has_scap && memo.k == has_kvpad && memo.bc == bc_mask &&
@@ -3623,7 +4476,7 @@ int ds4_gpu_test_decode_pipeline_fast_lookup_ext(void) {
         if (!memo_disabled) {
             memo = (typeof(memo)){ function_name, has_mask, has_sinks, has_bias,
                                    has_scap, has_kvpad, bc_mask, ns10, ns20,
-                                   nsg, cached };
+                                   nsg, g_pipeline_cache_generation, cached };
         }
         return cached;
     }
@@ -3662,7 +4515,7 @@ int ds4_gpu_test_decode_pipeline_fast_lookup_ext(void) {
     if (!memo_disabled) {
         memo = (typeof(memo)){ function_name, has_mask, has_sinks, has_bias,
                                has_scap, has_kvpad, bc_mask, ns10, ns20,
-                               nsg, pipeline };
+                               nsg, g_pipeline_cache_generation, pipeline };
     }
     return pipeline;
 }
@@ -3688,9 +4541,14 @@ int ds4_gpu_test_decode_pipeline_fast_lookup_ext(void) {
         const char *fn;
         bool m, s, b, c, k, sp;
         int32_t n10, n20, sg, wg;
+        uint64_t generation;
         id<MTLComputePipelineState> pipeline;
     } memo;
-    if (memo.pipeline && memo.fn != NULL && strcmp(memo.fn, function_name) == 0 &&
+    const bool memo_disabled =
+        getenv("DS4_METAL_DISABLE_PRE_M5_FLASH_ATTN_BATCHED_MEMO") != NULL;
+    if (!memo_disabled && memo.pipeline &&
+        memo.generation == g_pipeline_cache_generation && memo.fn != NULL &&
+        strcmp(memo.fn, function_name) == 0 &&
         memo.m == has_mask && memo.s == has_sinks && memo.b == has_bias &&
         memo.c == has_scap && memo.k == has_kvpad && memo.sp == shared_kvpad &&
         memo.n10 == ns10 && memo.n20 == ns20 && memo.sg == nsg && memo.wg == nwg) {
@@ -3711,9 +4569,13 @@ int ds4_gpu_test_decode_pipeline_fast_lookup_ext(void) {
                      (int)nwg];
     id<MTLComputePipelineState> cached = [g_pipeline_cache objectForKey:key];
     if (cached) {
-        memo = (typeof(memo)){ function_name, has_mask, has_sinks, has_bias,
-                               has_scap, has_kvpad, shared_kvpad, ns10, ns20,
-                               nsg, nwg, cached };
+        if (!memo_disabled) {
+            memo = (typeof(memo)){
+                function_name, has_mask, has_sinks, has_bias, has_scap,
+                has_kvpad, shared_kvpad, ns10, ns20, nsg, nwg,
+                g_pipeline_cache_generation, cached
+            };
+        }
         return cached;
     }
 
@@ -3749,9 +4611,13 @@ int ds4_gpu_test_decode_pipeline_fast_lookup_ext(void) {
     }
 
     [g_pipeline_cache setObject:pipeline forKey:key];
-    memo = (typeof(memo)){ function_name, has_mask, has_sinks, has_bias,
-                           has_scap, has_kvpad, shared_kvpad, ns10, ns20,
-                           nsg, nwg, pipeline };
+    if (!memo_disabled) {
+        memo = (typeof(memo)){
+            function_name, has_mask, has_sinks, has_bias, has_scap,
+            has_kvpad, shared_kvpad, ns10, ns20, nsg, nwg,
+            g_pipeline_cache_generation, pipeline
+        };
+    }
     return pipeline;
 }
 
@@ -3809,17 +4675,28 @@ int ds4_gpu_decode_attn_rope_fuse_available(void) {
         int32_t dv,
         int32_t nwg) {
     /* Same per-layer memo pattern as the vec getter above. */
-    static int32_t memo_dv, memo_nwg;
-    static id<MTLComputePipelineState> memo_pipeline;
-    if (memo_pipeline && memo_dv == dv && memo_nwg == nwg) {
-        return memo_pipeline;
+    static struct {
+        int32_t dv, nwg;
+        uint64_t generation;
+        id<MTLComputePipelineState> pipeline;
+    } memo;
+    const bool memo_disabled =
+        getenv("DS4_METAL_DISABLE_PRE_M5_FLASH_ATTN_BATCHED_MEMO") != NULL;
+    if (!memo_disabled && memo.pipeline &&
+        memo.generation == g_pipeline_cache_generation &&
+        memo.dv == dv && memo.nwg == nwg) {
+        return memo.pipeline;
     }
 
     NSString *key = [NSString stringWithFormat:@"kernel_flash_attn_ext_vec_reduce_dv=%d_nwg=%d",
                      (int)dv, (int)nwg];
     id<MTLComputePipelineState> cached = [g_pipeline_cache objectForKey:key];
     if (cached) {
-        memo_dv = dv; memo_nwg = nwg; memo_pipeline = cached;
+        if (!memo_disabled) {
+            memo = (typeof(memo)){
+                dv, nwg, g_pipeline_cache_generation, cached
+            };
+        }
         return cached;
     }
 
@@ -3846,7 +4723,11 @@ int ds4_gpu_decode_attn_rope_fuse_available(void) {
     }
 
     [g_pipeline_cache setObject:pipeline forKey:key];
-    memo_dv = dv; memo_nwg = nwg; memo_pipeline = pipeline;
+    if (!memo_disabled) {
+        memo = (typeof(memo)){
+            dv, nwg, g_pipeline_cache_generation, pipeline
+        };
+    }
     return pipeline;
 }
 
@@ -3858,6 +4739,12 @@ static uint32_t ds4_gpu_flash_attn_vec_nsg(uint32_t n_keys, uint32_t nwg, uint32
     return nsg;
 }
 
+static bool ds4_gpu_flash_attn_small_prefill_direct(uint32_t n_keys) {
+    return n_keys <= 32u &&
+        getenv("DS4_METAL_DISABLE_SMALL_PREFILL_DIRECT") == NULL &&
+        (g_test_flags & DS4_GPU_TEST_FLASH_ATTN_SMALL_PREFILL_NWG32) == 0u;
+}
+
 static int ds4_gpu_trace_allocs(void) {
     static int initialized;
     static int enabled;
@@ -4266,43 +5153,66 @@ void ds4_gpu_print_memory_report(const char *label) {
         if (entry->mask) cached_prefill_mask_bytes += entry->mask_bytes;
         if (entry->blk) cached_prefill_blk_bytes += entry->blk_bytes;
     }
-    const uint64_t scratch =
-        (uint64_t)g_flash_attn_mask_bytes +
-        (uint64_t)g_flash_attn_zero_mask_bytes +
-        cached_prefill_mask_bytes +
-        (uint64_t)g_flash_attn_pad_bytes +
-        (uint64_t)g_flash_attn_tmp_bytes +
-        (uint64_t)g_flash_attn_blk_bytes +
-        cached_prefill_blk_bytes +
-        (uint64_t)g_flash_attn_ring_bytes +
-        (uint64_t)g_flash_attn_kv_bytes +
-        (uint64_t)g_glm_flash_attn_mask_bytes +
-        (uint64_t)g_compressor_pool_kv_bytes +
-        (uint64_t)g_compressor_pool_score_bytes +
-        (uint64_t)g_compressor_pool_score_cont_bytes +
-        (uint64_t)g_compressor_pool_softmax_bytes +
-        (uint64_t)g_compressor_pool_product_bytes +
-        (uint64_t)g_compressor_store_ape_bytes +
-        (uint64_t)g_compressor_store_score_bytes +
-        (uint64_t)g_embed_rows_bytes +
-        (uint64_t)g_router_selection_bytes +
-        (uint64_t)g_router_weight_sum_bytes +
-        (uint64_t)g_indexer_head_scores_bytes +
-        (uint64_t)g_indexer_topk_bytes +
-        (uint64_t)g_indexed_topk_bytes +
-        (uint64_t)g_f16_round_scratch_bytes +
-        (uint64_t)g_raw_store_round_bytes +
-        (uint64_t)g_moe_gate_scratch_bytes +
-        (uint64_t)g_moe_down_scratch_bytes +
-        (uint64_t)g_moe_id_map_bytes +
-        (uint64_t)g_moe_q4_gate_slots_bytes +
-        (uint64_t)g_moe_q4_up_slots_bytes +
-        (uint64_t)g_moe_q4_down_slots_bytes;
+    uint64_t flash_mask = cached_prefill_mask_bytes;
+    uint64_t flash_pad = 0;
+    uint64_t flash_tmp = 0;
+    uint64_t flash_blk = cached_prefill_blk_bytes;
+    uint64_t flash_kv = 0;
+    uint64_t compressor = 0;
+    uint64_t router = 0;
+    uint64_t indexer = 0;
+    uint64_t moe = 0;
+    uint64_t q4_pair_rhs = 0;
+    uint64_t f16_round = 0;
+    uint64_t raw_store = 0;
+    for (int si = 0; si < DS4_GPU_MAX_STREAMS; si++) {
+        const ds4_gpu_stream_scratch_state *scratch_state =
+            &g_stream_scratch[si];
+        flash_mask += (uint64_t)scratch_state->flash_attn_mask_bytes +
+                      (uint64_t)scratch_state->glm_flash_attn_mask_bytes +
+                      (uint64_t)scratch_state->flash_attn_zero_mask_bytes;
+        flash_pad += (uint64_t)scratch_state->flash_attn_pad_bytes;
+        flash_tmp += (uint64_t)scratch_state->flash_attn_tmp_bytes;
+        flash_blk += (uint64_t)scratch_state->flash_attn_blk_bytes;
+        flash_kv += (uint64_t)scratch_state->flash_attn_kv_bytes;
+        compressor += (uint64_t)scratch_state->compressor_pool_kv_bytes +
+                      (uint64_t)scratch_state->compressor_pool_score_bytes +
+                      (uint64_t)scratch_state->compressor_pool_score_cont_bytes +
+                      (uint64_t)scratch_state->compressor_pool_softmax_bytes +
+                      (uint64_t)scratch_state->compressor_pool_product_bytes +
+                      (uint64_t)scratch_state->compressor_store_ape_bytes +
+                      (uint64_t)scratch_state->compressor_store_score_bytes +
+                      (uint64_t)scratch_state->embed_rows_bytes;
+        router += (uint64_t)scratch_state->router_selection_bytes +
+                  (uint64_t)scratch_state->router_weight_sum_bytes +
+                  (uint64_t)scratch_state->attn_out_group_ids_bytes;
+        indexer += (uint64_t)scratch_state->indexer_head_scores_bytes +
+                   (uint64_t)scratch_state->indexer_topk_bytes +
+                   (uint64_t)scratch_state->indexed_topk_bytes;
+        moe += (uint64_t)scratch_state->moe_gate_scratch_bytes +
+               (uint64_t)scratch_state->moe_down_scratch_bytes +
+               (uint64_t)scratch_state->moe_id_map_bytes +
+               (uint64_t)scratch_state->moe_q4_gate_slots_bytes +
+               (uint64_t)scratch_state->moe_q4_up_slots_bytes +
+               (uint64_t)scratch_state->moe_q4_down_slots_bytes;
+        q4_pair_rhs +=
+            (uint64_t)scratch_state->q4_pair_rhs_f16_bytes;
+        f16_round += (uint64_t)scratch_state->f16_round_scratch_bytes;
+        raw_store += (uint64_t)scratch_state->raw_store_round_bytes;
+    }
+    const uint64_t scratch = flash_mask + flash_pad + flash_tmp + flash_blk +
+                             flash_kv + compressor + router +
+                             indexer + moe + q4_pair_rhs + f16_round +
+                             raw_store;
 
     pthread_mutex_lock(&g_tensor_mu);
     const uint64_t tensor_live_snap = g_tensor_alloc_live_bytes;
     const uint64_t tensor_peak_snap = g_tensor_alloc_peak_bytes;
     pthread_mutex_unlock(&g_tensor_mu);
+    pthread_mutex_lock(&g_q4_attn_q_b_f16_cache_mu);
+    const uint64_t q4_q_b_f16_cache_snap =
+        g_q4_attn_q_b_f16_cache_bytes;
+    pthread_mutex_unlock(&g_q4_attn_q_b_f16_cache_mu);
 
     uint64_t tracked_live = tensor_live_snap;
     if (tracked_live > UINT64_MAX - g_stream_expert_cache_bytes) {
@@ -4310,18 +5220,25 @@ void ds4_gpu_print_memory_report(const char *label) {
     } else {
         tracked_live += g_stream_expert_cache_bytes;
     }
+    if (tracked_live > UINT64_MAX - q4_q_b_f16_cache_snap) {
+        tracked_live = UINT64_MAX;
+    } else {
+        tracked_live += q4_q_b_f16_cache_snap;
+    }
 
     const bool color = ds4_log_is_tty(stderr);
     const char *green = color ? "\x1b[32m" : "";
     const char *bright_green = color ? "\x1b[1;32m" : "";
     const char *reset = color ? "\x1b[0m" : "";
     fprintf(stderr,
-            "%sds4: Metal memory%s%s: runtime %.2f GiB + streaming experts %.2f GiB = %s%.2f GiB tracked live%s\n",
+            "%sds4: Metal memory%s%s: runtime %.2f GiB + streaming experts %.2f GiB "
+            "+ Q4 attn_q_b F16 %.2f GiB = %s%.2f GiB tracked live%s\n",
             green,
             label && label[0] ? " " : "",
             label && label[0] ? label : "",
             ds4_gpu_gib(tensor_live_snap),
             ds4_gpu_gib(g_stream_expert_cache_bytes),
+            ds4_gpu_gib(q4_q_b_f16_cache_snap),
             bright_green,
             ds4_gpu_gib(tracked_live),
             reset);
@@ -4388,6 +5305,22 @@ void ds4_gpu_print_memory_report(const char *label) {
                     ds4_gpu_gib(g_stream_expert_cache_willneed_advise_bytes),
                     ds4_gpu_gib(g_stream_expert_cache_pread_bytes),
                     g_stream_expert_cache_pread_ms);
+            fprintf(stderr,
+                    "ds4:   streaming expert pread descriptors "
+                    "cached_calls=%llu cached=%.2f GiB "
+                    "nocache_calls=%llu nocache=%.2f GiB\n",
+                    (unsigned long long)__atomic_load_n(
+                        &g_stream_expert_pread_cached_calls,
+                        __ATOMIC_RELAXED),
+                    ds4_gpu_gib(__atomic_load_n(
+                        &g_stream_expert_pread_cached_bytes,
+                        __ATOMIC_RELAXED)),
+                    (unsigned long long)__atomic_load_n(
+                        &g_stream_expert_pread_nocache_calls,
+                        __ATOMIC_RELAXED),
+                    ds4_gpu_gib(__atomic_load_n(
+                        &g_stream_expert_pread_nocache_bytes,
+                        __ATOMIC_RELAXED)));
         } else {
             fprintf(stderr,
                     "ds4:   streaming expert cache budget=%llu experts entries=%u expert=%.2f MiB target=%.2f GiB live=%.2f GiB, hits=%llu misses=%llu hit_rate=%.3f wraps=%llu evictions=%llu buffer_allocs=%llu buffer_reuses=%llu\n",
@@ -4520,6 +5453,23 @@ void ds4_gpu_print_memory_report(const char *label) {
             }
         }
     }
+    if (g_stream_expert_exact_persistent_calls != 0 ||
+        g_stream_expert_exact_transient_calls != 0 ||
+        g_stream_expert_exact_persistent_fallbacks != 0 ||
+        g_stream_expert_exact_persistent_failures != 0 ||
+        g_stream_expert_exact_mapped_view_calls != 0 ||
+        g_stream_expert_exact_max_unique != 0) {
+        fprintf(stderr,
+                "ds4:   exact-row cache persistent=%llu transient=%llu "
+                "fallbacks=%llu failures=%llu mapped_views=%llu "
+                "max_unique=%u\n",
+                (unsigned long long)g_stream_expert_exact_persistent_calls,
+                (unsigned long long)g_stream_expert_exact_transient_calls,
+                (unsigned long long)g_stream_expert_exact_persistent_fallbacks,
+                (unsigned long long)g_stream_expert_exact_persistent_failures,
+                (unsigned long long)g_stream_expert_exact_mapped_view_calls,
+                g_stream_expert_exact_max_unique);
+    }
     fprintf(stderr,
             "ds4:   model residency requests %llu%s\n",
             (unsigned long long)g_model_residency_count,
@@ -4549,39 +5499,20 @@ void ds4_gpu_print_memory_report(const char *label) {
                 (g_metal4_tensor_api_compile_supported ? "available" : "disabled"),
             g_metal4_m5_neural_accelerators_hint ? "likely" : "not detected");
     fprintf(stderr,
-            "ds4:   scratch %.2f MiB (flash mask %.2f, pad %.2f, tmp %.2f, blk %.2f, ring %.2f, kv %.2f, compressor %.2f, router %.2f, indexer %.2f, moe %.2f, f16 %.2f, raw-store %.2f)\n",
+            "ds4:   scratch %.2f MiB (flash mask %.2f, pad %.2f, tmp %.2f, blk %.2f, kv %.2f, compressor %.2f, router %.2f, indexer %.2f, moe %.2f, q4-pair-rhs %.2f, f16 %.2f, raw-store %.2f)\n",
             ds4_gpu_mib(scratch),
-            ds4_gpu_mib((uint64_t)g_flash_attn_mask_bytes +
-                        (uint64_t)g_glm_flash_attn_mask_bytes +
-                        (uint64_t)g_flash_attn_zero_mask_bytes +
-                        cached_prefill_mask_bytes),
-            ds4_gpu_mib((uint64_t)g_flash_attn_pad_bytes),
-            ds4_gpu_mib((uint64_t)g_flash_attn_tmp_bytes),
-            ds4_gpu_mib((uint64_t)g_flash_attn_blk_bytes +
-                        cached_prefill_blk_bytes),
-            ds4_gpu_mib((uint64_t)g_flash_attn_ring_bytes),
-            ds4_gpu_mib((uint64_t)g_flash_attn_kv_bytes),
-            ds4_gpu_mib((uint64_t)g_compressor_pool_kv_bytes +
-                          (uint64_t)g_compressor_pool_score_bytes +
-                          (uint64_t)g_compressor_pool_score_cont_bytes +
-                          (uint64_t)g_compressor_pool_softmax_bytes +
-                          (uint64_t)g_compressor_pool_product_bytes +
-                          (uint64_t)g_compressor_store_ape_bytes +
-                          (uint64_t)g_compressor_store_score_bytes +
-                          (uint64_t)g_embed_rows_bytes),
-            ds4_gpu_mib((uint64_t)g_router_selection_bytes +
-                          (uint64_t)g_router_weight_sum_bytes),
-            ds4_gpu_mib((uint64_t)g_indexer_head_scores_bytes +
-                          (uint64_t)g_indexer_topk_bytes +
-                          (uint64_t)g_indexed_topk_bytes),
-            ds4_gpu_mib((uint64_t)g_moe_gate_scratch_bytes +
-                          (uint64_t)g_moe_down_scratch_bytes +
-                          (uint64_t)g_moe_id_map_bytes +
-                          (uint64_t)g_moe_q4_gate_slots_bytes +
-                          (uint64_t)g_moe_q4_up_slots_bytes +
-                          (uint64_t)g_moe_q4_down_slots_bytes),
-            ds4_gpu_mib((uint64_t)g_f16_round_scratch_bytes),
-            ds4_gpu_mib((uint64_t)g_raw_store_round_bytes));
+            ds4_gpu_mib(flash_mask),
+            ds4_gpu_mib(flash_pad),
+            ds4_gpu_mib(flash_tmp),
+            ds4_gpu_mib(flash_blk),
+            ds4_gpu_mib(flash_kv),
+            ds4_gpu_mib(compressor),
+            ds4_gpu_mib(router),
+            ds4_gpu_mib(indexer),
+            ds4_gpu_mib(moe),
+            ds4_gpu_mib(q4_pair_rhs),
+            ds4_gpu_mib(f16_round),
+            ds4_gpu_mib(raw_store));
     if (color) fputs(reset, stderr);
 }
 
@@ -4594,9 +5525,37 @@ void ds4_gpu_set_glm_model(bool enabled) {
 }
 
 void ds4_gpu_set_ssd_streaming(bool enabled) {
+    const int was_ssd_streaming = g_ssd_streaming_mode;
     g_ssd_streaming_mode = enabled ? 1 : 0;
-    ds4_gpu_stream_expert_cache_clear_all(1);
-    if (g_ssd_streaming_mode) {
+    if (g_ssd_streaming_mode != was_ssd_streaming) {
+        ds4_gpu_q4_qb_transient_f16_admission_clear();
+    }
+    /* A real resident->streaming transition must re-evaluate sidecar
+     * admission against the streaming reserve. Reasserting an already active
+     * SSD mode is configuration plumbing, not a lifecycle boundary, and must
+     * not evict a successfully admitted hybrid cache. */
+    if (g_ssd_streaming_mode &&
+        (!was_ssd_streaming ||
+         __atomic_load_n(&g_q4_attn_q_b_f16_ssd_admission_blocked,
+                         __ATOMIC_ACQUIRE))) {
+        __atomic_store_n(&g_q4_attn_q_b_f16_ssd_admission_blocked,
+                         1,
+                         __ATOMIC_RELEASE);
+        if (!ds4_gpu_release_q4_attn_q_b_f16_sidecars()) {
+            fprintf(stderr,
+                    "ds4: WARNING: could not release resident Q4 attn_q_b "
+                    "F16 sidecars while enabling SSD streaming; the hybrid "
+                    "cache remains blocked\n");
+        }
+    } else if (!g_ssd_streaming_mode) {
+        __atomic_store_n(&g_q4_attn_q_b_f16_ssd_admission_blocked,
+                         0,
+                         __ATOMIC_RELEASE);
+    }
+    if (g_ssd_streaming_mode != was_ssd_streaming) {
+        ds4_gpu_stream_expert_cache_clear_all(1);
+    }
+    if (g_ssd_streaming_mode && !was_ssd_streaming) {
         fprintf(stderr,
                 "ds4: Metal SSD streaming mode enabled; full model residency and warmup are skipped\n");
     }
@@ -4622,6 +5581,12 @@ void ds4_gpu_set_streaming_expert_cache_expert_bytes(uint64_t bytes) {
      * deterministically from startup, instead of depending on which layer
      * happens to touch the cache first.
      */
+    if (bytes != g_stream_expert_cache_expert_bytes) {
+        ds4_gpu_stream_expert_cache_live_release();
+        g_stream_expert_cache_gate_class_bytes = 0;
+        g_stream_expert_cache_down_class_bytes = 0;
+        g_stream_expert_cache_class_conflict = 0;
+    }
     g_stream_expert_cache_expert_bytes = bytes;
 }
 
@@ -4671,7 +5636,6 @@ static int ds4_gpu_model_map_log_enabled(void) {
 "#endif\n"
 "#define N_SIMDWIDTH 32\n"
 "#define N_R0_Q8_0 2\n"
-"#define N_SG_Q8_0 4\n"
 "#define FC_MUL_MV 600\n"
 "#define FC_MUL_MM 700\n"
 "#define FC_BIN 1300\n"
@@ -4735,9 +5699,11 @@ static int ds4_gpu_model_map_log_enabled(void) {
     /*
      * Kernels are kept as separate files for review, then concatenated into one
      * Metal library.  Environment overrides are still honored so a diagnostic
-     * run can swap one source file without changing the executable.
+     * run can swap one source file without changing the executable. A one-item
+     * entry is a fixed shared source; two-item entries are [override, source].
      */
     NSArray<NSArray<NSString *> *> *required_sources = @[
+        @[@"metal/activations.metal"],
         @[@"DS4_METAL_FLASH_ATTN_SOURCE", @"metal/flash_attn.metal"],
         @[@"DS4_METAL_DENSE_SOURCE",      @"metal/dense.metal"],
         @[@"DS4_METAL_GLM53_BF16_SOURCE", @"metal/glm53_bf16.metal"],
@@ -4765,13 +5731,16 @@ static int ds4_gpu_model_map_log_enabled(void) {
 
     NSMutableString *source = [NSMutableString stringWithString:base];
     for (NSArray<NSString *> *spec in required_sources) {
-        const char *override_path = getenv([spec[0] UTF8String]);
+        NSString *override_name = spec.count > 1 ? spec[0] : nil;
+        NSString *source_path = spec.lastObject;
+        const char *override_path =
+            override_name ? getenv([override_name UTF8String]) : NULL;
         NSMutableArray<NSString *> *paths = [NSMutableArray array];
         if (override_path && override_path[0]) {
             [paths addObject:[NSString stringWithUTF8String:override_path]];
         }
-        [paths addObject:spec[1]];
-        [paths addObject:[@"./" stringByAppendingString:spec[1]]];
+        [paths addObject:source_path];
+        [paths addObject:[@"./" stringByAppendingString:source_path]];
 
         NSString *loaded = nil;
         NSString *loaded_path = nil;
@@ -4792,9 +5761,14 @@ static int ds4_gpu_model_map_log_enabled(void) {
         }
 
         if (!loaded) {
-            fprintf(stderr,
-                    "ds4: Metal source %s not found (set %s to override)\n",
-                    [spec[1] UTF8String], [spec[0] UTF8String]);
+            if (override_name) {
+                fprintf(stderr,
+                        "ds4: Metal source %s not found (set %s to override)\n",
+                        [source_path UTF8String], [override_name UTF8String]);
+            } else {
+                fprintf(stderr, "ds4: Metal source %s not found\n",
+                        [source_path UTF8String]);
+            }
             return nil;
         }
         [source appendFormat:@"\n// appended %@\n%@\n", loaded_path, loaded];
@@ -4990,6 +5964,12 @@ static int ds4_gpu_encode_cpy_f32_f16_1d(
         NSUInteger           dst_off,
         uint32_t             n);
 
+static bool ds4_gpu_tensor_prefixes_overlap(
+        const ds4_gpu_tensor *a,
+        uint64_t              a_bytes,
+        const ds4_gpu_tensor *b,
+        uint64_t              b_bytes);
+
 static int ds4_gpu_encode_cpy_f32_f16_2d(
         id<MTLCommandBuffer> cb,
         id<MTLBuffer>        src,
@@ -5239,6 +6219,31 @@ static int ds4_gpu_encode_sum_rows_f32(
     int16_t  r3;
 } ds4_gpu_q8_0_matvec_args;
 
+typedef struct {
+    uint32_t in_dim;
+    uint32_t out_rows;
+    uint32_t n_groups;
+    uint32_t n_tokens;
+    uint64_t weight_row_bytes;
+    uint64_t weight_group_bytes;
+    uint64_t input_group_bytes;
+    uint64_t input_token_bytes;
+    uint64_t output_group_bytes;
+    uint64_t output_token_bytes;
+} ds4_gpu_q4_attn_exactn_args;
+
+typedef char ds4_gpu_q4_attn_exactn_args_must_be_64_bytes[
+    sizeof(ds4_gpu_q4_attn_exactn_args) == 64 ? 1 : -1];
+
+static bool ds4_gpu_u64_mul_checked(
+        uint64_t a,
+        uint64_t b,
+        uint64_t *result) {
+    if (!result || (a != 0 && b > UINT64_MAX / a)) return false;
+    *result = a * b;
+    return true;
+}
+
 typedef struct {
     int32_t  ne00;
     int32_t  ne02;
@@ -5796,6 +6801,17 @@ static int ds4_gpu_encode_attn_out_low_mpp(
         id<MTLBuffer>                  dst,
         NSUInteger                     dst_off);
 
+static int ds4_gpu_encode_attn_out_low_q4_direct(
+        id<MTLCommandBuffer>           cb,
+        id<MTLComputePipelineState>    pipeline,
+        const ds4_gpu_mul_mm_id_args *mm_args,
+        id<MTLBuffer>                  src0,
+        NSUInteger                     src0_off,
+        id<MTLBuffer>                  src1,
+        NSUInteger                     src1_off,
+        id<MTLBuffer>                  dst,
+        NSUInteger                     dst_off);
+
 static ds4_gpu_mul_mm_id_map_args ds4_gpu_make_mul_mm_id_map_args(
         uint32_t src0_cols,
         uint32_t src0_experts,
@@ -6335,6 +7351,15 @@ static int ds4_gpu_encode_rope_tail_inplace(
     int32_t  len;
 } ds4_gpu_kargs_argsort_merge;
 
+/* Matches ds4_metal_args_dsv4_argmax_top1 in metal/argsort.metal. */
+typedef struct {
+    int32_t n_vocab;
+    int32_t n_tg;
+} ds4_gpu_dsv4_argmax_top1_args;
+
+_Static_assert(sizeof(ds4_gpu_dsv4_argmax_top1_args) == 8u,
+               "Metal top-1 argmax argument ABI changed");
+
 typedef struct {
     int64_t  ne00;
     int64_t  ne01;
@@ -6786,6 +7811,10 @@ static int ds4_gpu_encode_rope_tail_inplace(
 int ds4_gpu_init(void) {
     if (g_initialized) return 1;
 
+    ds4_gpu_iq2_stream_addr_mm_stats_reset();
+    g_iq2_stream_addr_mm_stats_enabled =
+        ds4_gpu_env_bool("DS4_METAL_IQ2_XXS_SSD_PREFILL_MM_STATS") > 0;
+
     @autoreleasepool {
         ds4_gpu_decode_pipeline_fast_cache_reset();
         g_device = MTLCreateSystemDefaultDevice();
@@ -6810,17 +7839,28 @@ int ds4_gpu_init(void) {
         g_q4_expert_table_cache = [NSMutableDictionary dictionary];
         g_q4_expert_layer_residency_cache = [NSMutableDictionary dictionary];
         g_pipeline_cache = [NSMutableDictionary dictionary];
+        g_pipeline_cache_generation++;
+        if (g_pipeline_cache_generation == 0) g_pipeline_cache_generation++;
         g_dsv4_completion_cache = [NSCache new];
         g_dsv4_completion_cache.countLimit = 256u;
-        g_transient_buffers = [NSMutableArray array];
-        g_pending_cbs = [NSMutableArray array];
+        if (!g_progress_completion_group) {
+            g_progress_completion_group = dispatch_group_create();
+        }
+        for (int si = 0; si < DS4_GPU_MAX_STREAMS; si++) {
+            g_transient_buffers_by_stream[si] = [NSMutableArray array];
+            g_pending_cbs_by_stream[si] = [NSMutableArray array];
+        }
         if (!g_model_buffer_cache || !g_q4_expert_table_cache ||
             !g_q4_expert_layer_residency_cache ||
             !g_pipeline_cache || !g_dsv4_completion_cache ||
-            !g_transient_buffers || !g_pending_cbs) {
+            !g_progress_completion_group ||
+            !g_transient_buffers_by_stream[0] ||
+            !g_pending_cbs_by_stream[0]) {
             fprintf(stderr, "ds4: Metal bookkeeping allocation failed\n");
-            g_pending_cbs = nil;
-            g_transient_buffers = nil;
+            for (int si = 0; si < DS4_GPU_MAX_STREAMS; si++) {
+                g_pending_cbs_by_stream[si] = nil;
+                g_transient_buffers_by_stream[si] = nil;
+            }
             g_dsv4_completion_cache = nil;
             g_pipeline_cache = nil;
             g_q4_expert_layer_residency_cache = nil;
@@ -7263,23 +8303,6 @@ int ds4_gpu_init(void) {
             return 0;
         }
 
-        fn = [library newFunctionWithName:@"kernel_swiglu_f32"];
-        if (!fn) {
-            fprintf(stderr, "ds4: Metal kernel_swiglu_f32 function not found\n");
-            g_queue = nil;
-            g_device = nil;
-            return 0;
-        }
-
-        g_swiglu_pipeline = [g_device newComputePipelineStateWithFunction:fn error:&error];
-        if (!g_swiglu_pipeline) {
-            fprintf(stderr, "ds4: Metal kernel_swiglu_f32 pipeline failed: %s\n",
-                    [[error localizedDescription] UTF8String]);
-            g_queue = nil;
-            g_device = nil;
-            return 0;
-        }
-
         fn = [library newFunctionWithName:@"kernel_swiglu_flat_f32"];
         if (!fn) {
             fprintf(stderr, "ds4: Metal kernel_swiglu_flat_f32 function not found\n");
@@ -7778,6 +8801,38 @@ int ds4_gpu_init(void) {
             return 0;
         }
 
+        error = nil;
+        fn = [library
+            newFunctionWithName:@"kernel_mul_mv_addr_iq2_xxs_pair_swiglu_mid_only_4096x2048_f32"
+                 constantValues:moe_mv_id_constants
+                          error:&error];
+        if (fn) {
+            g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_mid_only_4096x2048_pipeline =
+                [g_device newComputePipelineStateWithFunction:fn error:&error];
+        }
+        if (!g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_mid_only_4096x2048_pipeline) {
+            fprintf(stderr,
+                    "ds4: optional Metal M1 IQ2 addr mid-only pipeline unavailable: %s\n",
+                    error ? [[error localizedDescription] UTF8String] :
+                            "function not found");
+        }
+
+        error = nil;
+        fn = [library
+            newFunctionWithName:@"kernel_mul_mv_addr_iq2_xxs_pair_swiglu_mid_only_4096x2048_masked_f32"
+                 constantValues:moe_mv_id_constants
+                          error:&error];
+        if (fn) {
+            g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_mid_only_4096x2048_masked_pipeline =
+                [g_device newComputePipelineStateWithFunction:fn error:&error];
+        }
+        if (!g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_mid_only_4096x2048_masked_pipeline) {
+            fprintf(stderr,
+                    "ds4: optional Metal M1 IQ2 addr masked mid-only pipeline unavailable: %s\n",
+                    error ? [[error localizedDescription] UTF8String] :
+                            "function not found");
+        }
+
         error = nil;
         fn = [library newFunctionWithName:@"kernel_mul_mv_addr_iq2_xxs_f32"
                            constantValues:moe_mv_id_constants
@@ -8617,6 +9672,25 @@ int ds4_gpu_init(void) {
             return 0;
         }
 
+        error = nil;
+        fn = [library newFunctionWithName:@"kernel_dsv4_hc_weighted_sum_capture_last"];
+        if (!fn) {
+            fprintf(stderr, "ds4: Metal kernel_dsv4_hc_weighted_sum_capture_last function not found\n");
+            g_queue = nil;
+            g_device = nil;
+            return 0;
+        }
+        g_hc_weighted_sum_capture_last_pipeline =
+            [g_device newComputePipelineStateWithFunction:fn error:&error];
+        if (!g_hc_weighted_sum_capture_last_pipeline) {
+            fprintf(stderr,
+                    "ds4: Metal kernel_dsv4_hc_weighted_sum_capture_last pipeline failed: %s\n",
+                    [[error localizedDescription] UTF8String]);
+            g_queue = nil;
+            g_device = nil;
+            return 0;
+        }
+
         error = nil;
         fn = [library newFunctionWithName:@"kernel_dsv4_output_hc_weights4"];
         if (fn) {
@@ -8928,8 +10002,6 @@ int ds4_gpu_init(void) {
             ds4_gpu_get_pipeline("kernel_glm_indexer_scores_batch");
         g_glm_indexer_scores_tiled_pipeline =
             ds4_gpu_get_pipeline("kernel_glm_indexer_scores_tiled");
-        g_glm_indexer_scores_tiled_f32_pipeline =
-            ds4_gpu_get_pipeline("kernel_glm_indexer_scores_tiled_f32");
         g_glm_qk_lowrank_pipeline =
             ds4_gpu_get_pipeline("kernel_glm_qk_lowrank_q8_0");
         g_glm_qk_lowrank_glm52_pipeline =
@@ -8940,8 +10012,6 @@ int ds4_gpu_init(void) {
             ds4_gpu_get_pipeline("kernel_glm_qk_lowrank_q8_0_batch");
         g_glm_qk_lowrank_batch_glm52_t4_pipeline =
             ds4_gpu_get_pipeline("kernel_glm_qk_lowrank_q8_0_batch_glm52_t4");
-        g_glm_value_project_q8_0_pipeline =
-            ds4_gpu_get_pipeline("kernel_glm_value_project_q8_0");
         g_glm_value_project_q8_0_batch_heads_pipeline =
             ds4_gpu_get_pipeline("kernel_glm_value_project_q8_0_batch_heads");
         g_glm_value_project_q8_0_batch_heads_mma_pipeline =
@@ -8974,16 +10044,10 @@ int ds4_gpu_init(void) {
             ds4_gpu_get_pipeline("kernel_glm_attention_indexed_batch_lora_group8_vec_causal");
         g_glm_attention_indexed_batch_lora_group8_vec_causal_fullheads_pipeline =
             ds4_gpu_get_pipeline("kernel_glm_attention_indexed_batch_lora_group8_vec_causal_fullheads");
-        g_glm_q4_k_pair_swiglu_f32_pipeline =
-            ds4_gpu_get_pipeline("kernel_glm_q4_K_pair_swiglu_f32");
         g_glm_q4_k_pair_swiglu2_f32_pipeline =
             ds4_gpu_get_pipeline("kernel_glm_q4_K_pair_swiglu2_f32");
         g_glm_q4_k_pair_swiglu4_f32_pipeline =
             ds4_gpu_get_pipeline("kernel_glm_q4_K_pair_swiglu4_f32");
-        g_glm_q4_k_pair_swiglu2_mapped_f32_pipeline =
-            ds4_gpu_get_pipeline("kernel_glm_q4_K_pair_swiglu2_mapped_f32");
-        g_glm_q4_k_pair_swiglu2_mapped_row_f32_pipeline =
-            ds4_gpu_get_pipeline("kernel_glm_q4_K_pair_swiglu2_mapped_row_f32");
         g_glm_q2_k_pair_swiglu_f32_pipeline =
             ds4_gpu_get_pipeline("kernel_glm_q2_K_pair_swiglu_f32");
         g_glm_q2_k_addr_pair_swiglu2_f32_pipeline =
@@ -9004,10 +10068,6 @@ int ds4_gpu_init(void) {
             ds4_gpu_get_pipeline("kernel_glm_q4_K_addr_down_simd_f32");
         g_glm_q5_k_pair_swiglu_f32_pipeline =
             ds4_gpu_get_pipeline("kernel_glm_q5_K_pair_swiglu_f32");
-        g_glm_q5_k_pair_swiglu_mapped_f32_pipeline =
-            ds4_gpu_get_pipeline("kernel_glm_q5_K_pair_swiglu_mapped_f32");
-        g_glm_q5_k_pair_swiglu_mapped_row_f32_pipeline =
-            ds4_gpu_get_pipeline("kernel_glm_q5_K_pair_swiglu_mapped_row_f32");
         g_glm_q5_k_down_f32_pipeline =
             ds4_gpu_get_pipeline("kernel_glm_q5_K_down_f32");
         g_glm_q6_k_down_f32_pipeline =
@@ -9016,6 +10076,12 @@ int ds4_gpu_init(void) {
             ds4_gpu_get_pipeline("kernel_dsv4_router_weights_batch");
         g_dsv4_hc_expand4_pipeline =
             ds4_gpu_get_pipeline("kernel_dsv4_hc_expand4");
+        g_dsv4_attn_out_q8_mm_hc_expand4_pipeline =
+            ds4_gpu_get_pipeline(
+                "kernel_dsv4_attn_out_q8_mm_hc_expand4_batch");
+        g_dsv4_attn_out_q4_mm_hc_expand4_pipeline =
+            ds4_gpu_get_pipeline(
+                "kernel_dsv4_attn_out_q4_K_f16_rhs_mm_hc_expand4_batch");
         if (!g_dsv4_indexer_score_one_direct_pipeline ||
             !g_dsv4_compressor_store_one_pipeline ||
             !g_dsv4_sort_i32_rows_asc_pipeline ||
@@ -9047,13 +10113,11 @@ int ds4_gpu_init(void) {
             !g_glm_indexer_score_one_direct_pipeline ||
             !g_glm_indexer_scores_batch_pipeline ||
             !g_glm_indexer_scores_tiled_pipeline ||
-            !g_glm_indexer_scores_tiled_f32_pipeline ||
             !g_glm_qk_lowrank_pipeline ||
             !g_glm_qk_lowrank_glm52_pipeline ||
             !g_glm_qk_lowrank_glm52_sg_pipeline ||
             !g_glm_qk_lowrank_batch_pipeline ||
             !g_glm_qk_lowrank_batch_glm52_t4_pipeline ||
-            !g_glm_value_project_q8_0_pipeline ||
             !g_glm_value_project_q8_0_batch_heads_pipeline ||
             !g_glm_value_project_q8_0_batch_heads_mma_pipeline ||
             !g_glm_attention_indexed_decode_pipeline ||
@@ -9070,11 +10134,8 @@ int ds4_gpu_init(void) {
             !g_glm_attention_indexed_batch_lora_group8_vec_valid_fullheads_pipeline ||
             !g_glm_attention_indexed_batch_lora_group8_vec_causal_pipeline ||
             !g_glm_attention_indexed_batch_lora_group8_vec_causal_fullheads_pipeline ||
-            !g_glm_q4_k_pair_swiglu_f32_pipeline ||
             !g_glm_q4_k_pair_swiglu2_f32_pipeline ||
             !g_glm_q4_k_pair_swiglu4_f32_pipeline ||
-            !g_glm_q4_k_pair_swiglu2_mapped_f32_pipeline ||
-            !g_glm_q4_k_pair_swiglu2_mapped_row_f32_pipeline ||
             !g_glm_q2_k_pair_swiglu_f32_pipeline ||
             !g_glm_q2_k_addr_pair_swiglu2_f32_pipeline ||
             !g_glm_q2_k_addr_pair_swiglu2_masked_f32_pipeline ||
@@ -9085,8 +10146,6 @@ int ds4_gpu_init(void) {
             !g_glm_q2_k_addr_down_f32_pipeline ||
             !g_glm_q4_k_addr_down_f32_pipeline ||
             !g_glm_q5_k_pair_swiglu_f32_pipeline ||
-            !g_glm_q5_k_pair_swiglu_mapped_f32_pipeline ||
-            !g_glm_q5_k_pair_swiglu_mapped_row_f32_pipeline ||
             !g_glm_q5_k_down_f32_pipeline ||
             !g_glm_q6_k_down_f32_pipeline ||
             !g_dsv4_hc_expand4_pipeline) {
@@ -9139,8 +10198,389 @@ int ds4_gpu_test_mxfp4_down_half_lut(uint16_t *legacy_bits,
     return 1;
 }
 
+static uint64_t ds4_gpu_test_count_non_sentinel_bytes(
+        const uint8_t *p, size_t n, uint8_t sentinel) {
+    uint64_t count = 0;
+    for (size_t i = 0; i < n; i++) {
+        count += p[i] != sentinel;
+    }
+    return count;
+}
+
+int ds4_gpu_test_iq2_addr_mid_only_oracle(
+        ds4_gpu_iq2_mid_only_oracle_report *report) {
+    enum {
+        N_TOTAL_EXPERT = 256,
+        N_SELECTED = 6,
+        IN_DIM = 4096,
+        MID_DIM = 2048,
+        IQ2_BLOCK = 256,
+        IQ2_BLOCK_BYTES = 66,
+        GUARD_BYTES = 64,
+    };
+    typedef struct {
+        uint16_t d;
+        uint16_t qs[IQ2_BLOCK / 8];
+    } ds4_test_block_iq2_xxs;
+
+    if (!report || sizeof(ds4_test_block_iq2_xxs) != IQ2_BLOCK_BYTES) return 0;
+    memset(report, 0, sizeof(*report));
+    if (!g_initialized && !ds4_gpu_init()) return 0;
+    if (!g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_pipeline ||
+        !g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_mid_only_4096x2048_pipeline ||
+        !g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_masked_pipeline ||
+        !g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_mid_only_4096x2048_masked_pipeline) {
+        fprintf(stderr, "ds4: Metal IQ2 addr mid-only oracle pipelines unavailable\n");
+        return 0;
+    }
+
+    @autoreleasepool {
+        const NSUInteger row_bytes =
+            (IN_DIM / IQ2_BLOCK) * sizeof(ds4_test_block_iq2_xxs);
+        const NSUInteger matrix_bytes = (NSUInteger)MID_DIM * row_bytes;
+        const NSUInteger x_bytes = (NSUInteger)IN_DIM * sizeof(float);
+        const NSUInteger payload_bytes =
+            (NSUInteger)N_SELECTED * MID_DIM * sizeof(float);
+        const NSUInteger guarded_bytes = payload_bytes + 2u * GUARD_BYTES;
+        const NSUInteger table_bytes =
+            (NSUInteger)N_TOTAL_EXPERT * sizeof(uint64_t);
+        const NSUInteger ids_bytes = N_SELECTED * sizeof(int32_t);
+        const NSUInteger weights_bytes = N_SELECTED * sizeof(float);
+        const uint8_t sentinel = 0xa5u;
+
+        id<MTLBuffer> gate_weights = [g_device newBufferWithLength:matrix_bytes
+                                                           options:MTLResourceStorageModeShared];
+        id<MTLBuffer> up_weights = [g_device newBufferWithLength:matrix_bytes
+                                                         options:MTLResourceStorageModeShared];
+        id<MTLBuffer> x = [g_device newBufferWithLength:x_bytes
+                                                options:MTLResourceStorageModeShared];
+        id<MTLBuffer> gate_addrs = [g_device newBufferWithLength:table_bytes
+                                                         options:MTLResourceStorageModeShared];
+        id<MTLBuffer> up_addrs = [g_device newBufferWithLength:table_bytes
+                                                       options:MTLResourceStorageModeShared];
+        id<MTLBuffer> ids = [g_device newBufferWithLength:ids_bytes
+                                                  options:MTLResourceStorageModeShared];
+        id<MTLBuffer> weights = [g_device newBufferWithLength:weights_bytes
+                                                      options:MTLResourceStorageModeShared];
+        id<MTLBuffer> canonical_gate = [g_device newBufferWithLength:guarded_bytes
+                                                             options:MTLResourceStorageModeShared];
+        id<MTLBuffer> canonical_up = [g_device newBufferWithLength:guarded_bytes
+                                                           options:MTLResourceStorageModeShared];
+        id<MTLBuffer> canonical_mid = [g_device newBufferWithLength:guarded_bytes
+                                                            options:MTLResourceStorageModeShared];
+        id<MTLBuffer> candidate_gate = [g_device newBufferWithLength:guarded_bytes
+                                                             options:MTLResourceStorageModeShared];
+        id<MTLBuffer> candidate_up = [g_device newBufferWithLength:guarded_bytes
+                                                           options:MTLResourceStorageModeShared];
+        id<MTLBuffer> candidate_mid = [g_device newBufferWithLength:guarded_bytes
+                                                            options:MTLResourceStorageModeShared];
+        id<MTLBuffer> masked_canonical_gate = [g_device newBufferWithLength:guarded_bytes
+                                                                    options:MTLResourceStorageModeShared];
+        id<MTLBuffer> masked_canonical_up = [g_device newBufferWithLength:guarded_bytes
+                                                                  options:MTLResourceStorageModeShared];
+        id<MTLBuffer> masked_canonical_mid = [g_device newBufferWithLength:guarded_bytes
+                                                                   options:MTLResourceStorageModeShared];
+        id<MTLBuffer> masked_candidate_gate = [g_device newBufferWithLength:guarded_bytes
+                                                                    options:MTLResourceStorageModeShared];
+        id<MTLBuffer> masked_candidate_up = [g_device newBufferWithLength:guarded_bytes
+                                                                  options:MTLResourceStorageModeShared];
+        id<MTLBuffer> masked_candidate_mid = [g_device newBufferWithLength:guarded_bytes
+                                                                   options:MTLResourceStorageModeShared];
+        if (!gate_weights || !up_weights || !x || !gate_addrs || !up_addrs ||
+            !ids || !weights || !canonical_gate || !canonical_up ||
+            !canonical_mid || !candidate_gate || !candidate_up || !candidate_mid ||
+            !masked_canonical_gate || !masked_canonical_up || !masked_canonical_mid ||
+            !masked_candidate_gate || !masked_candidate_up || !masked_candidate_mid) {
+            fprintf(stderr, "ds4: Metal IQ2 addr mid-only oracle allocation failed\n");
+            return 0;
+        }
+
+        ds4_test_block_iq2_xxs *gate_blocks =
+            (ds4_test_block_iq2_xxs *)[gate_weights contents];
+        ds4_test_block_iq2_xxs *up_blocks =
+            (ds4_test_block_iq2_xxs *)[up_weights contents];
+        const NSUInteger n_blocks = matrix_bytes / sizeof(*gate_blocks);
+        uint32_t gate_state = 0x91e10da5u;
+        uint32_t up_state = 0x6d2b79f5u;
+        for (NSUInteger i = 0; i < n_blocks; i++) {
+            /* Positive, finite half scales in [0.03125, 0.234375]. */
+            gate_blocks[i].d = (uint16_t)(0x2800u + ((i % 7u) << 7u));
+            up_blocks[i].d = (uint16_t)(0x2a00u + ((i % 5u) << 7u));
+            for (NSUInteger q = 0; q < IQ2_BLOCK / 8; q++) {
+                gate_state ^= gate_state << 13;
+                gate_state ^= gate_state >> 17;
+                gate_state ^= gate_state << 5;
+                up_state ^= up_state << 13;
+                up_state ^= up_state >> 17;
+                up_state ^= up_state << 5;
+                gate_blocks[i].qs[q] = (uint16_t)gate_state;
+                up_blocks[i].qs[q] = (uint16_t)up_state;
+            }
+        }
+        float *x_f32 = (float *)[x contents];
+        for (NSUInteger i = 0; i < IN_DIM; i++) {
+            x_f32[i] = ((int)(i % 37u) - 18) * (1.0f / 256.0f);
+        }
+
+        static const int32_t selected_ids[N_SELECTED] = { 3, 17, 41, 89, 137, 251 };
+        static const float route_weights[N_SELECTED] = {
+            1.0f, 0.875f, 0.75f, 0.625f, 0.5f, 0.375f,
+        };
+        memcpy([ids contents], selected_ids, sizeof(selected_ids));
+        memcpy([weights contents], route_weights, sizeof(route_weights));
+        memset([gate_addrs contents], 0, table_bytes);
+        memset([up_addrs contents], 0, table_bytes);
+        uint64_t *gate_table = (uint64_t *)[gate_addrs contents];
+        uint64_t *up_table = (uint64_t *)[up_addrs contents];
+        const uint64_t gate_gpu_addr = (uint64_t)[gate_weights gpuAddress];
+        const uint64_t up_gpu_addr = (uint64_t)[up_weights gpuAddress];
+        if (gate_gpu_addr == 0 || up_gpu_addr == 0) {
+            fprintf(stderr, "ds4: Metal IQ2 addr mid-only oracle has no GPU addresses\n");
+            return 0;
+        }
+        for (NSUInteger i = 0; i < N_SELECTED; i++) {
+            gate_table[(uint32_t)selected_ids[i]] = gate_gpu_addr;
+            up_table[(uint32_t)selected_ids[i]] = up_gpu_addr;
+        }
+
+        memset([canonical_gate contents], sentinel, guarded_bytes);
+        memset([canonical_up contents], sentinel, guarded_bytes);
+        memset([canonical_mid contents], sentinel, guarded_bytes);
+        memset([candidate_gate contents], sentinel, guarded_bytes);
+        memset([candidate_up contents], sentinel, guarded_bytes);
+        memset([candidate_mid contents], sentinel, guarded_bytes);
+        memset([masked_canonical_gate contents], sentinel, guarded_bytes);
+        memset([masked_canonical_up contents], sentinel, guarded_bytes);
+        memset([masked_canonical_mid contents], sentinel, guarded_bytes);
+        memset([masked_candidate_gate contents], sentinel, guarded_bytes);
+        memset([masked_candidate_up contents], sentinel, guarded_bytes);
+        memset([masked_candidate_mid contents], sentinel, guarded_bytes);
+
+        ds4_gpu_mul_mv_id_args args = {
+            .nei0 = N_SELECTED,
+            .nei1 = 1,
+            .nbi1 = N_SELECTED * sizeof(int32_t),
+            .ne00 = IN_DIM,
+            .ne01 = MID_DIM,
+            .ne02 = N_TOTAL_EXPERT,
+            .nb00 = sizeof(ds4_test_block_iq2_xxs),
+            .nb01 = row_bytes,
+            .nb02 = matrix_bytes,
+            .ne10 = IN_DIM,
+            .ne11 = 1,
+            .ne12 = 1,
+            .ne13 = 1,
+            .nb10 = sizeof(float),
+            .nb11 = x_bytes,
+            .nb12 = x_bytes,
+            .ne0 = MID_DIM,
+            .ne1 = N_SELECTED,
+            .nb1 = (uint64_t)MID_DIM * sizeof(float),
+            .nr0 = 4,
+            .tp_world = 1,
+        };
+        ds4_gpu_dsv4_moe_swiglu_weight_args act = {
+            .width = MID_DIM,
+            .rows = N_SELECTED,
+            .gate_row_stride = (uint64_t)MID_DIM * sizeof(float),
+            .up_row_stride = (uint64_t)MID_DIM * sizeof(float),
+            .mid_row_stride = (uint64_t)MID_DIM * sizeof(float),
+            .weight_stride = sizeof(float),
+            .write_clamped = 0,
+            .clamp_value = 6.0f,
+        };
+
+        id<MTLCommandBuffer> cb = ds4_gpu_new_command_buffer();
+        id<MTLComputeCommandEncoder> enc = cb ? [cb computeCommandEncoder] : nil;
+        if (!cb || !enc) return 0;
+        [enc useResource:gate_weights usage:MTLResourceUsageRead];
+        [enc useResource:up_weights usage:MTLResourceUsageRead];
+        [enc setThreadgroupMemoryLength:256u * sizeof(uint64_t) + 128u * sizeof(uint8_t)
+                                atIndex:0];
+
+#define DS4_ENCODE_IQ2_ADDR_ORACLE(PIPELINE, GATE, UP, MID) do { \
+            [enc setComputePipelineState:(PIPELINE)]; \
+            [enc setBytes:&args length:sizeof(args) atIndex:0]; \
+            [enc setBytes:&act length:sizeof(act) atIndex:1]; \
+            [enc setBuffer:gate_addrs offset:0 atIndex:2]; \
+            [enc setBuffer:up_addrs offset:0 atIndex:3]; \
+            [enc setBuffer:x offset:0 atIndex:4]; \
+            [enc setBuffer:(GATE) offset:GUARD_BYTES atIndex:5]; \
+            [enc setBuffer:(UP) offset:GUARD_BYTES atIndex:6]; \
+            [enc setBuffer:(MID) offset:GUARD_BYTES atIndex:7]; \
+            [enc setBuffer:ids offset:0 atIndex:8]; \
+            [enc setBuffer:weights offset:0 atIndex:9]; \
+            [enc dispatchThreadgroups:MTLSizeMake(MID_DIM / 8u, 1, N_SELECTED) \
+                 threadsPerThreadgroup:MTLSizeMake(32u, 2u, 1u)]; \
+        } while (0)
+        DS4_ENCODE_IQ2_ADDR_ORACLE(
+            g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_pipeline,
+            canonical_gate, canonical_up, canonical_mid);
+        DS4_ENCODE_IQ2_ADDR_ORACLE(
+            g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_mid_only_4096x2048_pipeline,
+            candidate_gate, candidate_up, candidate_mid);
+#undef DS4_ENCODE_IQ2_ADDR_ORACLE
+        const ds4_gpu_stream_expert_split_args split_even = {
+            .active_mask = 0x15u,
+            .accumulate = 0u,
+        };
+        const ds4_gpu_stream_expert_split_args split_odd = {
+            .active_mask = 0x2au,
+            .accumulate = 0u,
+        };
+#define DS4_ENCODE_IQ2_ADDR_MASKED_ORACLE(PIPELINE, SPLIT, GATE, UP, MID) do { \
+            [enc setComputePipelineState:(PIPELINE)]; \
+            [enc setBytes:&args length:sizeof(args) atIndex:0]; \
+            [enc setBytes:&act length:sizeof(act) atIndex:1]; \
+            [enc setBytes:(SPLIT) length:sizeof(*(SPLIT)) atIndex:2]; \
+            [enc setBuffer:gate_addrs offset:0 atIndex:3]; \
+            [enc setBuffer:up_addrs offset:0 atIndex:4]; \
+            [enc setBuffer:x offset:0 atIndex:5]; \
+            [enc setBuffer:(GATE) offset:GUARD_BYTES atIndex:6]; \
+            [enc setBuffer:(UP) offset:GUARD_BYTES atIndex:7]; \
+            [enc setBuffer:(MID) offset:GUARD_BYTES atIndex:8]; \
+            [enc setBuffer:ids offset:0 atIndex:9]; \
+            [enc setBuffer:weights offset:0 atIndex:10]; \
+            [enc dispatchThreadgroups:MTLSizeMake(MID_DIM / 8u, 1, N_SELECTED) \
+                 threadsPerThreadgroup:MTLSizeMake(32u, 2u, 1u)]; \
+        } while (0)
+        DS4_ENCODE_IQ2_ADDR_MASKED_ORACLE(
+            g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_masked_pipeline,
+            &split_even, masked_canonical_gate, masked_canonical_up, masked_canonical_mid);
+        DS4_ENCODE_IQ2_ADDR_MASKED_ORACLE(
+            g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_mid_only_4096x2048_masked_pipeline,
+            &split_even, masked_candidate_gate, masked_candidate_up, masked_candidate_mid);
+        [enc endEncoding];
+        [cb commit];
+        if (!ds4_gpu_wait_command_buffer(cb, "IQ2 addr mid-only masked-even oracle")) {
+            return 0;
+        }
+
+        /* Validate active_mask before its complementary leg fills the other
+         * rows.  A final-only comparison after 0x15 + 0x2a cannot detect a
+         * kernel that simply ignores the mask. */
+        const NSUInteger masked_row_bytes = (NSUInteger)MID_DIM * sizeof(float);
+        const uint8_t *masked_even_outputs[] = {
+            (const uint8_t *)[masked_canonical_gate contents] + GUARD_BYTES,
+            (const uint8_t *)[masked_canonical_up contents] + GUARD_BYTES,
+            (const uint8_t *)[masked_canonical_mid contents] + GUARD_BYTES,
+            (const uint8_t *)[masked_candidate_gate contents] + GUARD_BYTES,
+            (const uint8_t *)[masked_candidate_up contents] + GUARD_BYTES,
+            (const uint8_t *)[masked_candidate_mid contents] + GUARD_BYTES,
+        };
+        for (NSUInteger output = 0;
+             output < sizeof(masked_even_outputs) / sizeof(masked_even_outputs[0]);
+             output++) {
+            for (NSUInteger row = 1u; row < N_SELECTED; row += 2u) {
+                report->masked_inactive_writes +=
+                    ds4_gpu_test_count_non_sentinel_bytes(
+                        masked_even_outputs[output] + row * masked_row_bytes,
+                        masked_row_bytes, sentinel);
+            }
+        }
+
+        cb = ds4_gpu_new_command_buffer();
+        enc = cb ? [cb computeCommandEncoder] : nil;
+        if (!cb || !enc) return 0;
+        [enc useResource:gate_weights usage:MTLResourceUsageRead];
+        [enc useResource:up_weights usage:MTLResourceUsageRead];
+        [enc setThreadgroupMemoryLength:256u * sizeof(uint64_t) + 128u * sizeof(uint8_t)
+                                atIndex:0];
+        DS4_ENCODE_IQ2_ADDR_MASKED_ORACLE(
+            g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_masked_pipeline,
+            &split_odd, masked_canonical_gate, masked_canonical_up, masked_canonical_mid);
+        DS4_ENCODE_IQ2_ADDR_MASKED_ORACLE(
+            g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_mid_only_4096x2048_masked_pipeline,
+            &split_odd, masked_candidate_gate, masked_candidate_up, masked_candidate_mid);
+#undef DS4_ENCODE_IQ2_ADDR_MASKED_ORACLE
+        [enc endEncoding];
+        [cb commit];
+        if (!ds4_gpu_wait_command_buffer(cb, "IQ2 addr mid-only tensor oracle")) {
+            return 0;
+        }
+
+        const uint8_t *cg = (const uint8_t *)[canonical_gate contents];
+        const uint8_t *cu = (const uint8_t *)[canonical_up contents];
+        const uint8_t *cm = (const uint8_t *)[canonical_mid contents];
+        const uint8_t *ng = (const uint8_t *)[candidate_gate contents];
+        const uint8_t *nu = (const uint8_t *)[candidate_up contents];
+        const uint8_t *nm = (const uint8_t *)[candidate_mid contents];
+        const uint8_t *mcg = (const uint8_t *)[masked_canonical_gate contents];
+        const uint8_t *mcu = (const uint8_t *)[masked_canonical_up contents];
+        const uint8_t *mcm = (const uint8_t *)[masked_canonical_mid contents];
+        const uint8_t *mng = (const uint8_t *)[masked_candidate_gate contents];
+        const uint8_t *mnu = (const uint8_t *)[masked_candidate_up contents];
+        const uint8_t *mnm = (const uint8_t *)[masked_candidate_mid contents];
+        const uint32_t sentinel_word = 0xa5a5a5a5u;
+        const uint32_t *cg_words = (const uint32_t *)(cg + GUARD_BYTES);
+        const uint32_t *cu_words = (const uint32_t *)(cu + GUARD_BYTES);
+        const uint32_t *cm_words = (const uint32_t *)(cm + GUARD_BYTES);
+        const uint32_t *nm_words = (const uint32_t *)(nm + GUARD_BYTES);
+        const uint32_t *mcg_words = (const uint32_t *)(mcg + GUARD_BYTES);
+        const uint32_t *mcu_words = (const uint32_t *)(mcu + GUARD_BYTES);
+        const uint32_t *mcm_words = (const uint32_t *)(mcm + GUARD_BYTES);
+        const uint32_t *mnm_words = (const uint32_t *)(mnm + GUARD_BYTES);
+        report->mid_words = payload_bytes / sizeof(uint32_t);
+        for (uint64_t i = 0; i < report->mid_words; i++) {
+            report->mid_mismatches += cm_words[i] != nm_words[i];
+            report->canonical_gate_unwritten += cg_words[i] == sentinel_word;
+            report->canonical_up_unwritten += cu_words[i] == sentinel_word;
+            report->masked_mid_mismatches += mcm_words[i] != mnm_words[i];
+            report->masked_canonical_gate_unwritten += mcg_words[i] == sentinel_word;
+            report->masked_canonical_up_unwritten += mcu_words[i] == sentinel_word;
+        }
+        report->candidate_gate_writes =
+            ds4_gpu_test_count_non_sentinel_bytes(ng, guarded_bytes, sentinel);
+        report->candidate_up_writes =
+            ds4_gpu_test_count_non_sentinel_bytes(nu, guarded_bytes, sentinel);
+        report->masked_gate_writes =
+            ds4_gpu_test_count_non_sentinel_bytes(mng, guarded_bytes, sentinel);
+        report->masked_up_writes =
+            ds4_gpu_test_count_non_sentinel_bytes(mnu, guarded_bytes, sentinel);
+
+#define DS4_COUNT_GUARDS(P) do { \
+            report->guard_byte_mismatches += \
+                ds4_gpu_test_count_non_sentinel_bytes((P), GUARD_BYTES, sentinel); \
+            report->guard_byte_mismatches += \
+                ds4_gpu_test_count_non_sentinel_bytes( \
+                    (P) + GUARD_BYTES + payload_bytes, GUARD_BYTES, sentinel); \
+        } while (0)
+        DS4_COUNT_GUARDS(cg);
+        DS4_COUNT_GUARDS(cu);
+        DS4_COUNT_GUARDS(cm);
+        DS4_COUNT_GUARDS(nm);
+        DS4_COUNT_GUARDS(mcg);
+        DS4_COUNT_GUARDS(mcu);
+        DS4_COUNT_GUARDS(mcm);
+        DS4_COUNT_GUARDS(mnm);
+#undef DS4_COUNT_GUARDS
+    }
+    return 1;
+}
+
 void ds4_gpu_test_set_flags(uint32_t flags) {
     g_test_flags = flags;
+    g_test_last_flash_attn_prefill_nwg = 0u;
+}
+
+double ds4_gpu_test_last_completed_gpu_ms(void) {
+    return g_last_completed_gpu_time_valid
+        ? g_last_completed_gpu_seconds * 1000.0 : -1.0;
+}
+
+uint32_t ds4_gpu_test_last_flash_attn_prefill_nwg(void) {
+    return g_test_last_flash_attn_prefill_nwg;
+}
+
+int ds4_gpu_test_reset_flash_attn_tmp(void) {
+    if (!ds4_gpu_synchronize()) return 0;
+    g_flash_attn_tmp_buffer = nil;
+    g_flash_attn_tmp_bytes = 0u;
+    return 1;
+}
+
+uint64_t ds4_gpu_test_flash_attn_tmp_bytes(void) {
+    return (uint64_t)g_flash_attn_tmp_bytes;
 }
 
 ds4_gpu_tensor *ds4_gpu_tensor_alloc(uint64_t bytes) {
@@ -9424,7 +10864,7 @@ int ds4_gpu_begin_commands(void) {
     if (!g_initialized && !ds4_gpu_init()) return 0;
     /* A failed concurrent FFN must never affect the next command batch. */
     ds4_gpu_parallel_ffn_reset_state(YES);
-    if (g_batch_cb) return 0;
+    if (g_batch_cb || g_stream_last_cb[g_ds4_stream]) return 0;
     /* Refresh once per command batch so same-engine A/B runs can toggle the
      * static PSO without paying for environment lookups in every layer. */
     g_use_dsv4_head_rms_norm_rope_tail_pipeline =
@@ -9472,6 +10912,41 @@ int ds4_gpu_flush_commands(void) {
     return 1;
 }
 
+/* Commit the current batch without draining the GPU.  `report` is a backend
+ * completion hook, not a user callback: it must only publish thread-safe
+ * state.  A later full drain joins this handler explicitly. */
+int ds4_gpu_flush_commands_progress(void (*report)(void *ctx), void *ctx) {
+    if (!g_initialized && !ds4_gpu_init()) return 0;
+    ds4_gpu_parallel_ffn_reset_state(YES);
+    if (!g_batch_cb || (report && !g_progress_completion_group)) return 0;
+
+    ds4_gpu_close_batch_encoder();
+    id<MTLCommandBuffer> cb = g_batch_cb;
+    g_batch_cb = nil;
+    g_batch_has_work = NO;
+    if (report) {
+        dispatch_group_enter(g_progress_completion_group);
+        [cb addCompletedHandler:^(id<MTLCommandBuffer> done_cb) {
+            (void)done_cb;
+            report(ctx);
+            dispatch_group_leave(g_progress_completion_group);
+        }];
+    }
+    [cb commit];
+    [g_pending_cbs addObject:cb];
+    ds4_gpu_stream_expert_cache_note_batch_committed();
+
+    g_batch_cb = ds4_gpu_new_command_buffer();
+    g_batch_has_work = NO;
+    if (g_batch_cb) ds4_gpu_stream_expert_cache_note_batch_created();
+    if (!g_batch_cb) {
+        (void)ds4_gpu_wait_pending_command_buffers("command batch");
+        [g_transient_buffers removeAllObjects];
+        return 0;
+    }
+    return 1;
+}
+
 int ds4_gpu_commands_active(void) {
     return g_batch_cb != nil;
 }
@@ -11405,6 +12880,37 @@ int ds4_gpu_end_commands(void) {
     return ds4_gpu_finish_command_buffer(cb, 1, "command batch");
 }
 
+int ds4_gpu_end_commands_async(void) {
+    if (!g_initialized || !g_batch_cb ||
+        g_stream_last_cb[g_ds4_stream]) {
+        return 0;
+    }
+    ds4_gpu_parallel_ffn_reset_state(YES);
+    ds4_gpu_close_batch_encoder();
+    id<MTLCommandBuffer> cb = g_batch_cb;
+    g_batch_cb = nil;
+    g_batch_has_work = NO;
+    [cb commit];
+    g_stream_last_cb[g_ds4_stream] = cb;
+    ds4_gpu_stream_expert_cache_note_batch_committed();
+    return 1;
+}
+
+int ds4_gpu_wait_stream(int idx) {
+    if (idx < 0 || idx >= DS4_GPU_MAX_STREAMS) return 0;
+    int ok = ds4_gpu_wait_pending_command_buffers_for_stream(
+            idx, "stream pending command batch");
+    id<MTLCommandBuffer> cb = g_stream_last_cb[idx];
+    if (cb) {
+        if (!ds4_gpu_wait_command_buffer(cb, "stream command batch")) ok = 0;
+        g_stream_last_cb[idx] = nil;
+        ds4_gpu_stream_expert_cache_note_owned_completed();
+    }
+    [g_transient_buffers_by_stream[idx] removeAllObjects];
+    ds4_gpu_model_buffer_cache_maybe_evict("stream command batch");
+    return ok;
+}
+
 static int ds4_gpu_flash_attn_stage_profile_boundary(
         id<MTLCommandBuffer> __strong *cbp,
         const char           *mode,
@@ -11451,14 +12957,18 @@ static int ds4_gpu_flash_attn_stage_profile_boundary(
 
 int ds4_gpu_synchronize(void) {
     if (!g_initialized && !ds4_gpu_init()) return 0;
-    if (g_batch_cb) return ds4_gpu_end_commands();
+    int ok = 1;
+    if (g_batch_cb && !ds4_gpu_end_commands()) ok = 0;
     ds4_gpu_parallel_ffn_reset_state(YES);
-    if ([g_pending_cbs count] != 0) {
-        int ok = ds4_gpu_wait_pending_command_buffers("synchronize");
-        [g_transient_buffers removeAllObjects];
-        ds4_gpu_model_buffer_cache_maybe_evict("synchronize");
-        return ok;
+    int had_stream_work = 0;
+    for (int i = 0; i < DS4_GPU_MAX_STREAMS; i++) {
+        if (g_stream_last_cb[i] ||
+            [g_pending_cbs_by_stream[i] count] != 0) {
+            had_stream_work = 1;
+            if (!ds4_gpu_wait_stream(i)) ok = 0;
+        }
     }
+    if (had_stream_work || !ok) return ok;
 
     id<MTLCommandBuffer> cb = ds4_gpu_new_command_buffer();
     if (!cb) return 0;
@@ -11466,7 +12976,11 @@ int ds4_gpu_synchronize(void) {
 }
 
 void ds4_gpu_cleanup(void) {
-    if (!g_initialized) return;
+    if (!g_initialized) {
+        g_iq2_stream_addr_mm_stats_enabled = 0;
+        ds4_gpu_iq2_stream_addr_mm_stats_reset();
+        return;
+    }
     ds4_gpu_queue_keepalive_stop_thread();
 
     @autoreleasepool {
@@ -11481,17 +12995,60 @@ void ds4_gpu_cleanup(void) {
                 g_stream_expert_cache_done_seq = g_stream_expert_cache_batch_seq;
             }
             g_stream_expert_cache_batch_seq = 0;
+            [g_transient_buffers removeAllObjects];
         }
-        (void)ds4_gpu_wait_pending_command_buffers("cleanup");
+        for (int si = 0; si < DS4_GPU_MAX_STREAMS; si++) {
+            (void)ds4_gpu_wait_stream(si);
+        }
+        ds4_gpu_iq2_stream_addr_mm_stats_print();
         if (ds4_gpu_stream_expert_timing_summary_enabled() &&
             getenv("DS4_METAL_MEMORY_REPORT") == NULL) {
             ds4_gpu_print_memory_report("at cleanup");
         }
         g_selected_readback_event = nil;
         g_selected_readback_event_value = 0;
-        [g_transient_buffers removeAllObjects];
+        if (getenv("DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_PROFILE") != NULL &&
+            (g_q4_attn_q_b_f16_cache_candidate_calls != 0 ||
+             g_q4_attn_q_b_f16_cache_builds != 0 ||
+             g_q4_attn_q_b_f16_cache_build_failures != 0 ||
+             g_q4_attn_q_b_f16_build_circuit_state !=
+                 DS4_Q4_ATTN_Q_B_F16_CIRCUIT_CLOSED)) {
+            fprintf(stderr,
+                    "ds4: Metal Q4 attn_q_b F16 cache entries=%u bytes=%.2f GiB "
+                    "calls=%llu hits=%llu misses=%llu builds=%llu failures=%llu "
+                    "fallbacks=%llu rejects=%llu circuit_state=%d\n",
+                    g_q4_attn_q_b_f16_cache_entries,
+                    ds4_gpu_gib(g_q4_attn_q_b_f16_cache_bytes),
+                    (unsigned long long)g_q4_attn_q_b_f16_cache_candidate_calls,
+                    (unsigned long long)g_q4_attn_q_b_f16_cache_hits,
+                    (unsigned long long)g_q4_attn_q_b_f16_cache_misses,
+                    (unsigned long long)g_q4_attn_q_b_f16_cache_builds,
+                    (unsigned long long)g_q4_attn_q_b_f16_cache_build_failures,
+                    (unsigned long long)g_q4_attn_q_b_f16_cache_fallbacks,
+                    (unsigned long long)g_q4_attn_q_b_f16_cache_rejects,
+                    (int)g_q4_attn_q_b_f16_build_circuit_state);
+        }
+        ds4_gpu_q4_attn_q_b_f16_cache_clear(1);
+        pthread_mutex_lock(&g_q4_qb_transient_f16_scratch_mu);
+        for (int si = 0; si < DS4_GPU_MAX_STREAMS; si++) {
+            g_q4_qb_transient_f16_scratch[si] = nil;
+            g_q4_qb_transient_f16_scratch_capacity[si] = 0u;
+            g_q4_qb_transient_f16_admission[si] =
+                (ds4_gpu_q4_qb_transient_f16_admission) {0};
+        }
+        pthread_mutex_unlock(&g_q4_qb_transient_f16_scratch_mu);
+        __atomic_store_n(&g_q4_qb_transient_f16_exact_views_created,
+                         0u,
+                         __ATOMIC_RELEASE);
+        __atomic_store_n(&g_q4_qb_transient_f16_exact_views_live,
+                         0u,
+                         __ATOMIC_RELEASE);
         ds4_gpu_stream_expert_pread_pool_shutdown();
         ds4_gpu_stream_expert_cache_clear_all(1);
+        ds4_gpu_stream_expert_cache_live_release();
+        g_stream_expert_cache_gate_class_bytes = 0;
+        g_stream_expert_cache_down_class_bytes = 0;
+        g_stream_expert_cache_class_conflict = 0;
         for (uint32_t layer = 0; layer < DS4_METAL_STREAM_EXPERT_CACHE_MAX_LAYER; layer++) {
             g_stream_expert_cache_gate_addr_buffers[layer] = nil;
             g_stream_expert_cache_up_addr_buffers[layer] = nil;
@@ -11519,7 +13076,6 @@ void ds4_gpu_cleanup(void) {
         g_cpy_contig_f16_f32_pipeline = nil;
         g_cpy_contig_f16_f16_pipeline = nil;
         g_flash_kv_stage_f16_pipeline = nil;
-        g_swiglu_pipeline = nil;
         g_swiglu_flat_pipeline = nil;
         g_add_pipeline = nil;
         g_add2_pipeline = nil;
@@ -11549,6 +13105,7 @@ void ds4_gpu_cleanup(void) {
         g_hc_split_weighted_sum_norm_pipeline = nil;
         g_dsv4_hc_producer_pre_norm_pipeline = nil;
         g_hc_weighted_sum_pipeline = nil;
+        g_hc_weighted_sum_capture_last_pipeline = nil;
         g_output_hc_weights4_pipeline = nil;
         g_hc_expand_pipeline = nil;
         g_moe_mul_mv_id_iq2_xxs_pipeline = nil;
@@ -11591,6 +13148,8 @@ void ds4_gpu_cleanup(void) {
         g_moe_mul_mv_slots6_mxfp4_pair_swiglu_pipeline = nil;
         g_moe_mul_mv_slots6_mxfp4_sum6_pipeline = nil;
         g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_pipeline = nil;
+        g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_mid_only_4096x2048_pipeline = nil;
+        g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_mid_only_4096x2048_masked_pipeline = nil;
         g_moe_mul_mv_addr_iq2_xxs_pipeline = nil;
         g_moe_mul_mv_addr_q2_k_sum6_pipeline = nil;
         g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_masked_pipeline = nil;
@@ -11623,6 +13182,8 @@ void ds4_gpu_cleanup(void) {
         g_soft_max_f32_4_pipeline = nil;
         g_argsort_f32_i32_desc_pipeline = nil;
         g_argsort_merge_f32_i32_desc_pipeline = nil;
+        g_dsv4_argmax_top1_stage1_pipeline = nil;
+        g_dsv4_argmax_top1_stage2_pipeline = nil;
         g_sum_rows_f32_f32_pipeline = nil;
         g_dsv4_topk_mask_pipeline = nil;
         g_dsv4_topk_mask_scatter_pipeline = nil;
@@ -11635,6 +13196,7 @@ void ds4_gpu_cleanup(void) {
         g_dsv4_indexed_attention_heads16_dual_pipeline = nil;
         g_dsv4_indexed_attention_heads8_split_pipeline = nil;
         g_dsv4_indexed_attention_heads8_split_reduce_pipeline = nil;
+        g_attn_out_low_q8_static_unavailable = false;
         g_dsv4_softplus_sqrt_pipeline = nil;
         g_dsv4_router_finalize_one_pipeline = nil;
         g_dsv4_router_finalize_one_simd_pipeline = nil;
@@ -11666,13 +13228,11 @@ void ds4_gpu_cleanup(void) {
         g_glm_indexer_score_one_direct_pipeline = nil;
         g_glm_indexer_scores_batch_pipeline = nil;
         g_glm_indexer_scores_tiled_pipeline = nil;
-        g_glm_indexer_scores_tiled_f32_pipeline = nil;
         g_glm_qk_lowrank_pipeline = nil;
         g_glm_qk_lowrank_glm52_pipeline = nil;
         g_glm_qk_lowrank_glm52_sg_pipeline = nil;
         g_glm_qk_lowrank_batch_pipeline = nil;
         g_glm_qk_lowrank_batch_glm52_t4_pipeline = nil;
-        g_glm_value_project_q8_0_pipeline = nil;
         g_glm_value_project_q8_0_batch_heads_pipeline = nil;
         g_glm_value_project_q8_0_batch_heads_mma_pipeline = nil;
         g_glm_attention_indexed_decode_pipeline = nil;
@@ -11689,11 +13249,8 @@ void ds4_gpu_cleanup(void) {
         g_glm_attention_indexed_batch_lora_group8_vec_valid_fullheads_pipeline = nil;
         g_glm_attention_indexed_batch_lora_group8_vec_causal_pipeline = nil;
         g_glm_attention_indexed_batch_lora_group8_vec_causal_fullheads_pipeline = nil;
-        g_glm_q4_k_pair_swiglu_f32_pipeline = nil;
         g_glm_q4_k_pair_swiglu2_f32_pipeline = nil;
         g_glm_q4_k_pair_swiglu4_f32_pipeline = nil;
-        g_glm_q4_k_pair_swiglu2_mapped_f32_pipeline = nil;
-        g_glm_q4_k_pair_swiglu2_mapped_row_f32_pipeline = nil;
         g_glm_q2_k_pair_swiglu_f32_pipeline = nil;
         g_glm_q2_k_addr_pair_swiglu2_f32_pipeline = nil;
         g_glm_q2_k_addr_pair_swiglu2_masked_f32_pipeline = nil;
@@ -11704,85 +13261,105 @@ void ds4_gpu_cleanup(void) {
         g_glm_q2_k_addr_down_f32_pipeline = nil;
         g_glm_q4_k_addr_down_f32_pipeline = nil;
         g_glm_q5_k_pair_swiglu_f32_pipeline = nil;
-        g_glm_q5_k_pair_swiglu_mapped_f32_pipeline = nil;
-        g_glm_q5_k_pair_swiglu_mapped_row_f32_pipeline = nil;
         g_glm_q5_k_down_f32_pipeline = nil;
         g_glm_q6_k_down_f32_pipeline = nil;
         g_dsv4_router_weights_batch_pipeline = nil;
         g_dsv4_hc_expand4_pipeline = nil;
-        g_flash_attn_mask_buffer = nil;
-        g_flash_attn_zero_mask_buffer = nil;
-        g_flash_attn_pad_buffer = nil;
-        g_flash_attn_tmp_buffer = nil;
-        g_flash_attn_blk_buffer = nil;
+        g_dsv4_attn_out_q8_mm_hc_expand4_pipeline = nil;
+        g_dsv4_attn_out_q4_mm_hc_expand4_pipeline = nil;
         ds4_gpu_clear_zero_prefix_prefill_mask_cache();
-        g_flash_attn_ring_buffer = nil;
-        g_flash_attn_kv_buffer = nil;
-        g_glm_flash_attn_mask_buffer = nil;
-        g_compressor_pool_kv_buffer = nil;
-        g_compressor_pool_score_buffer = nil;
-        g_compressor_pool_score_cont_buffer = nil;
-        g_compressor_pool_softmax_buffer = nil;
-        g_compressor_pool_product_buffer = nil;
-        g_compressor_store_ape_buffer = nil;
-        g_compressor_store_score_buffer = nil;
-        g_embed_rows_buffer = nil;
-        g_router_selection_buffer = nil;
-        g_router_weight_sum_buffer = nil;
-        g_indexer_head_scores_buffer = nil;
-        g_indexer_topk_buffer = nil;
-        g_indexed_topk_buffer = nil;
         g_stream_expert_validate_status_buffer = nil;
-        g_f16_round_scratch_buffer = nil;
-        g_raw_store_round_buffer = nil;
-        g_moe_gate_scratch_buffer = nil;
-        g_moe_down_scratch_buffer = nil;
-        g_moe_id_map_buffer = nil;
-        g_moe_q4_gate_slots_buffer = nil;
-        g_moe_q4_up_slots_buffer = nil;
-        g_moe_q4_down_slots_buffer = nil;
-        g_attn_out_group_ids_buffer = nil;
+        for (int si = 0; si < DS4_GPU_MAX_STREAMS; si++) {
+            g_ds4_stream = si;
+            g_flash_attn_mask_buffer = nil;
+            g_flash_attn_zero_mask_buffer = nil;
+            g_flash_attn_pad_buffer = nil;
+            g_flash_attn_tmp_buffer = nil;
+            g_flash_attn_blk_buffer = nil;
+            g_flash_attn_kv_buffer = nil;
+            g_glm_flash_attn_mask_buffer = nil;
+            g_compressor_pool_kv_buffer = nil;
+            g_compressor_pool_score_buffer = nil;
+            g_compressor_pool_score_cont_buffer = nil;
+            g_compressor_pool_softmax_buffer = nil;
+            g_compressor_pool_product_buffer = nil;
+            g_compressor_store_ape_buffer = nil;
+            g_compressor_store_score_buffer = nil;
+            g_embed_rows_buffer = nil;
+            g_router_selection_buffer = nil;
+            g_router_weight_sum_buffer = nil;
+            g_indexer_head_scores_buffer = nil;
+            g_indexer_topk_buffer = nil;
+            g_stream_scratch[si].argmax_top1_scratch_v = nil;
+            g_stream_scratch[si].argmax_top1_scratch_i = nil;
+            g_stream_scratch[si].argmax_top1_seq = 0u;
+            g_indexed_topk_buffer = nil;
+            g_q4_pair_rhs_f16_buffer = nil;
+            g_f16_round_scratch_buffer = nil;
+            g_raw_store_round_buffer = nil;
+            g_moe_gate_scratch_buffer = nil;
+            g_moe_down_scratch_buffer = nil;
+            g_moe_id_map_buffer = nil;
+            g_moe_q4_gate_slots_buffer = nil;
+            g_moe_q4_up_slots_buffer = nil;
+            g_moe_q4_down_slots_buffer = nil;
+            g_attn_out_group_ids_buffer = nil;
+        }
+        g_ds4_stream = 0;
         g_model_fd = -1;
+        if (g_model_fd_nocache >= 0) {
+            close(g_model_fd_nocache);
+            g_model_fd_nocache = -1;
+        }
+        g_model_fd_nocache_all = 0;
+        g_model_fd_nocache_prefill = 0;
+        g_model_fd_nocache_prefill_auto = 0;
         g_model_map_ptr = NULL;
         g_model_map_size = 0;
         g_model_mapped_offset = 0;
         g_model_mapped_size = 0;
         g_model_mapped_max_tensor_bytes = 0;
+        g_support_model_map_ptr = NULL;
+        g_support_model_map_size = 0;
         ds4_gpu_tensor_tracking_reset();
-        g_flash_attn_mask_bytes = 0;
-        g_flash_attn_zero_mask_bytes = 0;
-        g_flash_attn_pad_bytes = 0;
-        g_flash_attn_tmp_bytes = 0;
-        g_flash_attn_blk_bytes = 0;
-        g_flash_attn_ring_bytes = 0;
-        g_flash_attn_kv_bytes = 0;
-        g_glm_flash_attn_mask_bytes = 0;
+        for (int si = 0; si < DS4_GPU_MAX_STREAMS; si++) {
+            g_ds4_stream = si;
+            g_flash_attn_mask_bytes = 0;
+            g_flash_attn_zero_mask_bytes = 0;
+            g_flash_attn_pad_bytes = 0;
+            g_flash_attn_tmp_bytes = 0;
+            g_flash_attn_blk_bytes = 0;
+            g_flash_attn_kv_bytes = 0;
+            g_glm_flash_attn_mask_bytes = 0;
+            g_compressor_pool_kv_bytes = 0;
+            g_compressor_pool_score_bytes = 0;
+            g_compressor_pool_score_cont_bytes = 0;
+            g_compressor_pool_softmax_bytes = 0;
+            g_compressor_pool_product_bytes = 0;
+            g_compressor_store_ape_bytes = 0;
+            g_compressor_store_score_bytes = 0;
+            g_embed_rows_bytes = 0;
+            g_router_selection_bytes = 0;
+            g_router_weight_sum_bytes = 0;
+            g_indexer_head_scores_bytes = 0;
+            g_indexer_topk_bytes = 0;
+            g_indexed_topk_bytes = 0;
+            g_q4_pair_rhs_f16_bytes = 0;
+            g_f16_round_scratch_bytes = 0;
+            g_raw_store_round_bytes = 0;
+            g_moe_gate_scratch_bytes = 0;
+            g_moe_down_scratch_bytes = 0;
+            g_moe_id_map_bytes = 0;
+            g_moe_q4_gate_slots_bytes = 0;
+            g_moe_q4_up_slots_bytes = 0;
+            g_moe_q4_down_slots_bytes = 0;
+            g_attn_out_group_ids_bytes = 0;
+        }
+        g_ds4_stream = 0;
         g_glm_flash_attn_mask_valid = 0;
         g_glm_flash_attn_mask_pos0 = 0;
         g_glm_flash_attn_mask_tokens = 0;
         g_glm_flash_attn_mask_cache_len = 0;
-        g_compressor_pool_kv_bytes = 0;
-        g_compressor_pool_score_bytes = 0;
-        g_compressor_pool_score_cont_bytes = 0;
-        g_compressor_pool_softmax_bytes = 0;
-        g_compressor_pool_product_bytes = 0;
-        g_compressor_store_ape_bytes = 0;
-        g_compressor_store_score_bytes = 0;
-        g_embed_rows_bytes = 0;
-        g_router_selection_bytes = 0;
-        g_router_weight_sum_bytes = 0;
-        g_indexer_head_scores_bytes = 0;
-        g_indexer_topk_bytes = 0;
-        g_indexed_topk_bytes = 0;
-        g_f16_round_scratch_bytes = 0;
-        g_raw_store_round_bytes = 0;
-        g_moe_gate_scratch_bytes = 0;
-        g_moe_down_scratch_bytes = 0;
-        g_moe_id_map_bytes = 0;
-        g_moe_q4_gate_slots_bytes = 0;
-        g_moe_q4_up_slots_bytes = 0;
-        g_moe_q4_down_slots_bytes = 0;
-        g_attn_out_group_ids_bytes = 0;
         g_model_wrap_count = 0;
         g_model_wrap_bytes = 0;
         g_model_wrap_max_bytes = 0;
@@ -11791,6 +13368,9 @@ void ds4_gpu_cleanup(void) {
         g_model_buffer_cache_over_limit = 0;
         ds4_gpu_model_residency_clear();
         ds4_gpu_model_views_clear();
+        g_dsv4_hc_producer_last_completion = nil;
+        g_dsv4_hc_producer_last_mix_buffer = nil;
+        g_dsv4_hc_producer_last_mix_offset = 0;
         [g_pipeline_cache removeAllObjects];
         g_pipeline_cache = nil;
         [g_q4_expert_layer_residency_cache removeAllObjects];
@@ -11799,11 +13379,19 @@ void ds4_gpu_cleanup(void) {
         g_q4_expert_table_cache = nil;
         [g_model_buffer_cache removeAllObjects];
         g_model_buffer_cache = nil;
-        g_transient_buffers = nil;
-        g_pending_cbs = nil;
+        for (int si = 0; si < DS4_GPU_MAX_STREAMS; si++) {
+            [g_transient_buffers_by_stream[si] removeAllObjects];
+            g_transient_buffers_by_stream[si] = nil;
+            g_pending_cbs_by_stream[si] = nil;
+            g_stream_last_cb[si] = nil;
+            g_stream_queues[si] = nil;
+        }
         g_library = nil;
         g_queue = nil;
         g_device = nil;
+        g_ds4_stream = 0;
+        g_iq2_stream_addr_mm_stats_enabled = 0;
+        ds4_gpu_iq2_stream_addr_mm_stats_reset();
         g_initialized = 0;
     }
 }
@@ -12419,13 +14007,44 @@ int ds4_gpu_embed_token_hc_tensor(
             return 0;
         }
 
+        const uint64_t src_row_bytes = (uint64_t)n_embd * sizeof(uint16_t);
+        const uint64_t token_rel = (uint64_t)token * src_row_bytes;
         uint64_t inner_offset = 0;
-        id<MTLBuffer> wbuf =
-            ds4_gpu_wrap_model_range(model_map,
-                                     model_size,
-                                     weight_offset,
-                                     weight_bytes,
-                                     &inner_offset);
+        uint32_t token_for_kernel = token;
+        id<MTLBuffer> wbuf = nil;
+        const bool exact_token_row =
+            getenv("DS4_METAL_DISABLE_TOKEN_EMBED_EXACT_VIEW") == NULL &&
+            (g_ssd_streaming_mode ||
+             getenv("DS4_METAL_ENABLE_TOKEN_EMBED_EXACT_VIEW") != NULL);
+        if (exact_token_row) {
+            if (token_rel > weight_bytes ||
+                src_row_bytes > weight_bytes - token_rel) {
+                fprintf(stderr,
+                        "ds4: Metal graph embedding token row is outside the mapped table\n");
+                return 0;
+            }
+            wbuf = ds4_gpu_wrap_model_exact_range(model_map,
+                                                  model_size,
+                                                  weight_offset + token_rel,
+                                                  src_row_bytes,
+                                                  &inner_offset);
+            if (wbuf) {
+                token_for_kernel = 0;
+            } else {
+                inner_offset = 0;
+                wbuf = ds4_gpu_wrap_model_range(model_map,
+                                                model_size,
+                                                weight_offset,
+                                                weight_bytes,
+                                                &inner_offset);
+            }
+        } else {
+            wbuf = ds4_gpu_wrap_model_range(model_map,
+                                            model_size,
+                                            weight_offset,
+                                            weight_bytes,
+                                            &inner_offset);
+        }
         if (!wbuf) return 0;
 
         const NSUInteger row_bytes = (NSUInteger)n_embd * sizeof(float);
@@ -12440,8 +14059,7 @@ int ds4_gpu_embed_token_hc_tensor(
         id<MTLCommandBuffer> cb = ds4_gpu_command_buffer(&owned);
         if (!cb) return 0;
 
-        const int32_t token_i32 = (int32_t)token;
-        const uint64_t src_row_bytes = (uint64_t)n_embd * sizeof(uint16_t);
+        const int32_t token_i32 = (int32_t)token_for_kernel;
         const uint64_t dst_row_bytes = (uint64_t)n_embd * sizeof(float);
         ds4_gpu_get_rows_args args = {
             .ne00t = (int32_t)n_embd,
@@ -12594,9 +14212,43 @@ int ds4_gpu_set_model_map_range(const void *model_map, uint64_t model_size, uint
             }
         }
 
+        const double t0 = ds4_gpu_now_ms();
+        const uint32_t old_view_count = g_model_view_count;
+        uint64_t mapped_model_size = 0;
+        ds4_gpu_model_residency_clear();
+        if (!ds4_gpu_add_model_view_range(model_map,
+                                          model_size,
+                                          map_offset,
+                                          map_size,
+                                          max_tensor_bytes,
+                                          false,
+                                          &mapped_model_size)) {
+            ds4_gpu_model_views_truncate(old_view_count);
+            if (!ds4_gpu_model_residency_request_views()) {
+                ds4_gpu_model_residency_clear();
+                ds4_gpu_model_views_clear();
+                ds4_gpu_model_map_state_invalidate();
+            }
+            return 0;
+        }
+        if (!ds4_gpu_finish_model_views(t0, mapped_model_size, map_offset)) {
+            ds4_gpu_model_residency_clear();
+            ds4_gpu_model_views_truncate(old_view_count);
+            if (!ds4_gpu_model_residency_request_views()) {
+                ds4_gpu_model_residency_clear();
+                ds4_gpu_model_views_clear();
+                ds4_gpu_model_map_state_invalidate();
+            }
+            return 0;
+        }
         ds4_gpu_model_residency_clear();
-        if (!ds4_gpu_map_model_views(model_map, model_size, map_offset, map_size, max_tensor_bytes)) {
+        ds4_gpu_model_views_remove_map_prefix(model_map,
+                                              model_size,
+                                              old_view_count);
+        if (!ds4_gpu_model_residency_request_views()) {
             ds4_gpu_model_residency_clear();
+            ds4_gpu_model_views_clear();
+            ds4_gpu_model_map_state_invalidate();
             return 0;
         }
         g_model_map_ptr = model_map;
@@ -12644,18 +14296,22 @@ int ds4_gpu_set_model_map_spans(
         const double t0 = ds4_gpu_now_ms();
         max_tensor_bytes = ds4_gpu_effective_model_max_tensor_bytes(model_size, max_tensor_bytes);
 
+        for (uint32_t i = 0; i < count; i++) {
+            if (offsets[i] > model_size || sizes[i] == 0 ||
+                sizes[i] > model_size - offsets[i]) {
+                fprintf(stderr,
+                        "ds4: Metal model span %u is outside the GGUF mapping\n",
+                        i);
+                return 0;
+            }
+        }
+
+        const uint32_t old_view_count = g_model_view_count;
         ds4_gpu_model_residency_clear();
-        ds4_gpu_model_views_clear();
 
         uint64_t mapped_total = 0;
         uint64_t first_offset = UINT64_MAX;
         for (uint32_t i = 0; i < count; i++) {
-            if (offsets[i] > model_size || sizes[i] == 0 || sizes[i] > model_size - offsets[i]) {
-                fprintf(stderr, "ds4: Metal model span %u is outside the GGUF mapping\n", i);
-                ds4_gpu_model_residency_clear();
-                ds4_gpu_model_views_clear();
-                return 0;
-            }
             if (offsets[i] < first_offset) first_offset = offsets[i];
             uint64_t effective_max = max_tensor_bytes;
             if (effective_max > sizes[i]) effective_max = sizes[i];
@@ -12666,14 +14322,33 @@ int ds4_gpu_set_model_map_spans(
                                               effective_max,
                                               true,
                                               &mapped_total)) {
-                ds4_gpu_model_residency_clear();
-                ds4_gpu_model_views_clear();
+                ds4_gpu_model_views_truncate(old_view_count);
+                if (!ds4_gpu_model_residency_request_views()) {
+                    ds4_gpu_model_residency_clear();
+                    ds4_gpu_model_views_clear();
+                    ds4_gpu_model_map_state_invalidate();
+                }
                 return 0;
             }
         }
         if (!ds4_gpu_finish_model_views(t0, mapped_total, first_offset)) {
+            ds4_gpu_model_residency_clear();
+            ds4_gpu_model_views_truncate(old_view_count);
+            if (!ds4_gpu_model_residency_request_views()) {
+                ds4_gpu_model_residency_clear();
+                ds4_gpu_model_views_clear();
+                ds4_gpu_model_map_state_invalidate();
+            }
+            return 0;
+        }
+        ds4_gpu_model_residency_clear();
+        ds4_gpu_model_views_remove_map_prefix(model_map,
+                                              model_size,
+                                              old_view_count);
+        if (!ds4_gpu_model_residency_request_views()) {
             ds4_gpu_model_residency_clear();
             ds4_gpu_model_views_clear();
+            ds4_gpu_model_map_state_invalidate();
             return 0;
         }
         g_model_map_ptr = model_map;
@@ -12695,8 +14370,116 @@ int ds4_gpu_set_model_map(const void *model_map, uint64_t model_size) {
     return ds4_gpu_set_model_map_range(model_map, model_size, 0, model_size, 0);
 }
 
+int ds4_gpu_prepare_support_model(const void *model_map,
+                                  uint64_t model_size,
+                                  uint64_t map_offset,
+                                  uint64_t map_size,
+                                  uint64_t max_tensor_bytes) {
+    /* Metal's model-view registry is keyed by mmap identity, so installing the
+     * support GGUF preserves the independently replaceable target views. */
+    const int ok = ds4_gpu_set_model_map_range(model_map,
+                                               model_size,
+                                               map_offset,
+                                               map_size,
+                                               max_tensor_bytes);
+    if (ok) {
+        g_support_model_map_ptr = model_map;
+        g_support_model_map_size = model_size;
+    }
+    return ok;
+}
+
+/* DS4_METAL_STREAMING_EXPERT_NOCACHE keeps its established all-phase
+ * behavior. DS4_METAL_STREAMING_EXPERT_PREFILL_NOCACHE uses the same second
+ * descriptor only for batched-prefill expert tasks, preserving the cached
+ * descriptor and readahead policy for steady decode. */
+static int ds4_gpu_stream_expert_nocache_requested(void) {
+    const char *env = getenv("DS4_METAL_STREAMING_EXPERT_NOCACHE");
+    return env != NULL && env[0] != '\0' && env[0] != '0';
+}
+
+static int ds4_gpu_stream_expert_prefill_nocache_auto(int fd) {
+    if (fd < 0 || !g_ssd_streaming_mode || g_glm_model_mode ||
+        !ds4_gpu_device_is_pre_m5_apple_silicon()) {
+        return 0;
+    }
+    struct stat st;
+    if (fstat(fd, &st) != 0 || st.st_size <= 0) return 0;
+    const uint64_t working_set = ds4_gpu_recommended_working_set_size();
+    if (working_set == 0 || working_set > UINT64_MAX / 2u) return 0;
+    return (uint64_t)st.st_size >= 2u * working_set;
+}
+
+static int ds4_gpu_stream_expert_prefill_nocache_resolve(int fd,
+                                                          int *automatic) {
+    if (automatic) *automatic = 0;
+    if (fd < 0 || !g_ssd_streaming_mode) return 0;
+    const int enable = ds4_gpu_env_bool(
+        "DS4_METAL_STREAMING_EXPERT_PREFILL_NOCACHE");
+    const int disable = ds4_gpu_env_bool(
+        "DS4_METAL_DISABLE_STREAMING_EXPERT_PREFILL_NOCACHE");
+    if (disable == 1 || enable == 0) return 0;
+    if (enable == 1) return 1;
+    const int auto_enabled =
+        ds4_gpu_stream_expert_prefill_nocache_auto(fd);
+    if (automatic) *automatic = auto_enabled;
+    return auto_enabled;
+}
+
 int ds4_gpu_set_model_fd(int fd) {
     g_model_fd = fd;
+    if (g_model_fd_nocache >= 0) {
+        close(g_model_fd_nocache);
+        g_model_fd_nocache = -1;
+    }
+    g_model_fd_nocache_prefill_auto = 0;
+    g_model_fd_nocache_all =
+        fd >= 0 && ds4_gpu_stream_expert_nocache_requested();
+    g_model_fd_nocache_prefill =
+        !g_model_fd_nocache_all &&
+        ds4_gpu_stream_expert_prefill_nocache_resolve(
+            fd, &g_model_fd_nocache_prefill_auto);
+    if (fd >= 0 &&
+        (g_model_fd_nocache_all || g_model_fd_nocache_prefill)) {
+        /* A dup() would share the file description (and its F_NOCACHE flag)
+         * with the mmap-backed descriptor, so reopen the model by path. */
+        char path[1024] = {0};
+        int nfd = -1;
+        struct stat source_stat;
+        struct stat reopened_stat;
+        int same_file = 0;
+        if (fcntl(fd, F_GETPATH, path) == 0) nfd = open(path, O_RDONLY);
+        if (nfd >= 0 &&
+            fstat(fd, &source_stat) == 0 &&
+            fstat(nfd, &reopened_stat) == 0 &&
+            source_stat.st_dev == reopened_stat.st_dev &&
+            source_stat.st_ino == reopened_stat.st_ino) {
+            same_file = 1;
+        } else if (nfd >= 0) {
+            errno = ESTALE;
+        }
+        if (same_file && fcntl(nfd, F_NOCACHE, 1) == 0) {
+            (void)fcntl(nfd, F_SETFD, FD_CLOEXEC);
+            g_model_fd_nocache = nfd;
+            fprintf(stderr,
+                    "ds4: Metal streaming expert %s preads on a F_NOCACHE "
+                    "descriptor; page cache reserved for dense weights\n",
+                    g_model_fd_nocache_all ? "all-phase" :
+                    g_model_fd_nocache_prefill_auto ?
+                        "batched-prefill automatic" :
+                        "batched-prefill");
+        } else {
+            const int saved_errno = errno;
+            if (nfd >= 0) close(nfd);
+            g_model_fd_nocache_all = 0;
+            g_model_fd_nocache_prefill = 0;
+            g_model_fd_nocache_prefill_auto = 0;
+            fprintf(stderr,
+                    "ds4: WARNING: F_NOCACHE expert descriptor unavailable (%s); "
+                    "using cached preads\n",
+                    strerror(saved_errno));
+        }
+    }
     return 1;
 }
 
@@ -12908,6 +14691,53 @@ int ds4_gpu_set_model_fd_for_map(int fd, const void *model_map) {
                                                DS4_GPU_EXACT_VIEW_OWNED);
 }
 
+/* Keep the modern target-model view policy. Only DSpark's separately mapped
+ * support GGUF gets exact Q8 views during SSD streaming. */
+static bool ds4_gpu_support_q8_decode_exact_views_enabled(
+        const void *model_map,
+        uint64_t    model_size) {
+    return model_map != NULL && model_size != 0 &&
+           g_ssd_streaming_mode &&
+           model_map == g_support_model_map_ptr &&
+           model_size == g_support_model_map_size &&
+           getenv("DS4_METAL_DISABLE_SUPPORT_Q8_DECODE_EXACT_VIEWS") == NULL;
+}
+
+static id<MTLBuffer> ds4_gpu_wrap_q8_decode_model_range(
+        const void *model_map,
+        uint64_t    model_size,
+        uint64_t    offset,
+        uint64_t    len,
+        uint64_t    n_tokens,
+        uint64_t   *inner_offset) {
+    const uint64_t exact_decode_max_mib =
+        ds4_gpu_env_u64("DS4_METAL_Q8_DECODE_EXACT_VIEW_MAX_MIB",
+                        1024u,
+                        1u,
+                        4096u);
+    const uint64_t exact_decode_max_bytes =
+        exact_decode_max_mib * 1024ull * 1024ull;
+    const bool exact_support_batch =
+        n_tokens >= 1u && n_tokens <= 6u &&
+        len <= exact_decode_max_bytes &&
+        ds4_gpu_support_q8_decode_exact_views_enabled(model_map, model_size);
+    if (exact_support_batch) {
+        id<MTLBuffer> exact =
+            ds4_gpu_wrap_model_exact_range(model_map,
+                                           model_size,
+                                           offset,
+                                           len,
+                                           inner_offset);
+        if (exact) return exact;
+        if (inner_offset) *inner_offset = 0;
+    }
+    return ds4_gpu_wrap_model_range(model_map,
+                                    model_size,
+                                    offset,
+                                    len,
+                                    inner_offset);
+}
+
 uint32_t ds4_gpu_stream_expert_cache_configured_count(void) {
     uint32_t budget = ds4_gpu_stream_expert_cache_configured_budget();
     if (budget > DS4_METAL_STREAM_EXPERT_CACHE_MAX_ENTRIES) {
@@ -12949,9 +14779,19 @@ static int ds4_gpu_stream_expert_cache_note_expert_size(
     const uint64_t bytes = gate_expert_bytes * 2ull + down_expert_bytes;
     if (g_stream_expert_cache_expert_bytes == 0) {
         g_stream_expert_cache_expert_bytes = bytes;
-        return 1;
     }
-    return bytes == g_stream_expert_cache_expert_bytes;
+    if (bytes != g_stream_expert_cache_expert_bytes) return 0;
+    if (g_stream_expert_cache_gate_class_bytes == 0 &&
+        g_stream_expert_cache_down_class_bytes == 0) {
+        g_stream_expert_cache_gate_class_bytes = gate_expert_bytes;
+        g_stream_expert_cache_down_class_bytes = down_expert_bytes;
+    } else if (g_stream_expert_cache_gate_class_bytes != gate_expert_bytes ||
+               g_stream_expert_cache_down_class_bytes != down_expert_bytes) {
+        /* Preserve the cache's historical total-size admission contract, but
+         * never use the IQ2-only live index for an ambiguous size class. */
+        g_stream_expert_cache_class_conflict = 1;
+    }
+    return 1;
 }
 
 static uint32_t ds4_gpu_stream_expert_cache_requested_budget(void) {
@@ -12972,6 +14812,53 @@ static uint32_t ds4_gpu_stream_expert_cache_configured_budget(void) {
     return budget;
 }
 
+static int ds4_gpu_exact_rows_persistent_env_enabled(const char *name) {
+    const char *value = name ? getenv(name) : NULL;
+    return value && value[0] && strcmp(value, "0") != 0;
+}
+
+/* Persistent private snapshots are the default when the cache can retain the
+ * full exact-row union. DISABLE is the A/B rollback arm and always wins;
+ * REQUIRE fails closed instead of silently using transient packed experts. */
+static int ds4_gpu_exact_rows_persistent_policy(
+        uint32_t configured_count,
+        uint32_t unique_count,
+        int      size_class_ok) {
+    const int disabled = ds4_gpu_exact_rows_persistent_env_enabled(
+            "DS4_METAL_DISABLE_EXACT_ROWS_PERSISTENT_CACHE");
+    const int required = ds4_gpu_exact_rows_persistent_env_enabled(
+            "DS4_METAL_REQUIRE_EXACT_ROWS_PERSISTENT_CACHE");
+    if (disabled) return required ? -1 : 0;
+
+    const int eligible = size_class_ok && unique_count != 0 &&
+                         configured_count >= unique_count;
+    if (eligible) return 1;
+    return required ? -1 : 0;
+}
+
+int ds4_gpu_test_exact_rows_persistent_policy(
+        uint32_t configured_count,
+        uint32_t unique_count,
+        int      size_class_ok) {
+    return ds4_gpu_exact_rows_persistent_policy(configured_count,
+                                                unique_count,
+                                                size_class_ok);
+}
+
+void ds4_gpu_test_exact_rows_persistent_report(
+        ds4_gpu_exact_rows_persistent_report *report) {
+    if (!report) return;
+    *report = (ds4_gpu_exact_rows_persistent_report) {
+        .persistent_calls = g_stream_expert_exact_persistent_calls,
+        .transient_calls = g_stream_expert_exact_transient_calls,
+        .persistent_fallbacks =
+            g_stream_expert_exact_persistent_fallbacks,
+        .persistent_failures = g_stream_expert_exact_persistent_failures,
+        .mapped_view_calls = g_stream_expert_exact_mapped_view_calls,
+        .max_unique = g_stream_expert_exact_max_unique,
+    };
+}
+
 static uint32_t ds4_gpu_stream_expert_cache_effective_cap(
         uint32_t layer,
         uint32_t n_total_expert,
@@ -13128,11 +15015,36 @@ static void ds4_gpu_stream_expert_timing_note_cache_class(
     g_stream_expert_timing_cache_missing_experts += missing;
 }
 
+static int ds4_gpu_stream_prefill_nocache_for_tokens(uint32_t n_tokens) {
+    return g_model_fd_nocache_prefill && n_tokens >= 32u;
+}
+
 static int ds4_gpu_stream_expert_readahead_enabled(void) {
+    /* All-phase F_NOCACHE never consumes these page-cache hints.  The
+     * prefill-only mode keeps them for decode and suppresses them explicitly
+     * in ds4_gpu_stream_prefill_expert_readahead_enabled(). */
     return g_ssd_streaming_mode &&
+           !g_model_fd_nocache_all &&
            getenv("DS4_METAL_DISABLE_STREAMING_EXPERT_READAHEAD") == NULL;
 }
 
+static int ds4_gpu_stream_prefill_expert_readahead_enabled(
+        uint32_t n_tokens) {
+    /*
+     * The selected-batch prefill loader immediately follows this hint with
+     * parallel pread() calls for the exact same ranges.  On macOS,
+     * F_RDADVISE is costly enough that issuing both operations serially slows
+     * time-to-first-token.  Keep the hint for asynchronous decode loaders,
+     * but require an explicit opt-in for this immediate-pread path so the old
+     * policy remains available for cold-storage A/B tests.
+     */
+    if (!ds4_gpu_stream_expert_readahead_enabled() ||
+        ds4_gpu_stream_prefill_nocache_for_tokens(n_tokens)) return 0;
+    if (n_tokens < 32u) return 1;
+    return ds4_gpu_env_bool(
+            "DS4_METAL_ENABLE_STREAMING_PREFILL_EXPERT_READAHEAD") > 0;
+}
+
 static void ds4_gpu_stream_expert_readahead_range(uint64_t offset, uint64_t len) {
     if (!ds4_gpu_stream_expert_readahead_enabled() || g_model_fd < 0 || len == 0) {
         return;
@@ -13171,6 +15083,7 @@ static void ds4_gpu_stream_expert_readahead_range(uint64_t offset, uint64_t len)
     uint64_t offset;
     uint64_t len;
     uint8_t *dst;
+    int prefer_nocache;
     uint64_t read_bytes;
     double ms;
     int ok;
@@ -13188,6 +15101,7 @@ static void ds4_gpu_stream_expert_readahead_range(uint64_t offset, uint64_t len)
     uint32_t source_slots[DS4_METAL_STREAM_EXPERT_CACHE_MAX_SELECTED];
     uint32_t n_loads;
     uint32_t n_tasks;
+    uint32_t n_tensor_tasks;
     uint64_t gate_expert_bytes;
     uint64_t down_expert_bytes;
     int32_t selected_ids[DS4_METAL_STREAM_EXPERT_CACHE_MAX_SELECTED];
@@ -13200,7 +15114,8 @@ static void ds4_gpu_stream_expert_readahead_range(uint64_t offset, uint64_t len)
     NSUInteger gate_inners[DS4_METAL_STREAM_EXPERT_CACHE_MAX_SELECTED];
     NSUInteger up_inners[DS4_METAL_STREAM_EXPERT_CACHE_MAX_SELECTED];
     NSUInteger down_inners[DS4_METAL_STREAM_EXPERT_CACHE_MAX_SELECTED];
-    ds4_gpu_stream_expert_pread_task tasks[DS4_METAL_STREAM_EXPERT_CACHE_MAX_SELECTED * 3u];
+    ds4_gpu_stream_expert_pread_task
+        tasks[DS4_METAL_STREAM_EXPERT_PENDING_MAX_TASKS];
     double start_ms;
     double prepare_ms;
 } ds4_gpu_stream_expert_pending_load;
@@ -13277,11 +15192,17 @@ static int ds4_gpu_stream_expert_pread_into(
         uint64_t  offset,
         uint64_t  len,
         uint8_t  *dst,
+        int       prefer_nocache,
         uint64_t *read_bytes,
         double   *ms_out) {
     if (read_bytes) *read_bytes = 0;
     if (ms_out) *ms_out = 0.0;
-    if (g_model_fd < 0 ||
+    const int use_nocache =
+        g_model_fd_nocache >= 0 &&
+        (g_model_fd_nocache_all ||
+         (g_model_fd_nocache_prefill && prefer_nocache));
+    const int fd = use_nocache ? g_model_fd_nocache : g_model_fd;
+    if (fd < 0 ||
         !dst ||
         len == 0 ||
         offset > (uint64_t)LLONG_MAX ||
@@ -13297,7 +15218,7 @@ static int ds4_gpu_stream_expert_pread_into(
         const size_t want = rem > (uint64_t)SSIZE_MAX ? (size_t)SSIZE_MAX : (size_t)rem;
         ssize_t nread;
         do {
-            nread = pread(g_model_fd, dst + pos, want, (off_t)(offset + pos));
+            nread = pread(fd, dst + pos, want, (off_t)(offset + pos));
         } while (nread < 0 && errno == EINTR);
         if (nread <= 0) {
             ok = 0;
@@ -13306,6 +15227,21 @@ static int ds4_gpu_stream_expert_pread_into(
         pos += (uint64_t)nread;
     }
     const double dt = ds4_gpu_now_ms() - t0;
+    if (use_nocache) {
+        __atomic_add_fetch(&g_stream_expert_pread_nocache_calls,
+                           1,
+                           __ATOMIC_RELAXED);
+        __atomic_add_fetch(&g_stream_expert_pread_nocache_bytes,
+                           pos,
+                           __ATOMIC_RELAXED);
+    } else {
+        __atomic_add_fetch(&g_stream_expert_pread_cached_calls,
+                           1,
+                           __ATOMIC_RELAXED);
+        __atomic_add_fetch(&g_stream_expert_pread_cached_bytes,
+                           pos,
+                           __ATOMIC_RELAXED);
+    }
     if (read_bytes) *read_bytes = pos;
     if (ms_out) *ms_out = dt;
     if (!ok || pos != len) {
@@ -13327,6 +15263,7 @@ static int ds4_gpu_stream_expert_pread_into(
         task->ok = ds4_gpu_stream_expert_pread_into(task->offset,
                                                     task->len,
                                                     task->dst,
+                                                    task->prefer_nocache,
                                                     &task->read_bytes,
                                                     &task->ms);
     }
@@ -13386,6 +15323,7 @@ static int ds4_gpu_stream_expert_pread_pool_enabled(void) {
             task->ok = ds4_gpu_stream_expert_pread_into(task->offset,
                                                         task->len,
                                                         task->dst,
+                                                        task->prefer_nocache,
                                                         &task->read_bytes,
                                                         &task->ms);
 
@@ -13540,7 +15478,7 @@ static void ds4_gpu_stream_expert_pread_pool_shutdown(void) {
     pthread_mutex_unlock(&g_stream_expert_pread_pool_mutex);
 }
 
-static int ds4_gpu_stream_expert_pread_tasks(
+static int ds4_gpu_stream_expert_pread_tasks_run(
         ds4_gpu_stream_expert_pread_task *tasks,
         uint32_t n_tasks,
         uint64_t *total_bytes,
@@ -13605,6 +15543,174 @@ static int ds4_gpu_stream_expert_pread_tasks(
     return ok;
 }
 
+/* DS4_METAL_STREAMING_EXPERT_PREAD_SPLIT: split every expert slab pread into
+ * N disjoint ranges read concurrently. Decode misses queue only a handful of
+ * slabs per layer (~4 experts x 3 slabs), while NVMe drives reach their
+ * random-read ceiling around ~24 requests in flight: splitting raises the
+ * queue depth at identical bytes. Boundaries are 16 KB aligned so F_NOCACHE
+ * never reads the same page from two jobs. Automatic mode uses 1 for small
+ * caches and 4 from 64 entries upward; an explicit value always wins. */
+static uint32_t ds4_gpu_stream_expert_pread_split(void) {
+    static int checked;
+    static uint32_t explicit_split;
+    if (!checked) {
+        const char *env = getenv("DS4_METAL_STREAMING_EXPERT_PREAD_SPLIT");
+        if (env) {
+            int v = atoi(env);
+            if (v < 1) v = 1;
+            if (v > DS4_METAL_STREAM_EXPERT_PREAD_MAX_SPLIT) {
+                v = DS4_METAL_STREAM_EXPERT_PREAD_MAX_SPLIT;
+            }
+            explicit_split = (uint32_t)v;
+        }
+        checked = 1;
+    }
+    if (explicit_split != 0) return explicit_split;
+
+    /* Four 16-KiB-aligned requests help once a larger expert cache can
+     * sustain enough concurrent misses. With the tiny automatic cache, the
+     * extra requests cost more than they overlap, so retain one read. Query
+     * the current budget so an engine reload can choose again. */
+    return ds4_gpu_stream_expert_cache_configured_count() >= 64u ? 4u : 1u;
+}
+
+/* Expand the persistent early-load task list in place. The pool keeps a
+ * pointer to this storage until pending_load_finish(), so a stack or temporary
+ * allocation would be unsafe here. If a future shape exceeds the fixed bound,
+ * preserve the original unsplit tasks instead of dropping any bytes. */
+static uint32_t ds4_gpu_stream_expert_pread_expand_tasks_bounded(
+        ds4_gpu_stream_expert_pread_task *tasks,
+        uint32_t n_tasks,
+        uint32_t capacity) {
+    const uint32_t split = ds4_gpu_stream_expert_pread_split();
+    const uint64_t min_split_len = 256u << 10;
+    const uint64_t align = 16u << 10;
+    enum {
+        max_tensor_tasks = DS4_METAL_STREAM_EXPERT_CACHE_MAX_SELECTED * 3u,
+    };
+    if (!tasks || n_tasks == 0 || split <= 1 ||
+        n_tasks > max_tensor_tasks || capacity < n_tasks) {
+        return n_tasks;
+    }
+
+    ds4_gpu_stream_expert_pread_task original[max_tensor_tasks];
+    memcpy(original, tasks, (size_t)n_tasks * sizeof(original[0]));
+
+    uint32_t out = 0;
+    for (uint32_t i = 0; i < n_tasks; i++) {
+        const uint64_t len = original[i].len;
+        if (len < min_split_len) {
+            if (out >= capacity) goto unsplit_fallback;
+            tasks[out++] = original[i];
+            continue;
+        }
+
+        uint64_t chunk = len / split + (len % split != 0);
+        if (chunk > UINT64_MAX - (align - 1u)) goto unsplit_fallback;
+        chunk = ((chunk + align - 1u) / align) * align;
+        uint64_t off = 0;
+        while (off < len) {
+            if (out >= capacity || original[i].offset > UINT64_MAX - off) {
+                goto unsplit_fallback;
+            }
+            const uint64_t part = len - off < chunk ? len - off : chunk;
+            tasks[out++] = (ds4_gpu_stream_expert_pread_task) {
+                .offset = original[i].offset + off,
+                .len = part,
+                .dst = original[i].dst + off,
+                .prefer_nocache = original[i].prefer_nocache,
+                .read_bytes = 0,
+                .ms = 0.0,
+                .ok = 0,
+            };
+            off += part;
+        }
+    }
+    return out;
+
+unsplit_fallback:
+    memcpy(tasks, original, (size_t)n_tasks * sizeof(original[0]));
+    return n_tasks;
+}
+
+static int ds4_gpu_stream_expert_pread_tasks(
+        ds4_gpu_stream_expert_pread_task *tasks,
+        uint32_t n_tasks,
+        uint64_t *total_bytes,
+        double *wall_ms) {
+    if (total_bytes) *total_bytes = 0;
+    if (wall_ms) *wall_ms = 0.0;
+    if (!tasks || n_tasks == 0) return 1;
+
+    const uint32_t split = ds4_gpu_stream_expert_pread_split();
+    const uint64_t min_split_len = 256u << 10;
+    if (split <= 1) {
+        return ds4_gpu_stream_expert_pread_tasks_run(tasks, n_tasks, total_bytes, wall_ms);
+    }
+
+    uint32_t n_sub = 0;
+    for (uint32_t i = 0; i < n_tasks; i++) {
+        n_sub += tasks[i].len >= min_split_len ? split : 1;
+    }
+    if (n_sub == n_tasks || n_sub < n_tasks) {
+        return ds4_gpu_stream_expert_pread_tasks_run(tasks, n_tasks, total_bytes, wall_ms);
+    }
+
+    ds4_gpu_stream_expert_pread_task *sub =
+        malloc((size_t)n_sub * sizeof(sub[0]));
+    uint32_t *owner = malloc((size_t)n_sub * sizeof(owner[0]));
+    if (!sub || !owner) {
+        free(owner);
+        free(sub);
+        return ds4_gpu_stream_expert_pread_tasks_run(tasks, n_tasks, total_bytes, wall_ms);
+    }
+
+    const uint64_t align = 16u << 10;
+    uint32_t w = 0;
+    for (uint32_t i = 0; i < n_tasks; i++) {
+        const uint64_t len = tasks[i].len;
+        if (len < min_split_len) {
+            sub[w] = tasks[i];
+            owner[w++] = i;
+            continue;
+        }
+        uint64_t chunk = (len + split - 1) / split;
+        chunk = (chunk + align - 1) / align * align;
+        uint64_t off = 0;
+        while (off < len) {
+            const uint64_t part = len - off < chunk ? len - off : chunk;
+            sub[w].offset = tasks[i].offset + off;
+            sub[w].len = part;
+            sub[w].dst = tasks[i].dst + off;
+            sub[w].prefer_nocache = tasks[i].prefer_nocache;
+            sub[w].read_bytes = 0;
+            sub[w].ms = 0.0;
+            sub[w].ok = 0;
+            owner[w++] = i;
+            off += part;
+        }
+    }
+
+    const int ok = ds4_gpu_stream_expert_pread_tasks_run(sub, w, total_bytes, wall_ms);
+
+    /* Fold the split results back so callers keep per-slab ok/bytes/ms. */
+    for (uint32_t i = 0; i < n_tasks; i++) {
+        tasks[i].ok = 1;
+        tasks[i].read_bytes = 0;
+        tasks[i].ms = 0.0;
+    }
+    for (uint32_t j = 0; j < w; j++) {
+        ds4_gpu_stream_expert_pread_task *t = &tasks[owner[j]];
+        if (!sub[j].ok) t->ok = 0;
+        t->read_bytes += sub[j].read_bytes;
+        if (sub[j].ms > t->ms) t->ms = sub[j].ms;
+    }
+
+    free(owner);
+    free(sub);
+    return ok;
+}
+
 static void ds4_gpu_stream_expert_cache_warn_mlock_failure(
         uint64_t failed_len,
         int      err) {
@@ -13872,6 +15978,36 @@ static void ds4_gpu_stream_expert_slab_push_free_slot(uint32_t slot) {
         slot;
 }
 
+/* Prepared misses own their buffers until install_loaded() publishes them.
+ * A failed exact batch must return slab slots or unlock standalone buffers,
+ * otherwise repeated I/O failures silently shrink the cache. */
+static void ds4_gpu_stream_expert_cache_release_prepared_buffers(
+        id<MTLBuffer> __strong *gate_buf,
+        id<MTLBuffer> __strong *up_buf,
+        id<MTLBuffer> __strong *down_buf,
+        NSUInteger gate_inner) {
+    if (!gate_buf || !up_buf || !down_buf) return;
+
+    id<MTLBuffer> gate = *gate_buf;
+    id<MTLBuffer> up = *up_buf;
+    id<MTLBuffer> down = *down_buf;
+    uint32_t slab_slot = UINT32_MAX;
+    if (gate && gate == up && gate == down &&
+        ds4_gpu_stream_expert_slab_slot_for_buffer(
+                gate, gate_inner, &slab_slot)) {
+        ds4_gpu_stream_expert_slab_push_free_slot(slab_slot);
+    } else {
+        ds4_gpu_stream_expert_unlock_explicit_buffer(gate);
+        if (up != gate) ds4_gpu_stream_expert_unlock_explicit_buffer(up);
+        if (down != gate && down != up) {
+            ds4_gpu_stream_expert_unlock_explicit_buffer(down);
+        }
+    }
+    *gate_buf = nil;
+    *up_buf = nil;
+    *down_buf = nil;
+}
+
 static int ds4_gpu_stream_expert_slab_lock_slot(uint32_t slot) {
     if (slot >= DS4_METAL_STREAM_EXPERT_CACHE_MAX_ENTRIES ||
         g_stream_expert_cache_slab_slot_locked[slot]) {
@@ -14020,15 +16156,19 @@ static int ds4_gpu_stream_expert_alloc_slab_slot(
     if (g_stream_expert_cache_free_slot_count != 0) {
         const uint32_t slot =
             g_stream_expert_cache_free_slots[--g_stream_expert_cache_free_slot_count];
-        return ds4_gpu_stream_expert_slab_slot_buffers(slot,
-                                                       gate_expert_bytes,
-                                                       down_expert_bytes,
-                                                       gate_buf,
-                                                       up_buf,
-                                                       down_buf,
-                                                       gate_inner,
-                                                       up_inner,
-                                                       down_inner);
+        if (ds4_gpu_stream_expert_slab_slot_buffers(slot,
+                                                    gate_expert_bytes,
+                                                    down_expert_bytes,
+                                                    gate_buf,
+                                                    up_buf,
+                                                    down_buf,
+                                                    gate_inner,
+                                                    up_inner,
+                                                    down_inner)) {
+            return 1;
+        }
+        ds4_gpu_stream_expert_slab_push_free_slot(slot);
+        return 0;
     }
 
     uint32_t slab = g_stream_expert_cache_slab_count;
@@ -14072,6 +16212,9 @@ static int ds4_gpu_stream_expert_alloc_slab_slot(
 
         slab = g_stream_expert_cache_slab_count++;
         g_stream_expert_cache_slabs[slab] = slab_buffer;
+        ds4_gpu_atomic_u64_add_sat(
+            &g_stream_expert_cache_slab_allocated_bytes,
+            (uint64_t)[slab_buffer length]);
         g_stream_expert_cache_slab_start_slot[slab] =
             g_stream_expert_cache_slab_total_slots;
         g_stream_expert_cache_slab_slot_count[slab] = slots;
@@ -14082,15 +16225,19 @@ static int ds4_gpu_stream_expert_alloc_slab_slot(
     const uint32_t local_slot = g_stream_expert_cache_slab_slots_used[slab]++;
     const uint32_t slot =
         g_stream_expert_cache_slab_start_slot[slab] + local_slot;
-    return ds4_gpu_stream_expert_slab_slot_buffers(slot,
-                                                   gate_expert_bytes,
-                                                   down_expert_bytes,
-                                                   gate_buf,
-                                                   up_buf,
-                                                   down_buf,
-                                                   gate_inner,
-                                                   up_inner,
-                                                   down_inner);
+    if (ds4_gpu_stream_expert_slab_slot_buffers(slot,
+                                                gate_expert_bytes,
+                                                down_expert_bytes,
+                                                gate_buf,
+                                                up_buf,
+                                                down_buf,
+                                                gate_inner,
+                                                up_inner,
+                                                down_inner)) {
+        return 1;
+    }
+    ds4_gpu_stream_expert_slab_push_free_slot(slot);
+    return 0;
 }
 
 static uint64_t ds4_gpu_stream_expert_buffer_object_count(
@@ -14301,6 +16448,11 @@ static void ds4_gpu_stream_expert_cache_note_decode_token(void) {
     ds4_gpu_stream_expert_cache_maybe_decay_route_hotness();
 }
 
+static int ds4_gpu_m1_iq2_mid_only_enabled(void) {
+    return ds4_gpu_device_is_m1_apple_silicon() &&
+           ds4_gpu_env_bool("DS4_METAL_DISABLE_M1_IQ2_MID_ONLY") != 1;
+}
+
 static int ds4_gpu_stream_compact_addr_requested(void) {
     return g_ssd_streaming_mode &&
            getenv("DS4_METAL_ENABLE_STREAMING_COMPACT_ADDR") != NULL &&
@@ -14383,6 +16535,7 @@ static int ds4_gpu_stream_prefill_batch_selected_addr_enabled(
         uint32_t gate_type,
         uint32_t down_type) {
     if (!g_ssd_streaming_mode ||
+        g_tp_split_world != 1 ||
         n_tokens <= 1 ||
         n_total_expert == 0 ||
         n_expert != 6 ||
@@ -14395,10 +16548,12 @@ static int ds4_gpu_stream_prefill_batch_selected_addr_enabled(
         getenv("DS4_METAL_DISABLE_ROUTED_PAIR_SWIGLU_FUSION") != NULL) {
         return 0;
     }
-    /* All unique experts for one layer must fit simultaneously because the
-     * address-table kernels consume them in one dispatch. Once the global
-     * cache fills, preparation reuses entries owned by other layers. */
-    if (ds4_gpu_stream_expert_cache_configured_count() < n_total_expert) {
+    /* Ordinary prefill needs enough persistent slots to prepare a complete
+     * address table without evicting protected entries.  DSpark verification
+     * is only 2..5 rows: when that set exceeds the cache, preparation packs
+     * just its unique experts into transient buffers instead. */
+    if (ds4_gpu_stream_expert_cache_configured_count() < n_total_expert &&
+        n_tokens > 5u) {
         return 0;
     }
     if (getenv("DS4_METAL_ENABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR") != NULL) {
@@ -14699,9 +16854,9 @@ static int ds4_gpu_stream_expert_cache_set_addr_slot(
 
 static int ds4_gpu_stream_expert_cache_addr_buffers(
         uint32_t       layer,
-        id<MTLBuffer> *gate,
-        id<MTLBuffer> *up,
-        id<MTLBuffer> *down) {
+        id<MTLBuffer> __strong *gate,
+        id<MTLBuffer> __strong *up,
+        id<MTLBuffer> __strong *down) {
     if (!ds4_gpu_stream_expert_cache_ensure_addr_buffers(layer)) return 0;
     if (gate) *gate = g_stream_expert_cache_gate_addr_buffers[layer];
     if (up) *up = g_stream_expert_cache_up_addr_buffers[layer];
@@ -14711,6 +16866,88 @@ static int ds4_gpu_stream_expert_cache_addr_buffers(
            g_stream_expert_cache_down_addr_buffers[layer];
 }
 
+/* Exact-row tails cannot borrow mutable cache tables. Publish an immutable
+ * layer-local snapshot after every selected expert is resident and pinned. */
+static int ds4_gpu_stream_expert_exact_snapshot_addr_buffers(
+        uint32_t n_total_expert,
+        const int32_t *unique_ids,
+        uint32_t unique_count,
+        ds4_gpu_stream_expert_cache_entry * const *entries,
+        id<MTLBuffer> __strong *gate_out,
+        id<MTLBuffer> __strong *up_out,
+        id<MTLBuffer> __strong *down_out) {
+    if (gate_out) *gate_out = nil;
+    if (up_out) *up_out = nil;
+    if (down_out) *down_out = nil;
+    if (!unique_ids || !entries || !gate_out || !up_out || !down_out ||
+        n_total_expert == 0 ||
+        n_total_expert > DS4_METAL_STREAM_EXPERT_CACHE_MAX_EXPERT ||
+        unique_count == 0 || unique_count > n_total_expert) {
+        return 0;
+    }
+
+    const NSUInteger bytes =
+        (NSUInteger)n_total_expert * sizeof(uint64_t);
+    id<MTLBuffer> gate = ds4_gpu_new_transient_buffer(
+            bytes, "ds4_exact_rows_gate_addr_snapshot");
+    id<MTLBuffer> up = ds4_gpu_new_transient_buffer(
+            bytes, "ds4_exact_rows_up_addr_snapshot");
+    id<MTLBuffer> down = ds4_gpu_new_transient_buffer(
+            bytes, "ds4_exact_rows_down_addr_snapshot");
+    if (!gate || !up || !down) return 0;
+
+    uint64_t *gate_addrs = (uint64_t *)[gate contents];
+    uint64_t *up_addrs = (uint64_t *)[up contents];
+    uint64_t *down_addrs = (uint64_t *)[down contents];
+    if (!gate_addrs || !up_addrs || !down_addrs) return 0;
+    memset(gate_addrs, 0, bytes);
+    memset(up_addrs, 0, bytes);
+    memset(down_addrs, 0, bytes);
+
+    for (uint32_t u = 0; u < unique_count; u++) {
+        const int32_t selected_id = unique_ids[u];
+        ds4_gpu_stream_expert_cache_entry *entry = entries[u];
+        if (selected_id < 0 ||
+            (uint32_t)selected_id >= n_total_expert ||
+            !entry || !entry->valid ||
+            !entry->gate_buffer || !entry->up_buffer ||
+            !entry->down_buffer) {
+            return 0;
+        }
+        const uint64_t gate_addr =
+            ds4_gpu_buffer_address(entry->gate_buffer, entry->gate_inner);
+        const uint64_t up_addr =
+            ds4_gpu_buffer_address(entry->up_buffer, entry->up_inner);
+        const uint64_t down_addr =
+            ds4_gpu_buffer_address(entry->down_buffer, entry->down_inner);
+        if (gate_addr == 0 || up_addr == 0 || down_addr == 0) return 0;
+
+        const uint32_t expert = (uint32_t)selected_id;
+        if ((gate_addrs[expert] != 0 &&
+             gate_addrs[expert] != gate_addr) ||
+            (up_addrs[expert] != 0 && up_addrs[expert] != up_addr) ||
+            (down_addrs[expert] != 0 &&
+             down_addrs[expert] != down_addr)) {
+            fprintf(stderr,
+                    "ds4: Metal exact-row snapshot found inconsistent "
+                    "duplicate expert %u\n",
+                    expert);
+            return 0;
+        }
+        gate_addrs[expert] = gate_addr;
+        up_addrs[expert] = up_addr;
+        down_addrs[expert] = down_addr;
+    }
+
+    [gate didModifyRange:NSMakeRange(0, bytes)];
+    [up didModifyRange:NSMakeRange(0, bytes)];
+    [down didModifyRange:NSMakeRange(0, bytes)];
+    *gate_out = gate;
+    *up_out = up;
+    *down_out = down;
+    return 1;
+}
+
 static void ds4_gpu_stream_full_expert_addr_clear_layer(uint32_t layer) {
     if (layer >= DS4_METAL_STREAM_EXPERT_CACHE_MAX_LAYER) return;
     ds4_gpu_stream_expert_cache_entry *e = &g_stream_full_expert_addr_entry[layer];
@@ -14992,6 +17229,406 @@ static int ds4_gpu_stream_expert_cache_validate_selected(
     return 1;
 }
 
+enum {
+    DS4_METAL_IQ2_LIVE_GATE_EXPERT_BYTES = 2162688u,
+    DS4_METAL_IQ2_LIVE_DOWN_EXPERT_BYTES = 2752512u,
+    DS4_METAL_IQ2_LIVE_TOTAL_EXPERT_BYTES = 7077888u,
+};
+
+/* Dense resident IDs avoid the historical 80 x 384 victim scan.  The
+ * optimization is intentionally limited to the measured production IQ2
+ * cache shape; every failure falls back to the authoritative cache matrix. */
+static uint32_t *g_stream_expert_cache_live_ids;
+static uint32_t *g_stream_expert_cache_live_pos_plus_one;
+static uint32_t g_stream_expert_cache_live_count;
+static int g_stream_expert_cache_live_broken;
+static int g_stream_expert_cache_live_warned;
+static int g_stream_expert_cache_live_selftest_checked;
+static int g_stream_expert_cache_live_rebuild_required;
+
+static int ds4_gpu_stream_expert_cache_live_policy(
+        int ssd_streaming,
+        uint64_t gate_expert_bytes,
+        uint64_t down_expert_bytes,
+        int enable,
+        int disable) {
+    if (!ssd_streaming ||
+        gate_expert_bytes != DS4_METAL_IQ2_LIVE_GATE_EXPERT_BYTES ||
+        down_expert_bytes != DS4_METAL_IQ2_LIVE_DOWN_EXPERT_BYTES ||
+        gate_expert_bytes > (UINT64_MAX - down_expert_bytes) / 2u ||
+        2u * gate_expert_bytes + down_expert_bytes !=
+            DS4_METAL_IQ2_LIVE_TOTAL_EXPERT_BYTES) {
+        return 0;
+    }
+    if (disable == 1) return 0;
+    if (enable >= 0) return enable == 1;
+    return 1;
+}
+
+int ds4_gpu_test_stream_expert_live_index_policy(
+        int ssd_streaming,
+        uint64_t gate_expert_bytes,
+        uint64_t down_expert_bytes,
+        int enable,
+        int disable) {
+    return ds4_gpu_stream_expert_cache_live_policy(ssd_streaming,
+                                                    gate_expert_bytes,
+                                                    down_expert_bytes,
+                                                    enable,
+                                                    disable);
+}
+
+static int ds4_gpu_stream_expert_cache_live_eligible(void) {
+    return !g_stream_expert_cache_class_conflict &&
+           g_stream_expert_cache_expert_bytes ==
+               DS4_METAL_IQ2_LIVE_TOTAL_EXPERT_BYTES &&
+           ds4_gpu_stream_expert_cache_live_policy(
+                   g_ssd_streaming_mode,
+                   g_stream_expert_cache_gate_class_bytes,
+                   g_stream_expert_cache_down_class_bytes,
+                   -1,
+                   0);
+}
+
+static int ds4_gpu_stream_expert_cache_live_requested(void) {
+    if (g_stream_expert_cache_class_conflict ||
+        g_stream_expert_cache_expert_bytes !=
+            DS4_METAL_IQ2_LIVE_TOTAL_EXPERT_BYTES) {
+        return 0;
+    }
+    return ds4_gpu_stream_expert_cache_live_policy(
+            g_ssd_streaming_mode,
+            g_stream_expert_cache_gate_class_bytes,
+            g_stream_expert_cache_down_class_bytes,
+            ds4_gpu_env_bool("DS4_METAL_ENABLE_STREAMING_EXPERT_LIVE_INDEX"),
+            ds4_gpu_env_bool("DS4_METAL_DISABLE_STREAMING_EXPERT_LIVE_INDEX"));
+}
+
+static int ds4_gpu_stream_expert_cache_live_add_raw(
+        uint32_t *ids,
+        uint32_t *pos_plus_one,
+        uint32_t capacity,
+        uint32_t *count,
+        uint32_t id) {
+    if (!ids || !pos_plus_one || !count || id >= capacity) return 0;
+    if (pos_plus_one[id] != 0) return 1;
+    if (*count >= capacity) return 0;
+    ids[*count] = id;
+    pos_plus_one[id] = *count + 1u;
+    (*count)++;
+    return 1;
+}
+
+static int ds4_gpu_stream_expert_cache_live_remove_raw(
+        uint32_t *ids,
+        uint32_t *pos_plus_one,
+        uint32_t capacity,
+        uint32_t *count,
+        uint32_t id) {
+    if (!ids || !pos_plus_one || !count || id >= capacity ||
+        pos_plus_one[id] == 0 || *count == 0) {
+        return 0;
+    }
+    const uint32_t pos = pos_plus_one[id] - 1u;
+    if (pos >= *count) return 0;
+    const uint32_t last_pos = *count - 1u;
+    const uint32_t moved = ids[last_pos];
+    if (moved >= capacity) return 0;
+    ids[pos] = moved;
+    pos_plus_one[moved] = pos + 1u;
+    pos_plus_one[id] = 0;
+    *count = last_pos;
+    return 1;
+}
+
+static int ds4_gpu_stream_expert_cache_live_selftest(void) {
+    uint32_t ids[8] = {0};
+    uint32_t pos[8] = {0};
+    uint32_t count = 0;
+    if (!ds4_gpu_stream_expert_cache_live_add_raw(ids, pos, 8, &count, 1) ||
+        !ds4_gpu_stream_expert_cache_live_add_raw(ids, pos, 8, &count, 3) ||
+        !ds4_gpu_stream_expert_cache_live_add_raw(ids, pos, 8, &count, 5) ||
+        !ds4_gpu_stream_expert_cache_live_add_raw(ids, pos, 8, &count, 3) ||
+        count != 3 || pos[1] != 1 || pos[3] != 2 || pos[5] != 3) {
+        return 0;
+    }
+    if (!ds4_gpu_stream_expert_cache_live_remove_raw(ids, pos, 8, &count, 3) ||
+        count != 2 || pos[3] != 0 || ids[1] != 5 || pos[5] != 2) {
+        return 0;
+    }
+    return ds4_gpu_stream_expert_cache_live_remove_raw(
+               ids, pos, 8, &count, 1) &&
+           ds4_gpu_stream_expert_cache_live_remove_raw(
+               ids, pos, 8, &count, 5) &&
+           count == 0 && pos[1] == 0 && pos[5] == 0;
+}
+
+static void ds4_gpu_stream_expert_cache_live_mark_broken(
+        const char *reason) {
+    if (!g_stream_expert_cache_live_broken) {
+        g_stream_expert_live_index_fallbacks++;
+    }
+    g_stream_expert_cache_live_broken = 1;
+    if (!g_stream_expert_cache_live_warned) {
+        fprintf(stderr,
+                "ds4: Metal IQ2 streaming expert live index disabled (%s); using full cache scan\n",
+                reason ? reason : "invariant failure");
+        g_stream_expert_cache_live_warned = 1;
+    }
+}
+
+static int ds4_gpu_stream_expert_cache_live_rebuild(void) {
+    const uint32_t capacity = DS4_METAL_STREAM_EXPERT_CACHE_MAX_ENTRIES;
+    memset(g_stream_expert_cache_live_pos_plus_one,
+           0,
+           (size_t)capacity *
+               sizeof(g_stream_expert_cache_live_pos_plus_one[0]));
+    g_stream_expert_cache_live_count = 0;
+    for (uint32_t layer = 0;
+         layer < DS4_METAL_STREAM_EXPERT_CACHE_MAX_LAYER;
+         layer++) {
+        for (uint32_t expert = 0;
+             expert < DS4_METAL_STREAM_EXPERT_CACHE_MAX_EXPERT;
+             expert++) {
+            if (!g_stream_expert_cache[layer][expert].valid) continue;
+            const uint32_t id =
+                layer * DS4_METAL_STREAM_EXPERT_CACHE_MAX_EXPERT + expert;
+            if (!ds4_gpu_stream_expert_cache_live_add_raw(
+                        g_stream_expert_cache_live_ids,
+                        g_stream_expert_cache_live_pos_plus_one,
+                        capacity,
+                        &g_stream_expert_cache_live_count,
+                        id)) {
+                return 0;
+            }
+        }
+    }
+    g_stream_expert_cache_live_rebuild_required = 0;
+    return 1;
+}
+
+static int ds4_gpu_stream_expert_cache_live_ensure(void) {
+    if (!ds4_gpu_stream_expert_cache_live_requested() ||
+        g_stream_expert_cache_live_broken) {
+        return 0;
+    }
+    if ((g_test_flags & DS4_GPU_TEST_STREAMING_LIVE_INDEX_FAILURE) != 0u) {
+        ds4_gpu_stream_expert_cache_live_mark_broken("test fault injection");
+        return 0;
+    }
+    if (!g_stream_expert_cache_live_selftest_checked) {
+        g_stream_expert_cache_live_selftest_checked = 1;
+        if (!ds4_gpu_stream_expert_cache_live_selftest()) {
+            ds4_gpu_stream_expert_cache_live_mark_broken("selftest failure");
+            return 0;
+        }
+    }
+
+    const size_t capacity = DS4_METAL_STREAM_EXPERT_CACHE_MAX_ENTRIES;
+    if (!g_stream_expert_cache_live_ids) {
+        g_stream_expert_cache_live_ids =
+            calloc(capacity, sizeof(g_stream_expert_cache_live_ids[0]));
+        g_stream_expert_cache_live_pos_plus_one =
+            calloc(capacity,
+                   sizeof(g_stream_expert_cache_live_pos_plus_one[0]));
+        if (!g_stream_expert_cache_live_ids ||
+            !g_stream_expert_cache_live_pos_plus_one) {
+            free(g_stream_expert_cache_live_ids);
+            free(g_stream_expert_cache_live_pos_plus_one);
+            g_stream_expert_cache_live_ids = NULL;
+            g_stream_expert_cache_live_pos_plus_one = NULL;
+            ds4_gpu_stream_expert_cache_live_mark_broken(
+                    "allocation failure");
+            return 0;
+        }
+        g_stream_expert_cache_live_rebuild_required = 1;
+    }
+    if (g_stream_expert_cache_live_rebuild_required &&
+        !ds4_gpu_stream_expert_cache_live_rebuild()) {
+        ds4_gpu_stream_expert_cache_live_mark_broken("rebuild failure");
+        return 0;
+    }
+    return 1;
+}
+
+static int ds4_gpu_stream_expert_cache_live_ready(void) {
+    if (!ds4_gpu_stream_expert_cache_live_ensure()) return 0;
+    if (g_stream_expert_cache_live_count !=
+        g_stream_expert_cache_entry_count) {
+        ds4_gpu_stream_expert_cache_live_mark_broken("count invariant");
+        return 0;
+    }
+    for (uint32_t pos = 0; pos < g_stream_expert_cache_live_count; pos++) {
+        const uint32_t id = g_stream_expert_cache_live_ids[pos];
+        if (id >= DS4_METAL_STREAM_EXPERT_CACHE_MAX_ENTRIES ||
+            g_stream_expert_cache_live_pos_plus_one[id] != pos + 1u) {
+            ds4_gpu_stream_expert_cache_live_mark_broken(
+                    "position invariant");
+            return 0;
+        }
+        const uint32_t layer =
+            id / DS4_METAL_STREAM_EXPERT_CACHE_MAX_EXPERT;
+        const uint32_t expert =
+            id % DS4_METAL_STREAM_EXPERT_CACHE_MAX_EXPERT;
+        if (!g_stream_expert_cache[layer][expert].valid) {
+            ds4_gpu_stream_expert_cache_live_mark_broken(
+                    "resident invariant");
+            return 0;
+        }
+    }
+    return 1;
+}
+
+static void ds4_gpu_stream_expert_cache_live_insert(
+        uint32_t layer,
+        uint32_t expert) {
+    if (!ds4_gpu_stream_expert_cache_live_requested()) {
+        if (g_stream_expert_cache_live_ids) {
+            g_stream_expert_cache_live_rebuild_required = 1;
+        }
+        return;
+    }
+    if (!ds4_gpu_stream_expert_cache_live_ensure()) return;
+    const uint32_t id =
+        layer * DS4_METAL_STREAM_EXPERT_CACHE_MAX_EXPERT + expert;
+    if (!ds4_gpu_stream_expert_cache_live_add_raw(
+                g_stream_expert_cache_live_ids,
+                g_stream_expert_cache_live_pos_plus_one,
+                DS4_METAL_STREAM_EXPERT_CACHE_MAX_ENTRIES,
+                &g_stream_expert_cache_live_count,
+                id)) {
+        ds4_gpu_stream_expert_cache_live_mark_broken("insert failure");
+        return;
+    }
+    g_stream_expert_live_index_inserts++;
+}
+
+static void ds4_gpu_stream_expert_cache_live_remove(
+        uint32_t layer,
+        uint32_t expert) {
+    if (!ds4_gpu_stream_expert_cache_live_requested()) {
+        if (g_stream_expert_cache_live_ids) {
+            g_stream_expert_cache_live_rebuild_required = 1;
+        }
+        return;
+    }
+    if (!ds4_gpu_stream_expert_cache_live_ensure()) return;
+    const uint32_t id =
+        layer * DS4_METAL_STREAM_EXPERT_CACHE_MAX_EXPERT + expert;
+    if (!ds4_gpu_stream_expert_cache_live_remove_raw(
+                g_stream_expert_cache_live_ids,
+                g_stream_expert_cache_live_pos_plus_one,
+                DS4_METAL_STREAM_EXPERT_CACHE_MAX_ENTRIES,
+                &g_stream_expert_cache_live_count,
+                id)) {
+        ds4_gpu_stream_expert_cache_live_mark_broken("remove failure");
+        return;
+    }
+    g_stream_expert_live_index_removes++;
+}
+
+static void ds4_gpu_stream_expert_cache_live_reset(void) {
+    g_stream_expert_cache_live_count = 0;
+    g_stream_expert_cache_live_broken = 0;
+    g_stream_expert_cache_live_warned = 0;
+    g_stream_expert_cache_live_rebuild_required = 0;
+    if (g_stream_expert_cache_live_pos_plus_one) {
+        memset(g_stream_expert_cache_live_pos_plus_one,
+               0,
+               (size_t)DS4_METAL_STREAM_EXPERT_CACHE_MAX_ENTRIES *
+                   sizeof(g_stream_expert_cache_live_pos_plus_one[0]));
+    }
+}
+
+static void ds4_gpu_stream_expert_cache_live_release(void) {
+    free(g_stream_expert_cache_live_ids);
+    free(g_stream_expert_cache_live_pos_plus_one);
+    g_stream_expert_cache_live_ids = NULL;
+    g_stream_expert_cache_live_pos_plus_one = NULL;
+    g_stream_expert_cache_live_count = 0;
+    g_stream_expert_cache_live_broken = 0;
+    g_stream_expert_cache_live_warned = 0;
+    g_stream_expert_cache_live_selftest_checked = 0;
+    g_stream_expert_cache_live_rebuild_required = 0;
+}
+
+typedef struct {
+    uint32_t cursor;
+    uint32_t limit;
+    int live;
+} ds4_gpu_stream_expert_cache_iterator;
+
+static ds4_gpu_stream_expert_cache_iterator
+ds4_gpu_stream_expert_cache_iterator_begin(void) {
+    ds4_gpu_stream_expert_cache_iterator it = {
+        .cursor = 0,
+        .limit = DS4_METAL_STREAM_EXPERT_CACHE_MAX_ENTRIES,
+        .live = 0,
+    };
+    if (ds4_gpu_stream_expert_cache_live_requested() &&
+        ds4_gpu_stream_expert_cache_live_ready()) {
+        it.limit = g_stream_expert_cache_live_count;
+        it.live = 1;
+        g_stream_expert_live_index_scans++;
+    }
+    return it;
+}
+
+static int ds4_gpu_stream_expert_cache_iterator_next(
+        ds4_gpu_stream_expert_cache_iterator *it,
+        uint32_t *layer,
+        uint32_t *expert) {
+    if (!it || !layer || !expert || it->cursor >= it->limit) return 0;
+    const uint32_t pos = it->cursor++;
+    const uint32_t id =
+        it->live ? g_stream_expert_cache_live_ids[pos] : pos;
+    if (it->live) g_stream_expert_live_index_entries++;
+    *layer = id / DS4_METAL_STREAM_EXPERT_CACHE_MAX_EXPERT;
+    *expert = id % DS4_METAL_STREAM_EXPERT_CACHE_MAX_EXPERT;
+    return 1;
+}
+
+void ds4_gpu_test_stream_expert_live_index_report(
+        ds4_gpu_stream_expert_live_index_report *report) {
+    if (!report) return;
+    uint64_t resident_hash = UINT64_C(1469598103934665603);
+    for (uint32_t layer = 0;
+         layer < DS4_METAL_STREAM_EXPERT_CACHE_MAX_LAYER;
+         layer++) {
+        for (uint32_t expert = 0;
+             expert < DS4_METAL_STREAM_EXPERT_CACHE_MAX_EXPERT;
+             expert++) {
+            if (!g_stream_expert_cache[layer][expert].valid) continue;
+            const uint64_t id =
+                (uint64_t)layer * DS4_METAL_STREAM_EXPERT_CACHE_MAX_EXPERT +
+                expert;
+            resident_hash ^= id + UINT64_C(0x9e3779b97f4a7c15);
+            resident_hash *= UINT64_C(1099511628211);
+        }
+    }
+    *report = (ds4_gpu_stream_expert_live_index_report) {
+        .scans = g_stream_expert_live_index_scans,
+        .entries = g_stream_expert_live_index_entries,
+        .fallbacks = g_stream_expert_live_index_fallbacks,
+        .inserts = g_stream_expert_live_index_inserts,
+        .removes = g_stream_expert_live_index_removes,
+        .reuse_scan_calls = g_stream_expert_timing_reuse_scan_calls,
+        .reuse_scan_entries = g_stream_expert_timing_reuse_scan_entries,
+        .resident_hash = resident_hash,
+        .live_count = g_stream_expert_cache_live_count,
+        .cache_entries = g_stream_expert_cache_entry_count,
+        .eligible = (uint32_t)ds4_gpu_stream_expert_cache_live_eligible(),
+        .active = (uint32_t)(
+            ds4_gpu_stream_expert_cache_live_requested() &&
+            !g_stream_expert_cache_live_broken &&
+            !g_stream_expert_cache_live_rebuild_required &&
+            g_stream_expert_cache_live_ids != NULL &&
+            g_stream_expert_cache_live_count ==
+                g_stream_expert_cache_entry_count),
+        .broken = (uint32_t)(g_stream_expert_cache_live_broken != 0),
+    };
+}
+
 static void ds4_gpu_stream_expert_cache_clear_entry_internal(
         uint32_t                                  layer,
         uint32_t                                  expert,
@@ -15017,6 +17654,8 @@ static void ds4_gpu_stream_expert_cache_clear_entry_internal(
         return;
     }
 
+    ds4_gpu_stream_expert_cache_live_remove(layer, expert);
+
     const uint64_t bytes = e->logical_bytes;
     ds4_gpu_stream_expert_evict_dontneed_range(e->model_map,
                                                e->model_size,
@@ -15100,6 +17739,7 @@ static void ds4_gpu_stream_expert_cache_clear_entry(
 }
 
 static void ds4_gpu_stream_expert_cache_clear_all(int reset_stats) {
+    ds4_gpu_stream_expert_exact_rows_clear();
     ds4_gpu_stream_expert_pending_load_clear();
     g_stream_expert_cache_done_seq = g_stream_expert_cache_cb_seq;
     g_stream_expert_cache_batch_seq = 0;
@@ -15127,6 +17767,7 @@ static void ds4_gpu_stream_expert_cache_clear_all(int reset_stats) {
     }
     g_stream_expert_cache_bytes = 0;
     g_stream_expert_cache_entry_count = 0;
+    ds4_gpu_stream_expert_cache_live_reset();
     for (uint32_t i = 0; i < g_stream_expert_cache_slab_count; i++) {
         g_stream_expert_cache_slabs[i] = nil;
         g_stream_expert_cache_slab_start_slot[i] = 0;
@@ -15135,6 +17776,9 @@ static void ds4_gpu_stream_expert_cache_clear_all(int reset_stats) {
     }
     g_stream_expert_cache_slab_count = 0;
     g_stream_expert_cache_slab_total_slots = 0;
+    __atomic_store_n(&g_stream_expert_cache_slab_allocated_bytes,
+                     0,
+                     __ATOMIC_RELEASE);
     g_stream_expert_cache_free_slot_count = 0;
     g_stream_expert_cache_slab_slot_bytes = 0;
     memset(g_stream_expert_cache_slab_slot_locked,
@@ -15150,6 +17794,18 @@ static void ds4_gpu_stream_expert_cache_clear_all(int reset_stats) {
         g_stream_expert_cache_willneed_advise_bytes = 0;
         g_stream_expert_cache_pread_bytes = 0;
         g_stream_expert_cache_pread_ms = 0.0;
+        __atomic_store_n(&g_stream_expert_pread_cached_calls,
+                         0,
+                         __ATOMIC_RELAXED);
+        __atomic_store_n(&g_stream_expert_pread_cached_bytes,
+                         0,
+                         __ATOMIC_RELAXED);
+        __atomic_store_n(&g_stream_expert_pread_nocache_calls,
+                         0,
+                         __ATOMIC_RELAXED);
+        __atomic_store_n(&g_stream_expert_pread_nocache_bytes,
+                         0,
+                         __ATOMIC_RELAXED);
         g_stream_expert_cache_mlock_bytes = 0;
         g_stream_expert_cache_mlock_fail_bytes = 0;
         g_stream_expert_cache_mlock_failures = 0;
@@ -15194,6 +17850,17 @@ static void ds4_gpu_stream_expert_cache_clear_all(int reset_stats) {
         g_stream_expert_timing_reuse_scan_entries = 0;
         g_stream_expert_timing_reuse_scan_ms = 0.0;
         g_stream_expert_timing_reuse_clear_ms = 0.0;
+        g_stream_expert_live_index_scans = 0;
+        g_stream_expert_live_index_entries = 0;
+        g_stream_expert_live_index_fallbacks = 0;
+        g_stream_expert_live_index_inserts = 0;
+        g_stream_expert_live_index_removes = 0;
+        g_stream_expert_exact_persistent_calls = 0;
+        g_stream_expert_exact_transient_calls = 0;
+        g_stream_expert_exact_persistent_fallbacks = 0;
+        g_stream_expert_exact_persistent_failures = 0;
+        g_stream_expert_exact_mapped_view_calls = 0;
+        g_stream_expert_exact_max_unique = 0;
         g_stream_expert_timing_readahead_calls = 0;
         g_stream_expert_timing_readahead_bytes = 0;
         g_stream_expert_timing_readahead_ms = 0.0;
@@ -15342,40 +18009,40 @@ static int ds4_gpu_stream_expert_cache_take_reusable(
     const int timing = ds4_gpu_stream_expert_timing_summary_enabled();
     const double scan_t0 = timing ? ds4_gpu_now_ms() : 0.0;
     uint64_t scan_entries = 0;
-    for (uint32_t layer = 0;
-         layer < DS4_METAL_STREAM_EXPERT_CACHE_MAX_LAYER;
-         layer++) {
-        for (uint32_t expert = 0;
-             expert < DS4_METAL_STREAM_EXPERT_CACHE_MAX_EXPERT;
-             expert++) {
-            scan_entries++;
-            ds4_gpu_stream_expert_cache_entry *e =
-                &g_stream_expert_cache[layer][expert];
-            if (!ds4_gpu_stream_expert_cache_entry_reusable(e,
-                                                            gate_expert_bytes,
-                                                            down_expert_bytes)) {
-                continue;
-            }
-            if (ds4_gpu_stream_expert_cache_entry_inflight(e)) {
-                skipped_inflight = 1;
-                continue;
-            }
-            if (ds4_gpu_stream_expert_cache_entry_protected(layer,
-                                                            expert,
-                                                            protect_layer,
-                                                            protect_ids,
-                                                            n_protect)) {
-                continue;
-            }
-            const uint32_t hotness =
-                g_stream_expert_cache_route_hotness[layer][expert];
-            if (hotness < lowest_hotness ||
-                (hotness == lowest_hotness && e->last_used < oldest)) {
-                lowest_hotness = hotness;
-                oldest = e->last_used;
-                victim_layer = layer;
-                victim_expert = expert;
-            }
+    ds4_gpu_stream_expert_cache_iterator scan_it =
+        ds4_gpu_stream_expert_cache_iterator_begin();
+    uint32_t layer;
+    uint32_t expert;
+    while (ds4_gpu_stream_expert_cache_iterator_next(&scan_it,
+                                                     &layer,
+                                                     &expert)) {
+        scan_entries++;
+        ds4_gpu_stream_expert_cache_entry *e =
+            &g_stream_expert_cache[layer][expert];
+        if (!ds4_gpu_stream_expert_cache_entry_reusable(e,
+                                                        gate_expert_bytes,
+                                                        down_expert_bytes)) {
+            continue;
+        }
+        if (ds4_gpu_stream_expert_cache_entry_inflight(e)) {
+            skipped_inflight = 1;
+            continue;
+        }
+        if (ds4_gpu_stream_expert_cache_entry_protected(layer,
+                                                        expert,
+                                                        protect_layer,
+                                                        protect_ids,
+                                                        n_protect)) {
+            continue;
+        }
+        const uint32_t hotness =
+            g_stream_expert_cache_route_hotness[layer][expert];
+        if (hotness < lowest_hotness ||
+            (hotness == lowest_hotness && e->last_used < oldest)) {
+            lowest_hotness = hotness;
+            oldest = e->last_used;
+            victim_layer = layer;
+            victim_expert = expert;
         }
     }
     if (timing) {
@@ -15466,61 +18133,61 @@ static uint32_t ds4_gpu_stream_expert_cache_take_reusable_batch(
         victim_last_used[i] = UINT64_MAX;
     }
 
-    for (uint32_t layer = 0;
-         layer < DS4_METAL_STREAM_EXPERT_CACHE_MAX_LAYER;
-         layer++) {
-        for (uint32_t expert = 0;
-             expert < DS4_METAL_STREAM_EXPERT_CACHE_MAX_EXPERT;
-             expert++) {
-            scan_entries++;
-            ds4_gpu_stream_expert_cache_entry *e =
-                &g_stream_expert_cache[layer][expert];
-            if (!ds4_gpu_stream_expert_cache_entry_reusable(e,
-                                                            gate_expert_bytes,
-                                                            down_expert_bytes)) {
-                continue;
-            }
-            if (ds4_gpu_stream_expert_cache_entry_inflight(e)) {
-                skipped_inflight = 1;
-                continue;
-            }
-            if (ds4_gpu_stream_expert_cache_entry_protected(layer,
-                                                            expert,
-                                                            protect_layer,
-                                                            protect_ids,
-                                                            n_protect)) {
-                continue;
-            }
+    ds4_gpu_stream_expert_cache_iterator scan_it =
+        ds4_gpu_stream_expert_cache_iterator_begin();
+    uint32_t layer;
+    uint32_t expert;
+    while (ds4_gpu_stream_expert_cache_iterator_next(&scan_it,
+                                                     &layer,
+                                                     &expert)) {
+        scan_entries++;
+        ds4_gpu_stream_expert_cache_entry *e =
+            &g_stream_expert_cache[layer][expert];
+        if (!ds4_gpu_stream_expert_cache_entry_reusable(e,
+                                                        gate_expert_bytes,
+                                                        down_expert_bytes)) {
+            continue;
+        }
+        if (ds4_gpu_stream_expert_cache_entry_inflight(e)) {
+            skipped_inflight = 1;
+            continue;
+        }
+        if (ds4_gpu_stream_expert_cache_entry_protected(layer,
+                                                        expert,
+                                                        protect_layer,
+                                                        protect_ids,
+                                                        n_protect)) {
+            continue;
+        }
 
-            const uint32_t hotness =
-                g_stream_expert_cache_route_hotness[layer][expert];
-            const uint64_t last_used = e->last_used;
-            if (victim_count < n_needed) {
-                victim_layers[victim_count] = layer;
-                victim_experts[victim_count] = expert;
-                victim_hotness[victim_count] = hotness;
-                victim_last_used[victim_count] = last_used;
-                victim_count++;
-                continue;
-            }
+        const uint32_t hotness =
+            g_stream_expert_cache_route_hotness[layer][expert];
+        const uint64_t last_used = e->last_used;
+        if (victim_count < n_needed) {
+            victim_layers[victim_count] = layer;
+            victim_experts[victim_count] = expert;
+            victim_hotness[victim_count] = hotness;
+            victim_last_used[victim_count] = last_used;
+            victim_count++;
+            continue;
+        }
 
-            uint32_t worst = 0;
-            for (uint32_t i = 1; i < victim_count; i++) {
-                if (victim_hotness[i] > victim_hotness[worst] ||
-                    (victim_hotness[i] == victim_hotness[worst] &&
-                     victim_last_used[i] > victim_last_used[worst])) {
-                    worst = i;
-                }
-            }
-            if (hotness < victim_hotness[worst] ||
-                (hotness == victim_hotness[worst] &&
-                 last_used < victim_last_used[worst])) {
-                victim_layers[worst] = layer;
-                victim_experts[worst] = expert;
-                victim_hotness[worst] = hotness;
-                victim_last_used[worst] = last_used;
+        uint32_t worst = 0;
+        for (uint32_t i = 1; i < victim_count; i++) {
+            if (victim_hotness[i] > victim_hotness[worst] ||
+                (victim_hotness[i] == victim_hotness[worst] &&
+                 victim_last_used[i] > victim_last_used[worst])) {
+                worst = i;
             }
         }
+        if (hotness < victim_hotness[worst] ||
+            (hotness == victim_hotness[worst] &&
+             last_used < victim_last_used[worst])) {
+            victim_layers[worst] = layer;
+            victim_experts[worst] = expert;
+            victim_hotness[worst] = hotness;
+            victim_last_used[worst] = last_used;
+        }
     }
     if (timing) {
         ds4_gpu_stream_expert_timing_note_reuse_scan(scan_entries,
@@ -15614,36 +18281,36 @@ static uint32_t ds4_gpu_stream_expert_cache_release_mlock_margin(
         uint32_t lowest_hotness = UINT32_MAX;
         uint64_t oldest = UINT64_MAX;
 
-        for (uint32_t layer = 0;
-             layer < DS4_METAL_STREAM_EXPERT_CACHE_MAX_LAYER;
-             layer++) {
-            for (uint32_t expert = 0;
-                 expert < DS4_METAL_STREAM_EXPERT_CACHE_MAX_EXPERT;
-                 expert++) {
-                ds4_gpu_stream_expert_cache_entry *e =
-                    &g_stream_expert_cache[layer][expert];
-                if (!e->valid ||
-                    !e->slab_backed ||
-                    e->slab_slot >= DS4_METAL_STREAM_EXPERT_CACHE_MAX_ENTRIES ||
-                    !g_stream_expert_cache_slab_slot_locked[e->slab_slot] ||
-                    ds4_gpu_stream_expert_cache_entry_inflight(e) ||
-                    ds4_gpu_stream_expert_cache_entry_protected(layer,
-                                                                expert,
-                                                                protect_layer,
-                                                                protect_ids,
-                                                                n_protect)) {
-                    continue;
-                }
-                const uint32_t hotness =
-                    g_stream_expert_cache_route_hotness[layer][expert];
-                if (hotness < lowest_hotness ||
-                    (hotness == lowest_hotness && e->last_used < oldest)) {
-                    lowest_hotness = hotness;
-                    oldest = e->last_used;
-                    victim_layer = layer;
-                    victim_expert = expert;
-                    victim_slot = e->slab_slot;
-                }
+        ds4_gpu_stream_expert_cache_iterator scan_it =
+            ds4_gpu_stream_expert_cache_iterator_begin();
+        uint32_t layer;
+        uint32_t expert;
+        while (ds4_gpu_stream_expert_cache_iterator_next(&scan_it,
+                                                         &layer,
+                                                         &expert)) {
+            ds4_gpu_stream_expert_cache_entry *e =
+                &g_stream_expert_cache[layer][expert];
+            if (!e->valid ||
+                !e->slab_backed ||
+                e->slab_slot >= DS4_METAL_STREAM_EXPERT_CACHE_MAX_ENTRIES ||
+                !g_stream_expert_cache_slab_slot_locked[e->slab_slot] ||
+                ds4_gpu_stream_expert_cache_entry_inflight(e) ||
+                ds4_gpu_stream_expert_cache_entry_protected(layer,
+                                                            expert,
+                                                            protect_layer,
+                                                            protect_ids,
+                                                            n_protect)) {
+                continue;
+            }
+            const uint32_t hotness =
+                g_stream_expert_cache_route_hotness[layer][expert];
+            if (hotness < lowest_hotness ||
+                (hotness == lowest_hotness && e->last_used < oldest)) {
+                lowest_hotness = hotness;
+                oldest = e->last_used;
+                victim_layer = layer;
+                victim_expert = expert;
+                victim_slot = e->slab_slot;
             }
         }
 
@@ -15900,31 +18567,31 @@ static void ds4_gpu_stream_expert_cache_prune_global(
         uint32_t victim_expert = UINT32_MAX;
         uint32_t lowest_hotness = UINT32_MAX;
         uint64_t oldest = UINT64_MAX;
-        for (uint32_t layer = 0;
-             layer < DS4_METAL_STREAM_EXPERT_CACHE_MAX_LAYER;
-             layer++) {
-            for (uint32_t expert = 0;
-                 expert < DS4_METAL_STREAM_EXPERT_CACHE_MAX_EXPERT;
-                 expert++) {
-                ds4_gpu_stream_expert_cache_entry *e =
-                    &g_stream_expert_cache[layer][expert];
-                if (!e->valid ||
-                    ds4_gpu_stream_expert_cache_entry_protected(layer,
-                                                                expert,
-                                                                protect_layer,
-                                                                protect_ids,
-                                                                n_protect)) {
-                    continue;
-                }
-                const uint32_t hotness =
-                    g_stream_expert_cache_route_hotness[layer][expert];
-                if (hotness < lowest_hotness ||
-                    (hotness == lowest_hotness && e->last_used < oldest)) {
-                    lowest_hotness = hotness;
-                    oldest = e->last_used;
-                    victim_layer = layer;
-                    victim_expert = expert;
-                }
+        ds4_gpu_stream_expert_cache_iterator scan_it =
+            ds4_gpu_stream_expert_cache_iterator_begin();
+        uint32_t layer;
+        uint32_t expert;
+        while (ds4_gpu_stream_expert_cache_iterator_next(&scan_it,
+                                                         &layer,
+                                                         &expert)) {
+            ds4_gpu_stream_expert_cache_entry *e =
+                &g_stream_expert_cache[layer][expert];
+            if (!e->valid ||
+                ds4_gpu_stream_expert_cache_entry_protected(layer,
+                                                            expert,
+                                                            protect_layer,
+                                                            protect_ids,
+                                                            n_protect)) {
+                continue;
+            }
+            const uint32_t hotness =
+                g_stream_expert_cache_route_hotness[layer][expert];
+            if (hotness < lowest_hotness ||
+                (hotness == lowest_hotness && e->last_used < oldest)) {
+                lowest_hotness = hotness;
+                oldest = e->last_used;
+                victim_layer = layer;
+                victim_expert = expert;
             }
         }
         if (victim_layer == UINT32_MAX || victim_expert == UINT32_MAX) break;
@@ -16092,6 +18759,7 @@ static int ds4_gpu_stream_expert_cache_entry_matches(
     } else {
         g_stream_expert_cache_bytes += logical_bytes;
     }
+    ds4_gpu_stream_expert_cache_live_insert(layer, expert);
     g_stream_expert_cache_misses++;
     g_stream_expert_cache_layer_misses[layer]++;
     g_stream_expert_cache_wraps += 3;
@@ -16362,9 +19030,10 @@ static int ds4_gpu_stream_expert_pending_load_install(
     }
     if (ds4_gpu_stream_expert_pending_load_profile_enabled()) {
         fprintf(stderr,
-                "ds4: Metal streaming expert early-load finish layer=%u experts=%u tensors=%u bytes=%.2f GiB wall=%.3f ms\n",
+                "ds4: Metal streaming expert early-load finish layer=%u experts=%u tensors=%u requests=%u bytes=%.2f GiB wall=%.3f ms\n",
                 p->layer,
                 p->n_loads,
+                p->n_tensor_tasks,
                 p->n_tasks,
                 ds4_gpu_gib(read_bytes),
                 elapsed_ms);
@@ -16390,6 +19059,7 @@ static int ds4_gpu_stream_expert_pending_load_finish(
                                                               elapsed_ms);
     ds4_gpu_stream_expert_pending_load_release_buffers(p);
     p->n_tasks = 0;
+    p->n_tensor_tasks = 0;
     p->n_loads = 0;
     p->prepare_ms = 0.0;
     return ok;
@@ -16485,6 +19155,7 @@ int ds4_gpu_stream_expert_cache_begin_selected_load(
     p->missing_mask = 0;
     p->n_loads = 0;
     p->n_tasks = 0;
+    p->n_tensor_tasks = 0;
     p->gate_expert_bytes = gate_expert_bytes;
     p->down_expert_bytes = down_expert_bytes;
     p->prepare_ms = 0.0;
@@ -16684,6 +19355,11 @@ int ds4_gpu_stream_expert_cache_begin_selected_load(
         }
     }
 
+    p->n_tensor_tasks = p->n_tasks;
+    p->n_tasks = ds4_gpu_stream_expert_pread_expand_tasks_bounded(
+        p->tasks,
+        p->n_tasks,
+        DS4_METAL_STREAM_EXPERT_PENDING_MAX_TASKS);
     const uint32_t n_workers =
         ds4_gpu_stream_expert_pread_thread_count(p->n_tasks);
     p->start_ms = ds4_gpu_now_ms();
@@ -16696,9 +19372,10 @@ int ds4_gpu_stream_expert_cache_begin_selected_load(
         p->active = 1;
         if (ds4_gpu_stream_expert_pending_load_profile_enabled()) {
             fprintf(stderr,
-                    "ds4: Metal streaming expert early-load begin layer=%u experts=%u tensors=%u threads=%u\n",
+                    "ds4: Metal streaming expert early-load begin layer=%u experts=%u tensors=%u requests=%u threads=%u\n",
                     layer,
                     p->n_loads,
+                    p->n_tensor_tasks,
                     p->n_tasks,
                     n_workers);
         }
@@ -16707,10 +19384,10 @@ int ds4_gpu_stream_expert_cache_begin_selected_load(
 
     uint64_t read_bytes = 0;
     double read_ms = 0.0;
-    if (!ds4_gpu_stream_expert_pread_tasks(p->tasks,
-                                           p->n_tasks,
-                                           &read_bytes,
-                                           &read_ms)) {
+    if (!ds4_gpu_stream_expert_pread_tasks_run(p->tasks,
+                                               p->n_tasks,
+                                               &read_bytes,
+                                               &read_ms)) {
         ds4_gpu_stream_expert_pending_load_release_buffers(p);
         return 0;
     }
@@ -16721,6 +19398,7 @@ int ds4_gpu_stream_expert_cache_begin_selected_load(
     }
     ds4_gpu_stream_expert_pending_load_release_buffers(p);
     p->n_tasks = 0;
+    p->n_tensor_tasks = 0;
     p->n_loads = 0;
     p->prepare_ms = 0.0;
     return 1;
@@ -17318,6 +19996,320 @@ static void ds4_gpu_stream_expert_cache_clear_layer(uint32_t layer) {
     g_stream_expert_cache_layer_count[layer] = 0;
 }
 
+/* Tiny speculative batches may route to more unique experts than the user's
+ * persistent SSD cache can hold.  Loading the entire routed tensors for that
+ * case defeats streaming (and is especially costly for every verifier layer).
+ * Reuse matching persistent-cache entries, pack only the remaining selected
+ * expert slabs into transient shared buffers, and give the address-table
+ * kernels private transient tables.  This avoids both redundant preads and
+ * dangling GPU addresses in the persistent decode-cache tables after the
+ * command completes.  Cached entries are returned as explicit resources and
+ * marked in-flight by the encoder; the transient registry retains every miss
+ * buffer until the enclosing Metal command buffer completes. */
+static int ds4_gpu_stream_expert_prepare_transient_selected_batch(
+        const void    *model_map,
+        uint64_t       model_size,
+        uint32_t       layer,
+        const int32_t *unique_ids,
+        const uint32_t *frequency,
+        uint32_t       unique_count,
+        uint32_t       n_total_expert,
+        uint32_t       n_selected,
+        uint64_t       gate_offset,
+        uint64_t       up_offset,
+        uint64_t       down_offset,
+        uint64_t       gate_expert_bytes,
+        uint64_t       down_expert_bytes,
+        id<MTLBuffer> __strong *gate_addrs,
+        id<MTLBuffer> __strong *up_addrs,
+        id<MTLBuffer> __strong *down_addrs,
+        id<MTLBuffer> __strong *packed_gate,
+        id<MTLBuffer> __strong *packed_up,
+        id<MTLBuffer> __strong *packed_down,
+        ds4_gpu_stream_expert_cache_entry **resources,
+        uint32_t      *n_resources) {
+    if (!model_map || !unique_ids || !frequency ||
+        unique_count == 0 || unique_count > n_total_expert ||
+        !gate_addrs || !up_addrs || !down_addrs ||
+        !packed_gate || !packed_up || !packed_down ||
+        !resources || !n_resources ||
+        layer >= DS4_METAL_STREAM_EXPERT_CACHE_MAX_LAYER ||
+        n_total_expert > DS4_METAL_STREAM_EXPERT_CACHE_MAX_EXPERT ||
+        n_selected == 0 ||
+        gate_expert_bytes == 0 || down_expert_bytes == 0 ||
+        (uint64_t)unique_count > UINT64_MAX / gate_expert_bytes ||
+        (uint64_t)unique_count > UINT64_MAX / down_expert_bytes) {
+        return 0;
+    }
+
+    *gate_addrs = nil;
+    *up_addrs = nil;
+    *down_addrs = nil;
+    *packed_gate = nil;
+    *packed_up = nil;
+    *packed_down = nil;
+    *n_resources = 0;
+
+    uint64_t gate_abs_offsets[DS4_METAL_STREAM_EXPERT_CACHE_MAX_EXPERT];
+    uint64_t up_abs_offsets[DS4_METAL_STREAM_EXPERT_CACHE_MAX_EXPERT];
+    uint64_t down_abs_offsets[DS4_METAL_STREAM_EXPERT_CACHE_MAX_EXPERT];
+    uint32_t miss_slots[DS4_METAL_STREAM_EXPERT_CACHE_MAX_EXPERT];
+    ds4_gpu_stream_expert_cache_entry
+        *hit_entries[DS4_METAL_STREAM_EXPERT_CACHE_MAX_EXPERT] = { NULL };
+    uint32_t miss_count = 0;
+    int ok = 1;
+    for (uint32_t u = 0; u < unique_count; u++) {
+        const int32_t selected_id = unique_ids[u];
+        if (selected_id < 0 || (uint32_t)selected_id >= n_total_expert) {
+            ok = 0;
+            break;
+        }
+        const uint64_t expert = (uint64_t)(uint32_t)selected_id;
+        if (expert > UINT64_MAX / gate_expert_bytes ||
+            expert > UINT64_MAX / down_expert_bytes) {
+            ok = 0;
+            break;
+        }
+        const uint64_t gate_rel = expert * gate_expert_bytes;
+        const uint64_t down_rel = expert * down_expert_bytes;
+        if (gate_rel > UINT64_MAX - gate_offset ||
+            gate_rel > UINT64_MAX - up_offset ||
+            down_rel > UINT64_MAX - down_offset) {
+            ok = 0;
+            break;
+        }
+        const uint64_t gate_abs = gate_offset + gate_rel;
+        const uint64_t up_abs = up_offset + gate_rel;
+        const uint64_t down_abs = down_offset + down_rel;
+        if (gate_abs > model_size || gate_expert_bytes > model_size - gate_abs ||
+            up_abs > model_size || gate_expert_bytes > model_size - up_abs ||
+            down_abs > model_size || down_expert_bytes > model_size - down_abs) {
+            ok = 0;
+            break;
+        }
+        gate_abs_offsets[u] = gate_abs;
+        up_abs_offsets[u] = up_abs;
+        down_abs_offsets[u] = down_abs;
+
+        ds4_gpu_stream_expert_cache_entry *entry =
+            ds4_gpu_stream_expert_cache_peek(model_map,
+                                             model_size,
+                                             layer,
+                                             (uint32_t)expert,
+                                             n_total_expert,
+                                             n_selected,
+                                             gate_abs,
+                                             up_abs,
+                                             down_abs,
+                                             gate_expert_bytes,
+                                             down_expert_bytes);
+        hit_entries[u] = entry;
+        if (entry) {
+            miss_slots[u] = UINT32_MAX;
+        } else {
+            miss_slots[u] = miss_count++;
+        }
+    }
+    if (!ok) return 0;
+
+    const uint64_t gate_bytes = (uint64_t)miss_count * gate_expert_bytes;
+    const uint64_t down_bytes = (uint64_t)miss_count * down_expert_bytes;
+    if (gate_bytes > (uint64_t)NSUIntegerMax ||
+        down_bytes > (uint64_t)NSUIntegerMax) {
+        return 0;
+    }
+
+    id<MTLBuffer> gate_buf = nil;
+    id<MTLBuffer> up_buf = nil;
+    id<MTLBuffer> down_buf = nil;
+    if (miss_count != 0) {
+        gate_buf =
+            ds4_gpu_new_transient_buffer((NSUInteger)gate_bytes,
+                                         "ds4_ssd_tiny_selected_gate");
+        up_buf =
+            ds4_gpu_new_transient_buffer((NSUInteger)gate_bytes,
+                                         "ds4_ssd_tiny_selected_up");
+        down_buf =
+            ds4_gpu_new_transient_buffer((NSUInteger)down_bytes,
+                                         "ds4_ssd_tiny_selected_down");
+        if (!gate_buf || !up_buf || !down_buf) return 0;
+    }
+    const uint64_t addr_bytes64 =
+        (uint64_t)n_total_expert * sizeof(uint64_t);
+    if (addr_bytes64 == 0 || addr_bytes64 > (uint64_t)NSUIntegerMax) return 0;
+    const NSUInteger addr_bytes = (NSUInteger)addr_bytes64;
+    id<MTLBuffer> gate_addr_buf =
+        ds4_gpu_new_transient_buffer(addr_bytes,
+                                     "ds4_ssd_tiny_selected_gate_addrs");
+    id<MTLBuffer> up_addr_buf =
+        ds4_gpu_new_transient_buffer(addr_bytes,
+                                     "ds4_ssd_tiny_selected_up_addrs");
+    id<MTLBuffer> down_addr_buf =
+        ds4_gpu_new_transient_buffer(addr_bytes,
+                                     "ds4_ssd_tiny_selected_down_addrs");
+    if (!gate_addr_buf || !up_addr_buf || !down_addr_buf) {
+        return 0;
+    }
+
+    uint8_t *gate_dst = miss_count != 0 ? (uint8_t *)[gate_buf contents] : NULL;
+    uint8_t *up_dst = miss_count != 0 ? (uint8_t *)[up_buf contents] : NULL;
+    uint8_t *down_dst = miss_count != 0 ? (uint8_t *)[down_buf contents] : NULL;
+    uint64_t *gate_addr = (uint64_t *)[gate_addr_buf contents];
+    uint64_t *up_addr = (uint64_t *)[up_addr_buf contents];
+    uint64_t *down_addr = (uint64_t *)[down_addr_buf contents];
+    if ((miss_count != 0 && (!gate_dst || !up_dst || !down_dst)) ||
+        !gate_addr || !up_addr || !down_addr) {
+        return 0;
+    }
+    memset(gate_addr, 0, addr_bytes);
+    memset(up_addr, 0, addr_bytes);
+    memset(down_addr, 0, addr_bytes);
+
+    ds4_gpu_stream_expert_pread_task *tasks = NULL;
+    if (miss_count != 0) {
+        tasks = calloc((size_t)miss_count * 3u, sizeof(tasks[0]));
+        if (!tasks) return 0;
+    }
+
+    uint32_t n_tasks = 0;
+    for (uint32_t u = 0; u < unique_count; u++) {
+        if (hit_entries[u]) continue;
+        const uint64_t gate_inner =
+            (uint64_t)miss_slots[u] * gate_expert_bytes;
+        const uint64_t down_inner =
+            (uint64_t)miss_slots[u] * down_expert_bytes;
+        tasks[n_tasks++] = (ds4_gpu_stream_expert_pread_task) {
+            .offset = gate_abs_offsets[u],
+            .len = gate_expert_bytes,
+            .dst = gate_dst + gate_inner,
+        };
+        tasks[n_tasks++] = (ds4_gpu_stream_expert_pread_task) {
+            .offset = up_abs_offsets[u],
+            .len = gate_expert_bytes,
+            .dst = up_dst + gate_inner,
+        };
+        tasks[n_tasks++] = (ds4_gpu_stream_expert_pread_task) {
+            .offset = down_abs_offsets[u],
+            .len = down_expert_bytes,
+            .dst = down_dst + down_inner,
+        };
+    }
+
+    uint64_t read_bytes = 0;
+    double read_ms = 0.0;
+    if (ok) {
+        ok = ds4_gpu_stream_expert_pread_tasks(tasks,
+                                               n_tasks,
+                                               &read_bytes,
+                                               &read_ms);
+    }
+    free(tasks);
+    if (!ok) return 0;
+
+    if (miss_count != 0) {
+        ds4_gpu_stream_expert_cache_note_pread(layer, read_bytes, read_ms);
+        if (g_stream_expert_cache_misses > UINT64_MAX - miss_count) {
+            g_stream_expert_cache_misses = UINT64_MAX;
+        } else {
+            g_stream_expert_cache_misses += miss_count;
+        }
+        if (g_stream_expert_cache_layer_misses[layer] >
+            UINT64_MAX - miss_count) {
+            g_stream_expert_cache_layer_misses[layer] = UINT64_MAX;
+        } else {
+            g_stream_expert_cache_layer_misses[layer] += miss_count;
+        }
+        [gate_buf didModifyRange:NSMakeRange(0, (NSUInteger)gate_bytes)];
+        [up_buf didModifyRange:NSMakeRange(0, (NSUInteger)gate_bytes)];
+        [down_buf didModifyRange:NSMakeRange(0, (NSUInteger)down_bytes)];
+    }
+
+    for (uint32_t u = 0; u < unique_count; u++) {
+        const uint32_t expert = (uint32_t)unique_ids[u];
+        ds4_gpu_stream_expert_cache_entry *entry = hit_entries[u];
+        if (entry) {
+            gate_addr[expert] =
+                ds4_gpu_buffer_address(entry->gate_buffer, entry->gate_inner);
+            up_addr[expert] =
+                ds4_gpu_buffer_address(entry->up_buffer, entry->up_inner);
+            down_addr[expert] =
+                ds4_gpu_buffer_address(entry->down_buffer, entry->down_inner);
+            const uint32_t extra_uses =
+                frequency[expert] > 0 ? frequency[expert] - 1u : 0;
+            if (extra_uses != 0) {
+                if (entry->use_count > UINT64_MAX - extra_uses) {
+                    entry->use_count = UINT64_MAX;
+                } else {
+                    entry->use_count += extra_uses;
+                }
+                if (g_stream_expert_cache_hits > UINT64_MAX - extra_uses) {
+                    g_stream_expert_cache_hits = UINT64_MAX;
+                } else {
+                    g_stream_expert_cache_hits += extra_uses;
+                }
+                if (g_stream_expert_cache_layer_hits[layer] >
+                    UINT64_MAX - extra_uses) {
+                    g_stream_expert_cache_layer_hits[layer] = UINT64_MAX;
+                } else {
+                    g_stream_expert_cache_layer_hits[layer] += extra_uses;
+                }
+            }
+            resources[*n_resources] = entry;
+            (*n_resources)++;
+        } else {
+            const NSUInteger gate_inner = (NSUInteger)(
+                    (uint64_t)miss_slots[u] * gate_expert_bytes);
+            const NSUInteger down_inner = (NSUInteger)(
+                    (uint64_t)miss_slots[u] * down_expert_bytes);
+            gate_addr[expert] = ds4_gpu_buffer_address(gate_buf, gate_inner);
+            up_addr[expert] = ds4_gpu_buffer_address(up_buf, gate_inner);
+            down_addr[expert] = ds4_gpu_buffer_address(down_buf, down_inner);
+            const uint32_t extra_uses =
+                frequency[expert] > 0 ? frequency[expert] - 1u : 0;
+            if (extra_uses != 0) {
+                if (g_stream_expert_cache_hits > UINT64_MAX - extra_uses) {
+                    g_stream_expert_cache_hits = UINT64_MAX;
+                } else {
+                    g_stream_expert_cache_hits += extra_uses;
+                }
+                if (g_stream_expert_cache_layer_hits[layer] >
+                    UINT64_MAX - extra_uses) {
+                    g_stream_expert_cache_layer_hits[layer] = UINT64_MAX;
+                } else {
+                    g_stream_expert_cache_layer_hits[layer] += extra_uses;
+                }
+            }
+        }
+        if (gate_addr[expert] == 0 || up_addr[expert] == 0 ||
+            down_addr[expert] == 0) {
+            return 0;
+        }
+    }
+    [gate_addr_buf didModifyRange:NSMakeRange(0, addr_bytes)];
+    [up_addr_buf didModifyRange:NSMakeRange(0, addr_bytes)];
+    [down_addr_buf didModifyRange:NSMakeRange(0, addr_bytes)];
+
+    *gate_addrs = gate_addr_buf;
+    *up_addrs = up_addr_buf;
+    *down_addrs = down_addr_buf;
+    *packed_gate = gate_buf;
+    *packed_up = up_buf;
+    *packed_down = down_buf;
+    if (getenv("DS4_METAL_STREAMING_EXPERT_PREAD_PROFILE") != NULL) {
+        fprintf(stderr,
+                "ds4: Metal SSD tiny selected batch layer=%u experts=%u "
+                "cache_hits=%u transient_misses=%u "
+                "bytes=%.2f MiB wall=%.3f ms\n",
+                layer,
+                unique_count,
+                *n_resources,
+                miss_count,
+                ds4_gpu_mib(read_bytes),
+                read_ms);
+    }
+    return 1;
+}
+
 static int ds4_gpu_stream_expert_cache_prepare_selected_batch(
         const void    *model_map,
         uint64_t       model_size,
@@ -17331,15 +20323,16 @@ static int ds4_gpu_stream_expert_cache_prepare_selected_batch(
         uint64_t       down_offset,
         uint64_t       gate_expert_bytes,
         uint64_t       down_expert_bytes,
-        id<MTLBuffer> *gate_addrs,
-        id<MTLBuffer> *up_addrs,
-        id<MTLBuffer> *down_addrs,
+        id<MTLBuffer> __strong *gate_addrs,
+        id<MTLBuffer> __strong *up_addrs,
+        id<MTLBuffer> __strong *down_addrs,
         ds4_gpu_stream_expert_cache_entry **resources,
         uint32_t      *n_resources,
         uint32_t      *unique_out,
-        id<MTLBuffer> *overflow_gate,
-        id<MTLBuffer> *overflow_up,
-        id<MTLBuffer> *overflow_down) {
+        id<MTLBuffer> __strong *overflow_gate,
+        id<MTLBuffer> __strong *overflow_up,
+        id<MTLBuffer> __strong *overflow_down,
+        bool           require_private_addr_tables) {
     if (overflow_gate) *overflow_gate = nil;
     if (overflow_up) *overflow_up = nil;
     if (overflow_down) *overflow_down = nil;
@@ -17366,6 +20359,8 @@ static int ds4_gpu_stream_expert_cache_prepare_selected_batch(
         return 0;
     }
     if (n_tokens > UINT32_MAX / n_selected) return 0;
+    const int prefer_prefill_nocache =
+        ds4_gpu_stream_prefill_nocache_for_tokens(n_tokens);
 
     const uint64_t n_ids = (uint64_t)n_tokens * n_selected;
     if (n_ids > SIZE_MAX / sizeof(int32_t)) return 0;
@@ -17387,7 +20382,7 @@ static int ds4_gpu_stream_expert_cache_prepare_selected_batch(
     uint32_t unique_count = 0;
     *n_resources = 0;
     *unique_out = 0;
-    if (ok) {
+    if (ok && !require_private_addr_tables) {
         if (!ds4_gpu_stream_expert_cache_ensure_addr_buffers(layer)) {
             ok = 0;
         }
@@ -17415,6 +20410,87 @@ static int ds4_gpu_stream_expert_cache_prepare_selected_batch(
         ds4_gpu_stream_expert_cache_note_frequency_hotness(layer,
                                                            frequency,
                                                            n_total_expert);
+        if (require_private_addr_tables &&
+            unique_count > g_stream_expert_exact_max_unique) {
+            g_stream_expert_exact_max_unique = unique_count;
+        }
+    }
+    const uint32_t configured_count =
+        ds4_gpu_stream_expert_cache_configured_count();
+    const int private_persistent_disabled =
+        require_private_addr_tables &&
+        ds4_gpu_exact_rows_persistent_env_enabled(
+                "DS4_METAL_DISABLE_EXACT_ROWS_PERSISTENT_CACHE");
+    const uint32_t private_size_class_count =
+        ok && require_private_addr_tables && !private_persistent_disabled ?
+        ds4_gpu_stream_expert_cache_budget_for_expert_size(
+                gate_expert_bytes, down_expert_bytes) : 0;
+    const int private_size_class_ok =
+        private_size_class_count != 0 &&
+        !g_stream_expert_cache_class_conflict &&
+        g_stream_expert_cache_gate_class_bytes == gate_expert_bytes &&
+        g_stream_expert_cache_down_class_bytes == down_expert_bytes;
+    const int private_persistent_policy =
+        ok && require_private_addr_tables ?
+        ds4_gpu_exact_rows_persistent_policy(private_size_class_count,
+                                             unique_count,
+                                             private_size_class_ok) : 0;
+    if (ok && require_private_addr_tables &&
+        private_persistent_policy < 0) {
+        g_stream_expert_exact_persistent_failures++;
+        fprintf(stderr,
+                "ds4: Metal exact-row persistent cache is required but "
+                "disabled or ineligible (cache_budget=%u "
+                "size_class_budget=%u unique=%u)\n",
+                configured_count,
+                private_size_class_count,
+                unique_count);
+        free(ids);
+        return 0;
+    }
+    const bool persistent_private_tables =
+        private_persistent_policy > 0;
+    if (ok && require_private_addr_tables) {
+        if (persistent_private_tables) {
+            g_stream_expert_exact_persistent_calls++;
+        } else {
+            g_stream_expert_exact_transient_calls++;
+            g_stream_expert_exact_persistent_fallbacks++;
+        }
+    }
+    const bool use_transient_selected =
+        ok &&
+        n_tokens <= 5u &&
+        ((require_private_addr_tables && !persistent_private_tables) ||
+         (!require_private_addr_tables &&
+          configured_count < n_total_expert));
+    if (use_transient_selected) {
+        ok = ds4_gpu_stream_expert_prepare_transient_selected_batch(
+                model_map,
+                model_size,
+                layer,
+                unique_ids,
+                frequency,
+                unique_count,
+                n_total_expert,
+                n_selected,
+                gate_offset,
+                up_offset,
+                down_offset,
+                gate_expert_bytes,
+                down_expert_bytes,
+                gate_addrs,
+                up_addrs,
+                down_addrs,
+                overflow_gate,
+                overflow_up,
+                overflow_down,
+                resources,
+                n_resources);
+        free(ids);
+        if (!ok) return 0;
+        *unique_out = unique_count;
+        return 1;
     }
     /*
      * When the layer's unique selected set does not fit the cache budget, the
@@ -17459,6 +20535,7 @@ static int ds4_gpu_stream_expert_cache_prepare_selected_batch(
 
     ds4_gpu_stream_expert_pread_task *tasks = NULL;
     uint32_t n_loads = 0;
+    uint32_t n_prepared = 0;
     uint32_t n_tasks = 0;
     double load_prepare_ms = 0.0;
     double load_modify_ms = 0.0;
@@ -17516,12 +20593,14 @@ static int ds4_gpu_stream_expert_cache_prepare_selected_batch(
 
             const int force_reuse =
                 cache_budget != 0 && reserved_entries >= cache_budget;
-            ds4_gpu_stream_expert_readahead_range(unique_gate_offsets[u],
-                                                  gate_expert_bytes);
-            ds4_gpu_stream_expert_readahead_range(unique_up_offsets[u],
-                                                  gate_expert_bytes);
-            ds4_gpu_stream_expert_readahead_range(unique_down_offsets[u],
-                                                  down_expert_bytes);
+            if (ds4_gpu_stream_prefill_expert_readahead_enabled(n_tokens)) {
+                ds4_gpu_stream_expert_readahead_range(unique_gate_offsets[u],
+                                                      gate_expert_bytes);
+                ds4_gpu_stream_expert_readahead_range(unique_up_offsets[u],
+                                                      gate_expert_bytes);
+                ds4_gpu_stream_expert_readahead_range(unique_down_offsets[u],
+                                                      down_expert_bytes);
+            }
             const double buffer_t0 = load_timing ? ds4_gpu_now_ms() : 0.0;
             const int prepared =
                 ds4_gpu_stream_expert_cache_prepare_load_buffers(layer,
@@ -17532,12 +20611,12 @@ static int ds4_gpu_stream_expert_cache_prepare_selected_batch(
                                                                  gate_expert_bytes,
                                                                  down_expert_bytes,
                                                                  force_reuse,
-                                                                 &gate_bufs[n_loads],
-                                                                 &up_bufs[n_loads],
-                                                                 &down_bufs[n_loads],
-                                                                 &gate_inners[n_loads],
-                                                                 &up_inners[n_loads],
-                                                                 &down_inners[n_loads]);
+                                                                 &gate_bufs[n_prepared],
+                                                                 &up_bufs[n_prepared],
+                                                                 &down_bufs[n_prepared],
+                                                                 &gate_inners[n_prepared],
+                                                                 &up_inners[n_prepared],
+                                                                 &down_inners[n_prepared]);
             if (load_timing) {
                 ds4_gpu_stream_expert_timing_note_prepare_buffer(
                         ds4_gpu_now_ms() - buffer_t0);
@@ -17546,43 +20625,50 @@ static int ds4_gpu_stream_expert_cache_prepare_selected_batch(
                 ok = 0;
                 break;
             }
+            const uint32_t prepared_i = n_prepared++;
+            load_unique[prepared_i] = u;
             if (!force_reuse && reserved_entries < UINT32_MAX) {
                 reserved_entries++;
             }
-            if (!gate_bufs[n_loads] ||
-                !up_bufs[n_loads] ||
-                !down_bufs[n_loads]) {
+            if (!gate_bufs[prepared_i] ||
+                !up_bufs[prepared_i] ||
+                !down_bufs[prepared_i]) {
                 ok = 0;
                 break;
             }
 
-            uint8_t *gate_dst = (uint8_t *)[gate_bufs[n_loads] contents] +
-                                 gate_inners[n_loads];
-            uint8_t *up_dst = (uint8_t *)[up_bufs[n_loads] contents] +
-                               up_inners[n_loads];
-            uint8_t *down_dst = (uint8_t *)[down_bufs[n_loads] contents] +
-                                 down_inners[n_loads];
+            uint8_t *gate_dst =
+                (uint8_t *)[gate_bufs[prepared_i] contents] +
+                gate_inners[prepared_i];
+            uint8_t *up_dst =
+                (uint8_t *)[up_bufs[prepared_i] contents] +
+                up_inners[prepared_i];
+            uint8_t *down_dst =
+                (uint8_t *)[down_bufs[prepared_i] contents] +
+                down_inners[prepared_i];
             if (!gate_dst || !up_dst || !down_dst) {
                 ok = 0;
                 break;
             }
 
-            load_unique[n_loads] = u;
             const double task_t0 = load_timing ? ds4_gpu_now_ms() : 0.0;
             tasks[n_tasks++] = (ds4_gpu_stream_expert_pread_task) {
                 .offset = unique_gate_offsets[u],
                 .len = gate_expert_bytes,
                 .dst = gate_dst,
+                .prefer_nocache = prefer_prefill_nocache,
             };
             tasks[n_tasks++] = (ds4_gpu_stream_expert_pread_task) {
                 .offset = unique_up_offsets[u],
                 .len = gate_expert_bytes,
                 .dst = up_dst,
+                .prefer_nocache = prefer_prefill_nocache,
             };
             tasks[n_tasks++] = (ds4_gpu_stream_expert_pread_task) {
                 .offset = unique_down_offsets[u],
                 .len = down_expert_bytes,
                 .dst = down_dst,
+                .prefer_nocache = prefer_prefill_nocache,
             };
             if (load_timing) {
                 ds4_gpu_stream_expert_timing_note_prepare_task(
@@ -17668,6 +20754,40 @@ static int ds4_gpu_stream_expert_cache_prepare_selected_batch(
                                                           load_install_ms);
         }
     }
+    if (!ok) {
+        /* install_loaded() may fail after publishing an entry. Classify every
+         * prepared tuple by authoritative cache identity before recycling;
+         * clear_layer() below owns all tuples already transferred to cache. */
+        for (uint32_t load_i = 0; load_i < n_prepared; load_i++) {
+            bool cache_owned = false;
+            const uint32_t u = load_unique[load_i];
+            if (u < unique_count && unique_ids[u] >= 0 &&
+                (uint32_t)unique_ids[u] <
+                    DS4_METAL_STREAM_EXPERT_CACHE_MAX_EXPERT) {
+                const ds4_gpu_stream_expert_cache_entry *entry =
+                    &g_stream_expert_cache[layer][(uint32_t)unique_ids[u]];
+                cache_owned =
+                    entry->valid &&
+                    entry->gate_buffer == gate_bufs[load_i] &&
+                    entry->up_buffer == up_bufs[load_i] &&
+                    entry->down_buffer == down_bufs[load_i] &&
+                    entry->gate_inner == gate_inners[load_i] &&
+                    entry->up_inner == up_inners[load_i] &&
+                    entry->down_inner == down_inners[load_i];
+            }
+            if (!cache_owned) {
+                ds4_gpu_stream_expert_cache_release_prepared_buffers(
+                        &gate_bufs[load_i],
+                        &up_bufs[load_i],
+                        &down_bufs[load_i],
+                        gate_inners[load_i]);
+            } else {
+                gate_bufs[load_i] = nil;
+                up_bufs[load_i] = nil;
+                down_bufs[load_i] = nil;
+            }
+        }
+    }
     if (tasks) free(tasks);
     if (ok) {
         for (uint32_t u = 0; u < unique_count; u++) {
@@ -17677,6 +20797,15 @@ static int ds4_gpu_stream_expert_cache_prepare_selected_batch(
                 const uint64_t gate_rel = unique_gate_offsets[u] - gate_offset;
                 const uint64_t down_rel = unique_down_offsets[u] - down_offset;
                 if (!*overflow_gate) {
+                    if (gate_expert_bytes > UINT64_MAX / n_total_expert ||
+                        down_expert_bytes > UINT64_MAX / n_total_expert) {
+                        fprintf(stderr,
+                                "ds4: Metal streaming prefill batch selected addr "
+                                "expert tensor size overflow at layer %u\n",
+                                layer);
+                        ok = 0;
+                        break;
+                    }
                     const uint64_t gate_tensor_bytes =
                         (uint64_t)n_total_expert * gate_expert_bytes;
                     const uint64_t down_tensor_bytes =
@@ -17753,13 +20882,41 @@ static int ds4_gpu_stream_expert_cache_prepare_selected_batch(
             (*n_resources)++;
         }
     }
+    if (require_private_addr_tables && view_served != 0) {
+        g_stream_expert_exact_mapped_view_calls++;
+    }
     free(ids);
 
     if (!ok || (*n_resources == 0 && view_served == 0)) {
+        if (persistent_private_tables) {
+            g_stream_expert_exact_persistent_failures++;
+        }
         ds4_gpu_stream_expert_cache_clear_layer(layer);
         return 0;
     }
-    if (!ds4_gpu_stream_expert_cache_addr_buffers(layer,
+    if (require_private_addr_tables &&
+        (!persistent_private_tables || view_served != 0 ||
+         *n_resources != unique_count ||
+         !ds4_gpu_stream_expert_exact_snapshot_addr_buffers(
+                 n_total_expert,
+                 unique_ids,
+                 unique_count,
+                 unique_entries,
+                 gate_addrs,
+                 up_addrs,
+                 down_addrs))) {
+        fprintf(stderr,
+                "ds4: Metal exact-row persistent cache could not publish "
+                "a complete private address snapshot at layer %u\n",
+                layer);
+        if (persistent_private_tables) {
+            g_stream_expert_exact_persistent_failures++;
+        }
+        ds4_gpu_stream_expert_cache_clear_layer(layer);
+        return 0;
+    }
+    if (!require_private_addr_tables &&
+        !ds4_gpu_stream_expert_cache_addr_buffers(layer,
                                                   gate_addrs,
                                                   up_addrs,
                                                   down_addrs)) {
@@ -17801,6 +20958,285 @@ static int ds4_gpu_stream_expert_cache_prepare_selected_batch(
     return 1;
 }
 
+int ds4_gpu_stream_expert_exact_rows_begin_collect(void) {
+    if (!g_initialized && !ds4_gpu_init()) return 0;
+    ds4_gpu_stream_expert_exact_rows_scope *scope =
+        &g_stream_expert_exact_rows_scope;
+    if (!g_ssd_streaming_mode || scope->collecting || scope->active ||
+        scope->row_armed) {
+        return 0;
+    }
+
+    /* A failed ordinary decode must not leak host routing state into the
+     * matrix collected below.  Finish any outstanding read before the exact
+     * scope begins so it cannot install/evict entries while the union address
+     * table is being assembled. */
+    g_routed_moe_selected_override_n = 0;
+    if (!ds4_gpu_stream_expert_pending_load_finish(NULL)) {
+        ds4_gpu_stream_expert_exact_rows_clear();
+        return 0;
+    }
+    scope->collecting = 1;
+    return 1;
+}
+
+static int ds4_gpu_stream_expert_exact_rows_prepare_impl(
+        const ds4_gpu_stream_expert_table *table,
+        const ds4_gpu_tensor              *selected_rows,
+        uint32_t                           n_rows,
+        uint32_t                           n_selected) {
+    if (!g_initialized && !ds4_gpu_init()) return 0;
+    if (!g_ssd_streaming_mode || !table || !selected_rows ||
+        !table->model_map ||
+        table->layer >= DS4_METAL_STREAM_EXPERT_CACHE_MAX_LAYER ||
+        table->n_total_expert == 0 ||
+        table->n_total_expert > DS4_METAL_STREAM_EXPERT_CACHE_MAX_EXPERT ||
+        table->gate_expert_bytes == 0 || table->down_expert_bytes == 0 ||
+        n_rows == 0 || n_rows > DS4_METAL_EXACT_ROWS_MAX ||
+        n_selected == 0 ||
+        n_selected > DS4_METAL_STREAM_EXPERT_CACHE_MAX_SELECTED ||
+        n_rows > UINT32_MAX / n_selected) {
+        return 0;
+    }
+    const uint64_t n_ids = (uint64_t)n_rows * n_selected;
+    const uint64_t selected_bytes = n_ids * sizeof(int32_t);
+    if (ds4_gpu_tensor_bytes(selected_rows) < selected_bytes) return 0;
+
+    ds4_gpu_stream_expert_exact_rows_scope *scope =
+        &g_stream_expert_exact_rows_scope;
+    if (!scope->collecting || scope->active || scope->row_armed || !g_batch_cb) {
+        fprintf(stderr,
+                "ds4: Metal exact-row expert union requires one active, un-nested command batch\n");
+        return 0;
+    }
+
+    /* Router rows are GPU-produced shared memory.  A plain end/commit is not
+     * sufficient for a CPU read; wait on a signal encoded after all routers,
+     * then continue the MoE tails in a fresh ordered command buffer. */
+    if (!ds4_gpu_signal_batch_and_wait_event(
+                "exact-row expert union router readback")) {
+        return 0;
+    }
+    /* The signal is encoded at the end of the router command buffer, hence
+     * all resources registered before it are no longer in use.  Retire that
+     * completed buffer now so layer-local packed experts do not accumulate
+     * over the model's full layer count. */
+    if (!ds4_gpu_wait_pending_command_buffers(
+                "exact-row expert union router boundary")) {
+        return 0;
+    }
+    [g_transient_buffers removeAllObjects];
+    if (!ds4_gpu_tensor_read(selected_rows,
+                             0,
+                             scope->selected_ids,
+                             selected_bytes)) {
+        ds4_gpu_stream_expert_exact_rows_clear();
+        return 0;
+    }
+
+    uint32_t n_resources = 0;
+    uint32_t unique_count = 0;
+    id<MTLBuffer> gate_addrs = nil;
+    id<MTLBuffer> up_addrs = nil;
+    id<MTLBuffer> down_addrs = nil;
+    id<MTLBuffer> overflow_gate = nil;
+    id<MTLBuffer> overflow_up = nil;
+    id<MTLBuffer> overflow_down = nil;
+    if (!ds4_gpu_stream_expert_cache_prepare_selected_batch(
+                table->model_map,
+                table->model_size,
+                table->layer,
+                selected_rows,
+                n_rows,
+                table->n_total_expert,
+                n_selected,
+                table->gate_offset,
+                table->up_offset,
+                table->down_offset,
+                table->gate_expert_bytes,
+                table->down_expert_bytes,
+                &gate_addrs,
+                &up_addrs,
+                &down_addrs,
+                scope->resources,
+                &n_resources,
+                &unique_count,
+                &overflow_gate,
+                &overflow_up,
+                &overflow_down,
+                true)) {
+        ds4_gpu_stream_expert_exact_rows_clear();
+        return 0;
+    }
+    if (n_resources > DS4_METAL_EXACT_ROWS_MAX_RESOURCES ||
+        unique_count == 0 || !gate_addrs || !up_addrs || !down_addrs ||
+        gate_addrs == g_stream_expert_cache_gate_addr_buffers[table->layer] ||
+        up_addrs == g_stream_expert_cache_up_addr_buffers[table->layer] ||
+        down_addrs == g_stream_expert_cache_down_addr_buffers[table->layer] ||
+        gate_addrs == g_stream_compact_gate_addr_buffers[table->layer] ||
+        up_addrs == g_stream_compact_up_addr_buffers[table->layer] ||
+        down_addrs == g_stream_compact_down_addr_buffers[table->layer]) {
+        fprintf(stderr,
+                "ds4: Metal exact-row expert union did not produce private address tables\n");
+        ds4_gpu_stream_expert_exact_rows_clear();
+        return 0;
+    }
+
+    for (uint32_t i = 0; i < n_resources; i++) {
+        ds4_gpu_stream_expert_cache_entry *entry = scope->resources[i];
+        if (!entry || !entry->valid || !entry->gate_buffer ||
+            !entry->up_buffer || !entry->down_buffer) {
+            ds4_gpu_stream_expert_exact_rows_clear();
+            return 0;
+        }
+        /* Retain immutable resources independently of cache ownership.  The
+         * address table remains valid even if a later implementation permits
+         * cache maintenance while the exact-row command buffer is in flight. */
+        scope->resource_gate[i] = entry->gate_buffer;
+        scope->resource_up[i] = entry->up_buffer;
+        scope->resource_down[i] = entry->down_buffer;
+    }
+    /* Pin every persistent-cache hit to the command-buffer epoch before the
+     * scope becomes visible.  Exact rows deliberately bypass all ordinary
+     * cache setup/pruning below, but this also makes an accidental cache
+     * maintenance attempt fail safely instead of invalidating an entry whose
+     * GPU address is already published in the private table. */
+    if (n_resources != 0 &&
+        !ds4_gpu_stream_expert_cache_mark_entries_inflight(scope->resources,
+                                                           n_resources,
+                                                           0)) {
+        fprintf(stderr,
+                "ds4: Metal exact-row expert union could not pin cache resources\n");
+        ds4_gpu_stream_expert_exact_rows_clear();
+        return 0;
+    }
+    scope->table = *table;
+    scope->selected_buffer = ds4_gpu_tensor_buffer(selected_rows);
+    scope->selected_offset = ds4_gpu_tensor_offset(selected_rows);
+    scope->gate_addrs = gate_addrs;
+    scope->up_addrs = up_addrs;
+    scope->down_addrs = down_addrs;
+    scope->overflow_gate = overflow_gate;
+    scope->overflow_up = overflow_up;
+    scope->overflow_down = overflow_down;
+    scope->n_rows = n_rows;
+    scope->n_selected = n_selected;
+    scope->n_resources = n_resources;
+    scope->unique_count = unique_count;
+    scope->row = 0;
+    scope->next_row = 0;
+    scope->row_armed = 0;
+    scope->active = 1;
+
+    if (getenv("DS4_METAL_DSPARK_EXACT_ROWS_PROFILE") != NULL) {
+        fprintf(stderr,
+                "ds4: Metal exact-row expert union layer=%u rows=%u "
+                "selected=%u unique=%u resident=%u transient=%u\n",
+                table->layer,
+                n_rows,
+                n_selected,
+                unique_count,
+                n_resources,
+                scope->overflow_gate != nil);
+    }
+    return 1;
+}
+
+int ds4_gpu_stream_expert_exact_rows_prepare(
+        const ds4_gpu_stream_expert_table *table,
+        const ds4_gpu_tensor              *selected_rows,
+        uint32_t                           n_rows,
+        uint32_t                           n_selected) {
+    /* ARC may autorelease intermediate Metal objects created by the selected
+     * loader. The exact scope has already retained everything needed by the
+     * GPU, so drain those temporary references at every layer boundary. */
+    @autoreleasepool {
+        return ds4_gpu_stream_expert_exact_rows_prepare_impl(
+                table, selected_rows, n_rows, n_selected);
+    }
+}
+
+int ds4_gpu_stream_expert_exact_rows_set_row(uint32_t row) {
+    ds4_gpu_stream_expert_exact_rows_scope *scope =
+        &g_stream_expert_exact_rows_scope;
+    if (!scope->active || scope->row_armed || row >= scope->n_rows ||
+        row != scope->next_row) {
+        return 0;
+    }
+    scope->row = row;
+    scope->row_armed = 1;
+    return 1;
+}
+
+static int ds4_gpu_stream_expert_exact_rows_end_async_impl(void) {
+    if (!g_initialized && !ds4_gpu_init()) return 0;
+    ds4_gpu_stream_expert_exact_rows_scope *scope =
+        &g_stream_expert_exact_rows_scope;
+    if (!scope->collecting || !scope->active || scope->row_armed ||
+        scope->next_row != scope->n_rows || !g_batch_cb) {
+        return 0;
+    }
+
+    /* Command buffers normally retain encoded resources, but the diagnostic
+     * DS4_METAL_UNRETAINED_COMMAND_BUFFERS mode deliberately disables that.
+     * Capture the complete immutable scope in the completion handler so the
+     * asynchronous boundary is correct in both modes.  Cache-entry structs
+     * themselves live in the static cache; their three buffers are retained
+     * independently so later cache maintenance cannot invalidate an address
+     * already published in a private table. */
+    NSMutableArray<id<MTLBuffer>> *resources = [NSMutableArray array];
+#define DS4_EXACT_ROWS_RETAIN(buffer_) do { \
+        id<MTLBuffer> retained_buffer_ = (buffer_); \
+        if (retained_buffer_) [resources addObject:retained_buffer_]; \
+    } while (0)
+    DS4_EXACT_ROWS_RETAIN(scope->selected_buffer);
+    DS4_EXACT_ROWS_RETAIN(scope->gate_addrs);
+    DS4_EXACT_ROWS_RETAIN(scope->up_addrs);
+    DS4_EXACT_ROWS_RETAIN(scope->down_addrs);
+    DS4_EXACT_ROWS_RETAIN(scope->overflow_gate);
+    DS4_EXACT_ROWS_RETAIN(scope->overflow_up);
+    DS4_EXACT_ROWS_RETAIN(scope->overflow_down);
+    for (uint32_t i = 0; i < scope->n_resources; i++) {
+        DS4_EXACT_ROWS_RETAIN(scope->resource_gate[i]);
+        DS4_EXACT_ROWS_RETAIN(scope->resource_up[i]);
+        DS4_EXACT_ROWS_RETAIN(scope->resource_down[i]);
+    }
+#undef DS4_EXACT_ROWS_RETAIN
+    NSArray<id<MTLBuffer>> *retained_resources = [resources copy];
+
+    ds4_gpu_close_batch_encoder();
+    id<MTLCommandBuffer> cb = g_batch_cb;
+    g_batch_cb = nil;
+    g_batch_has_work = NO;
+    [cb addCompletedHandler:^(id<MTLCommandBuffer> completed) {
+        (void)completed;
+        /* Referencing the array is intentional: the block owns the last
+         * scope-independent strong refs until Metal invokes it. */
+        (void)[retained_resources count];
+    }];
+    [cb commit];
+    [g_pending_cbs addObject:cb];
+    ds4_gpu_stream_expert_cache_note_batch_committed();
+    return 1;
+}
+
+int ds4_gpu_stream_expert_exact_rows_end_async(void) {
+    /* The completion block owns its copied array; drain the temporary mutable
+     * array instead of retaining all expert buffers in the C caller's pool. */
+    @autoreleasepool {
+        return ds4_gpu_stream_expert_exact_rows_end_async_impl();
+    }
+}
+
+void ds4_gpu_stream_expert_exact_rows_release(void) {
+    /* release() is also the fail-clean exit for a collection that did not
+     * reach prepare().  Neither ordinary selected state is allowed to escape
+     * the exact scope. */
+    g_routed_moe_selected_override_n = 0;
+    ds4_gpu_stream_expert_pending_load_clear();
+    ds4_gpu_stream_expert_exact_rows_clear();
+}
+
 int ds4_gpu_stream_expert_cache_seed_selected(
         const ds4_gpu_stream_expert_table *table,
         const int32_t                     *selected_ids,
@@ -18267,7 +21703,18 @@ static id ds4_gpu_q4_expert_layer_residency_set(DS4MetalQ4ExpertTable *gate_tabl
                          gate_table, up_table, down_table];
         DS4MetalQ4LayerResidency *cached =
             [g_q4_expert_layer_residency_cache objectForKey:key];
-        if (cached) return cached.residencySet;
+        if (cached) {
+            if (queue_residency) cached.queueResident = YES;
+            if (cached.queueResident) {
+                for (int i = 0; i < DS4_GPU_MAX_STREAMS; i++) {
+                    id<MTLCommandQueue> queue =
+                        i == 0 ? g_queue : g_stream_queues[i];
+                    ds4_gpu_q4_attach_layer_residency_to_queue(
+                            cached, queue, i);
+                }
+            }
+            return cached.residencySet;
+        }
 
         const NSUInteger capacity =
             [gate_table.expertBuffers count] +
@@ -18293,9 +21740,13 @@ static id ds4_gpu_q4_expert_layer_residency_set(DS4MetalQ4ExpertTable *gate_tabl
 
         DS4MetalQ4LayerResidency *entry = [DS4MetalQ4LayerResidency new];
         entry.residencySet = residency_set;
+        entry.queueResident = queue_residency;
         if (queue_residency) {
-            [g_queue addResidencySet:residency_set];
-            entry.addedToQueue = YES;
+            for (int i = 0; i < DS4_GPU_MAX_STREAMS; i++) {
+                id<MTLCommandQueue> queue =
+                    i == 0 ? g_queue : g_stream_queues[i];
+                ds4_gpu_q4_attach_layer_residency_to_queue(entry, queue, i);
+            }
         }
         [g_q4_expert_layer_residency_cache setObject:entry forKey:key];
         return residency_set;
@@ -19006,6 +22457,230 @@ int ds4_gpu_indexer_scores_decode_batch_tensor(
                                                  scale);
 }
 
+typedef struct {
+    uint32_t vocab;
+    uint32_t rank_blocks;
+    uint32_t hidden_dim;
+    uint32_t n_drafts;
+    uint32_t draft;
+    uint32_t reuse_confidence0;
+    float confidence_threshold;
+    float confidence0;
+} ds4_metal_dspark_device_args;
+
+_Static_assert(sizeof(ds4_gpu_dspark_device_proposal) == 64u,
+               "DSpark device proposal ABI must stay 64 bytes");
+_Static_assert(offsetof(ds4_gpu_dspark_device_proposal, tokens) == 0u &&
+               offsetof(ds4_gpu_dspark_device_proposal,
+                        confidence_logits) == 24u &&
+               offsetof(ds4_gpu_dspark_device_proposal, proposal_len) == 48u &&
+               offsetof(ds4_gpu_dspark_device_proposal, confidence_len) == 52u &&
+               offsetof(ds4_gpu_dspark_device_proposal, status) == 56u &&
+               offsetof(ds4_gpu_dspark_device_proposal, reserved) == 60u,
+               "DSpark device proposal field offsets changed");
+_Static_assert(DS4_GPU_DSPARK_DEVICE_PROPOSAL_BYTES >= 1168u,
+               "DSpark Metal device proposal scratch is too small");
+
+static int ds4_gpu_metal_dspark_device_proposer_enabled(void) {
+    return ds4_gpu_env_bool("DS4_METAL_DSPARK_DEVICE_PROPOSER") == 1 &&
+           getenv("DS4_METAL_DSPARK_NO_DEVICE_PROPOSER") == NULL;
+}
+
+int ds4_gpu_dspark_markov_confidence_q8_tensor(
+        ds4_gpu_tensor       *out_result,
+        const ds4_gpu_tensor *logits_rows,
+        const ds4_gpu_tensor *hidden_rows,
+        const void           *model_map,
+        uint64_t              model_size,
+        uint64_t              w1_offset,
+        uint64_t              w2_offset,
+        uint64_t              confidence_offset,
+        uint32_t              first_prev_token,
+        uint32_t              vocab,
+        uint32_t              rank,
+        uint32_t              hidden_dim,
+        uint32_t              n_drafts,
+        float                 confidence_threshold,
+        int                   reuse_confidence0,
+        float                 confidence0) {
+    if (!g_initialized && !ds4_gpu_init()) return 0;
+    if (!ds4_gpu_metal_dspark_device_proposer_enabled() ||
+        !out_result || !logits_rows || !hidden_rows || !model_map ||
+        g_batch_cb || g_quality_mode || vocab == 0u ||
+        first_prev_token >= vocab || rank == 0u || rank > 256u ||
+        (rank & 31u) != 0u || hidden_dim == 0u ||
+        (hidden_dim & 31u) != 0u || n_drafts == 0u ||
+        n_drafts > DS4_GPU_DSPARK_MAX_DRAFTS ||
+        !(confidence_threshold > 0.0f && confidence_threshold <= 1.0f) ||
+        ds4_gpu_tensor_bytes(out_result) <
+            DS4_GPU_DSPARK_DEVICE_PROPOSAL_BYTES) {
+        return 0;
+    }
+    if ((uint64_t)n_drafts > UINT64_MAX / vocab / sizeof(float) ||
+        ds4_gpu_tensor_bytes(logits_rows) <
+            (uint64_t)n_drafts * vocab * sizeof(float) ||
+        (uint64_t)n_drafts > UINT64_MAX / hidden_dim / sizeof(float) ||
+        ds4_gpu_tensor_bytes(hidden_rows) <
+            (uint64_t)n_drafts * hidden_dim * sizeof(float)) {
+        return 0;
+    }
+
+    const uint32_t rank_blocks = rank / 32u;
+    const uint64_t markov_row_bytes = (uint64_t)rank_blocks * 34u;
+    if ((uint64_t)vocab > UINT64_MAX / markov_row_bytes) return 0;
+    const uint64_t markov_bytes = (uint64_t)vocab * markov_row_bytes;
+    const uint64_t feature_dim = (uint64_t)hidden_dim + rank;
+    if (feature_dim > UINT32_MAX || (feature_dim & 31u) != 0u) return 0;
+    const uint64_t confidence_bytes = (feature_dim / 32u) * 34u;
+    if (w1_offset > model_size || markov_bytes > model_size - w1_offset ||
+        w2_offset > model_size || markov_bytes > model_size - w2_offset ||
+        confidence_offset > model_size ||
+        confidence_bytes > model_size - confidence_offset) {
+        return 0;
+    }
+
+    @autoreleasepool {
+        id<MTLBuffer> resultbuf = ds4_gpu_tensor_buffer(out_result);
+        id<MTLBuffer> logitsbuf = ds4_gpu_tensor_buffer(logits_rows);
+        id<MTLBuffer> hiddenbuf = ds4_gpu_tensor_buffer(hidden_rows);
+        if (!resultbuf || !logitsbuf || !hiddenbuf) return 0;
+
+        uint64_t w1_inner = 0;
+        uint64_t w2_inner = 0;
+        uint64_t confidence_inner = 0;
+        id<MTLBuffer> w1buf = ds4_gpu_wrap_q8_decode_model_range(
+                model_map, model_size, w1_offset, markov_bytes, n_drafts,
+                &w1_inner);
+        id<MTLBuffer> w2buf = ds4_gpu_wrap_q8_decode_model_range(
+                model_map, model_size, w2_offset, markov_bytes, n_drafts,
+                &w2_inner);
+        id<MTLBuffer> confidencebuf =
+            ds4_gpu_wrap_q8_decode_model_range(
+                model_map, model_size, confidence_offset, confidence_bytes,
+                n_drafts, &confidence_inner);
+        if (!w1buf || !w2buf || !confidencebuf) return 0;
+
+        id<MTLComputePipelineState> init_pipeline = ds4_gpu_get_pipeline(
+                "kernel_dsv4_dspark_device_proposal_init");
+        id<MTLComputePipelineState> confidence_pipeline =
+            ds4_gpu_get_pipeline(
+                "kernel_dsv4_dspark_device_confidence_q8");
+        id<MTLComputePipelineState> scan_pipeline = ds4_gpu_get_pipeline(
+                "kernel_dsv4_dspark_device_markov_scan_q8");
+        id<MTLComputePipelineState> reduce_pipeline = ds4_gpu_get_pipeline(
+                "kernel_dsv4_dspark_device_markov_reduce");
+        id<MTLComputePipelineState> export_pipeline = ds4_gpu_get_pipeline(
+                "kernel_dsv4_dspark_device_proposal_export");
+        if (!init_pipeline || !confidence_pipeline || !scan_pipeline ||
+            !reduce_pipeline || !export_pipeline) {
+            return 0;
+        }
+
+        int owned = 0;
+        id<MTLCommandBuffer> cb = ds4_gpu_command_buffer(&owned);
+        if (!cb || !owned) return 0;
+
+        const NSUInteger result_offset =
+            ds4_gpu_tensor_offset(out_result);
+        const NSUInteger state_offset =
+            result_offset + sizeof(ds4_gpu_dspark_device_proposal);
+        const NSUInteger logits_offset =
+            ds4_gpu_tensor_offset(logits_rows);
+        const NSUInteger hidden_offset =
+            ds4_gpu_tensor_offset(hidden_rows);
+        const NSUInteger logits_row_bytes =
+            (NSUInteger)vocab * sizeof(float);
+        id<MTLResource> state_barrier[1] = { resultbuf };
+
+        id<MTLComputeCommandEncoder> enc = ds4_gpu_compute_encoder(cb);
+        [enc setComputePipelineState:init_pipeline];
+        [enc setBytes:&first_prev_token
+               length:sizeof(first_prev_token)
+              atIndex:0];
+        [enc setBuffer:resultbuf offset:state_offset atIndex:1];
+        [enc dispatchThreadgroups:MTLSizeMake(1, 1, 1)
+             threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
+        [enc memoryBarrierWithResources:state_barrier count:1u];
+        ds4_gpu_end_compute_encoder(cb, enc);
+
+        ds4_metal_dspark_device_args args = {
+            .vocab = vocab,
+            .rank_blocks = rank_blocks,
+            .hidden_dim = hidden_dim,
+            .n_drafts = n_drafts,
+            .draft = 0,
+            .reuse_confidence0 = reuse_confidence0 ? 1u : 0u,
+            .confidence_threshold = confidence_threshold,
+            .confidence0 = confidence0,
+        };
+        for (uint32_t draft = 0; draft < n_drafts; draft++) {
+            args.draft = draft;
+
+            enc = ds4_gpu_compute_encoder(cb);
+            [enc setComputePipelineState:confidence_pipeline];
+            [enc setBytes:&args length:sizeof(args) atIndex:0];
+            [enc setBuffer:hiddenbuf offset:hidden_offset atIndex:1];
+            [enc setBuffer:w1buf offset:(NSUInteger)w1_inner atIndex:2];
+            [enc setBuffer:confidencebuf
+                     offset:(NSUInteger)confidence_inner
+                    atIndex:3];
+            [enc setBuffer:resultbuf offset:state_offset atIndex:4];
+            [enc dispatchThreadgroups:MTLSizeMake(1, 1, 1)
+                 threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
+            [enc memoryBarrierWithResources:state_barrier count:1u];
+            ds4_gpu_end_compute_encoder(cb, enc);
+
+            enc = ds4_gpu_compute_encoder(cb);
+            [enc setComputePipelineState:scan_pipeline];
+            [enc setBytes:&args length:sizeof(args) atIndex:0];
+            [enc setBuffer:logitsbuf
+                     offset:logits_offset +
+                            (NSUInteger)draft * logits_row_bytes
+                    atIndex:1];
+            [enc setBuffer:w1buf offset:(NSUInteger)w1_inner atIndex:2];
+            [enc setBuffer:w2buf offset:(NSUInteger)w2_inner atIndex:3];
+            [enc setBuffer:resultbuf offset:state_offset atIndex:4];
+            [enc dispatchThreadgroups:MTLSizeMake(128, 1, 1)
+                 threadsPerThreadgroup:MTLSizeMake(256, 1, 1)];
+            [enc memoryBarrierWithResources:state_barrier count:1u];
+            ds4_gpu_end_compute_encoder(cb, enc);
+
+            enc = ds4_gpu_compute_encoder(cb);
+            [enc setComputePipelineState:reduce_pipeline];
+            [enc setBytes:&args length:sizeof(args) atIndex:0];
+            [enc setBuffer:resultbuf offset:state_offset atIndex:1];
+            [enc dispatchThreadgroups:MTLSizeMake(1, 1, 1)
+                 threadsPerThreadgroup:MTLSizeMake(128, 1, 1)];
+            [enc memoryBarrierWithResources:state_barrier count:1u];
+            ds4_gpu_end_compute_encoder(cb, enc);
+        }
+
+        enc = ds4_gpu_compute_encoder(cb);
+        [enc setComputePipelineState:export_pipeline];
+        [enc setBuffer:resultbuf offset:result_offset atIndex:0];
+        [enc setBuffer:resultbuf offset:state_offset atIndex:1];
+        [enc dispatchThreadgroups:MTLSizeMake(1, 1, 1)
+             threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
+        [enc memoryBarrierWithResources:state_barrier count:1u];
+        ds4_gpu_end_compute_encoder(cb, enc);
+
+        if (!ds4_gpu_finish_command_buffer(
+                cb, owned, "DSpark Metal device proposer")) {
+            return 0;
+        }
+
+        static int logged = 0;
+        if (!logged) {
+            logged = 1;
+            fprintf(stderr,
+                    "ds4: Metal DSpark device proposer enabled "
+                    "(Q8 confidence/Markov, max_drafts=%u)\n",
+                    DS4_GPU_DSPARK_MAX_DRAFTS);
+        }
+    }
+    return 1;
+}
+
 int ds4_gpu_indexer_topk_tensor(
         ds4_gpu_tensor       *selected,
         const ds4_gpu_tensor *scores,
@@ -19137,17 +22812,145 @@ int ds4_gpu_indexer_topk_tensor(
     return 1;
 }
 
-int ds4_gpu_argmax_tensor(
+/* Dedicated two-dispatch top-1 reduction for decode logits.  The generic
+ * indexer path sorts every block and merges its winners even though argmax
+ * consumes one index.  Stage 1 scans fixed vocabulary chunks; stage 2 merges
+ * the stage-1 (value,index) pairs.  Separate dispatches provide the device-wide
+ * ordering that a single-dispatch completion counter cannot guarantee. */
+static int ds4_gpu_argmax_top1_tensor(
         ds4_gpu_tensor       *out_idx,
         const ds4_gpu_tensor *logits,
         uint32_t                n_vocab) {
+    enum {
+        DS4_ARGMAX_TOP1_SLOTS = 8u,
+        DS4_ARGMAX_TOP1_GROUPS = 128u,
+        DS4_ARGMAX_TOP1_THREADS = 256u,
+    };
+    if (!g_initialized && !ds4_gpu_init()) return 0;
     if (!out_idx || !logits || n_vocab == 0) return 0;
     if (ds4_gpu_tensor_bytes(out_idx) < sizeof(int32_t) ||
         ds4_gpu_tensor_bytes(logits) < (uint64_t)n_vocab * sizeof(float)) {
-        fprintf(stderr, "ds4: Metal graph argmax received undersized buffers\n");
+        fprintf(stderr, "ds4: Metal top-1 argmax received undersized buffers\n");
         return 0;
     }
 
+    @autoreleasepool {
+        ds4_gpu_stream_scratch_state *stream_scratch =
+            &g_stream_scratch[g_ds4_stream];
+        const uint32_t slot =
+            stream_scratch->argmax_top1_seq++ % DS4_ARGMAX_TOP1_SLOTS;
+        if (!stream_scratch->argmax_top1_scratch_v ||
+            !stream_scratch->argmax_top1_scratch_i) {
+            stream_scratch->argmax_top1_scratch_v = nil;
+            stream_scratch->argmax_top1_scratch_i = nil;
+            stream_scratch->argmax_top1_scratch_v =
+                [g_device newBufferWithLength:
+                    DS4_ARGMAX_TOP1_SLOTS * DS4_ARGMAX_TOP1_GROUPS *
+                        sizeof(float)
+                    options:MTLResourceStorageModeShared];
+            stream_scratch->argmax_top1_scratch_i =
+                [g_device newBufferWithLength:
+                    DS4_ARGMAX_TOP1_SLOTS * DS4_ARGMAX_TOP1_GROUPS *
+                        sizeof(int32_t)
+                    options:MTLResourceStorageModeShared];
+            if (!stream_scratch->argmax_top1_scratch_v ||
+                !stream_scratch->argmax_top1_scratch_i) {
+                stream_scratch->argmax_top1_scratch_v = nil;
+                stream_scratch->argmax_top1_scratch_i = nil;
+                return -1;
+            }
+        }
+        if (!g_dsv4_argmax_top1_stage1_pipeline ||
+            !g_dsv4_argmax_top1_stage2_pipeline) {
+            g_dsv4_argmax_top1_stage1_pipeline =
+                ds4_gpu_get_pipeline(
+                    "kernel_dsv4_argmax_top1_stage1_f32");
+            g_dsv4_argmax_top1_stage2_pipeline =
+                ds4_gpu_get_pipeline(
+                    "kernel_dsv4_argmax_top1_stage2_f32");
+            if (!g_dsv4_argmax_top1_stage1_pipeline ||
+                !g_dsv4_argmax_top1_stage2_pipeline) {
+                g_dsv4_argmax_top1_stage1_pipeline = nil;
+                g_dsv4_argmax_top1_stage2_pipeline = nil;
+                return -1;
+            }
+        }
+        if (g_dsv4_argmax_top1_stage1_pipeline.threadExecutionWidth != 32u ||
+            g_dsv4_argmax_top1_stage2_pipeline.threadExecutionWidth != 32u ||
+            g_dsv4_argmax_top1_stage1_pipeline.maxTotalThreadsPerThreadgroup <
+                DS4_ARGMAX_TOP1_THREADS ||
+            g_dsv4_argmax_top1_stage2_pipeline.maxTotalThreadsPerThreadgroup <
+                DS4_ARGMAX_TOP1_THREADS) {
+            return -1;
+        }
+
+        const NSUInteger scratch_v_offset =
+            (NSUInteger)slot * DS4_ARGMAX_TOP1_GROUPS * sizeof(float);
+        const NSUInteger scratch_i_offset =
+            (NSUInteger)slot * DS4_ARGMAX_TOP1_GROUPS * sizeof(int32_t);
+
+        ds4_gpu_dsv4_argmax_top1_args args = {
+            .n_vocab = (int32_t)n_vocab,
+            .n_tg = DS4_ARGMAX_TOP1_GROUPS,
+        };
+        int owned = 0;
+        id<MTLCommandBuffer> cb = ds4_gpu_command_buffer(&owned);
+        if (!cb) return 0;
+
+        id<MTLComputeCommandEncoder> enc = ds4_gpu_compute_encoder(cb);
+        if (!enc) return 0;
+        [enc setComputePipelineState:g_dsv4_argmax_top1_stage1_pipeline];
+        [enc setBytes:&args length:sizeof(args) atIndex:0];
+        [enc setBuffer:ds4_gpu_tensor_buffer(logits)
+                offset:ds4_gpu_tensor_offset(logits) atIndex:1];
+        [enc setBuffer:stream_scratch->argmax_top1_scratch_v
+                offset:scratch_v_offset atIndex:2];
+        [enc setBuffer:stream_scratch->argmax_top1_scratch_i
+                offset:scratch_i_offset atIndex:3];
+        [enc setThreadgroupMemoryLength:64u atIndex:0];
+        [enc dispatchThreadgroups:MTLSizeMake(DS4_ARGMAX_TOP1_GROUPS, 1, 1)
+             threadsPerThreadgroup:
+                 MTLSizeMake(DS4_ARGMAX_TOP1_THREADS, 1, 1)];
+
+        [enc setComputePipelineState:g_dsv4_argmax_top1_stage2_pipeline];
+        [enc setBytes:&args length:sizeof(args) atIndex:0];
+        [enc setBuffer:stream_scratch->argmax_top1_scratch_v
+                offset:scratch_v_offset atIndex:1];
+        [enc setBuffer:stream_scratch->argmax_top1_scratch_i
+                offset:scratch_i_offset atIndex:2];
+        [enc setBuffer:ds4_gpu_tensor_buffer(out_idx)
+                offset:ds4_gpu_tensor_offset(out_idx) atIndex:3];
+        [enc setThreadgroupMemoryLength:64u atIndex:0];
+        [enc dispatchThreadgroups:MTLSizeMake(1, 1, 1)
+             threadsPerThreadgroup:
+                 MTLSizeMake(DS4_ARGMAX_TOP1_THREADS, 1, 1)];
+        ds4_gpu_end_compute_encoder(cb, enc);
+
+        if (!ds4_gpu_finish_command_buffer(cb, owned,
+                                           "top-1 argmax")) {
+            return 0;
+        }
+    }
+    return 1;
+}
+
+int ds4_gpu_argmax_tensor(
+        ds4_gpu_tensor       *out_idx,
+        const ds4_gpu_tensor *logits,
+        uint32_t                n_vocab) {
+    /* Production decode has 129,280 logits.  Small rows already fit the
+     * generic one-pass sorter and do not amortize the scan grid. */
+    const bool large_row =
+        n_vocab >= 4096u && n_vocab <= (uint32_t)INT32_MAX;
+    const bool require_top1 =
+        getenv("DS4_METAL_REQUIRE_DECODE_ARGMAX_TOP1") != NULL;
+    if (large_row && !g_quality_mode && !g_batch_encoder_concurrent &&
+        getenv("DS4_METAL_DISABLE_DECODE_ARGMAX_TOP1") == NULL) {
+        const int top1 =
+            ds4_gpu_argmax_top1_tensor(out_idx, logits, n_vocab);
+        if (top1 >= 0) return top1;
+    }
+    if (large_row && require_top1) return 0;
     return ds4_gpu_indexer_topk_tensor(out_idx, logits, n_vocab, 1, 1);
 }
 
@@ -19278,7 +23081,7 @@ static int ds4_gpu_matmul_q8_0_legacy_tensor(
                     [enc setBuffer:wbuf offset:(NSUInteger)inner_offset atIndex:1];
                     [enc setBuffer:xbuf offset:ds4_gpu_tensor_offset(x) atIndex:2];
                     [enc setBuffer:outbuf offset:ds4_gpu_tensor_offset(out) atIndex:3];
-                    [enc setThreadgroupMemoryLength:64u * 32u * sizeof(uint16_t) atIndex:0];
+                    [enc setThreadgroupMemoryLength:DS4_METAL_MPP_DIRECT_RHS_SMEM atIndex:0];
                     [enc dispatchThreadgroups:MTLSizeMake(1u,
                                                           ((NSUInteger)out_dim + 63u) / 64u,
                                                           1u)
@@ -19403,7 +23206,7 @@ static int ds4_gpu_matmul_q8_0_legacy_tensor(
                 [enc setBuffer:wbuf offset:(NSUInteger)inner_offset atIndex:1];
                 [enc setBuffer:xbuf offset:ds4_gpu_tensor_offset(x) atIndex:2];
                 [enc setBuffer:outbuf offset:ds4_gpu_tensor_offset(out) atIndex:3];
-                [enc setThreadgroupMemoryLength:2u * 64u * 32u * sizeof(uint16_t) atIndex:0];
+                [enc setThreadgroupMemoryLength:DS4_METAL_MPP_DIRECT_RHS_SMEM atIndex:0];
                 [enc dispatchThreadgroups:MTLSizeMake((NSUInteger)(nax_rows / nax_tile_n),
                                                       (NSUInteger)out_dim / 64u,
                                                       1)
@@ -19665,10 +23468,38 @@ int ds4_gpu_matmul_q8_0_decode_mpp_model_view_tensor(
     }
 }
 
-static const char *ds4_gpu_q4_mm_name(uint32_t weight_type) {
+static const char *ds4_gpu_q4_mm_name(
+        uint32_t weight_type,
+        uint64_t in_dim,
+        uint64_t out_dim,
+        uint64_t n_tok) {
     switch (weight_type) {
     case DS4_METAL_TENSOR_Q4_0: return "kernel_mul_mm_q4_0_f32";
-    case DS4_METAL_TENSOR_Q4_K: return "kernel_mul_mm_q4_K_f32";
+    case DS4_METAL_TENSOR_Q4_K: {
+        /*
+         * The legacy 64x32 kernel maps token rows 0..15 to SIMDgroups 0/1
+         * and rows 16..31 to SIMDgroups 2/3.  When the final token tile has
+         * at most 16 rows, the latter pair can skip its fragment loads, MMA,
+         * and store while all four SIMDgroups still perform cooperative
+         * staging and reach every threadgroup barrier.  Keep a narrow
+         * rollback and avoid the candidate when it cannot cull a whole pair.
+         */
+        const uint64_t tail = n_tok % 32u;
+        const bool measured_platform =
+            ds4_gpu_device_is_pre_m5_apple_silicon();
+        const bool measured_output_geometry = (out_dim % 64u) == 0u;
+        const bool production_q_b =
+            in_dim == 1024u && out_dim == 32768u && n_tok <= 65u;
+        const bool single_tile_prefill = n_tok <= 16u;
+        const bool tail_cull =
+            n_tok > 8u && tail > 0u && tail <= 16u &&
+            measured_platform &&
+            measured_output_geometry &&
+            (single_tile_prefill || production_q_b) &&
+            getenv("DS4_METAL_DISABLE_Q4_PREFILL_TAIL_SIMDGROUP_CULL") == NULL;
+        return tail_cull ? "kernel_mul_mm_q4_K_f32_tail_cull"
+                         : "kernel_mul_mm_q4_K_f32";
+    }
     default: return NULL;
     }
 }
@@ -19863,7 +23694,7 @@ static int ds4_gpu_matmul_quant_impl_tensor(
                 [enc setBuffer:wbuf offset:(NSUInteger)inner_offset atIndex:1];
                 [enc setBuffer:xbuf offset:ds4_gpu_tensor_offset(x) atIndex:2];
                 [enc setBuffer:outbuf offset:ds4_gpu_tensor_offset(out) atIndex:3];
-                [enc setThreadgroupMemoryLength:64u * 32u * sizeof(uint16_t) atIndex:0];
+                [enc setThreadgroupMemoryLength:DS4_METAL_MPP_DIRECT_RHS_SMEM atIndex:0];
                 [enc dispatchThreadgroups:MTLSizeMake((NSUInteger)(n_tok / nax_tile_n),
                                                       (NSUInteger)out_dim / 64u,
                                                       1)
@@ -19876,7 +23707,8 @@ static int ds4_gpu_matmul_quant_impl_tensor(
             ds4_gpu_warn_mpp_fallback();
         }
 
-        const char *mm_fn = ds4_gpu_q4_mm_name(weight_type);
+        const char *mm_fn = ds4_gpu_q4_mm_name(
+            weight_type, in_dim, out_dim, n_tok);
         const bool bc_inp = (in_dim % 32u) != 0;
         const bool bc_out = (out_dim % 64u) != 0 || (n_tok % 32u) != 0;
         id<MTLComputePipelineState> pipeline =
@@ -20170,6 +24002,491 @@ int ds4_gpu_matmul_q8_0_pair_tensor(
     return 1;
 }
 
+/* Experimental q_a/KV prefill pair.  The legacy M64xN32 Q4 kernel narrows
+ * every 32xK RHS tile once per 64-row output band.  q_a and KV therefore
+ * replay the same F32->F16 conversion 24 times at the 1024+512 production
+ * shape.  Materialize one stream-local half RHS and feed it to both unchanged
+ * 128-thread kernels instead.  A fixed-capacity sidecar makes reuse safe for
+ * retained and unretained command-buffer modes; command-queue ordering plus
+ * tracked resource hazards serialize consecutive uses on one stream. */
+static int ds4_gpu_try_q4_K_prefill_pair_f16_rhs(
+        ds4_gpu_tensor *out0, ds4_gpu_tensor *out1,
+        const void *model_map, uint64_t model_size,
+        uint64_t weight0_offset, uint64_t weight1_offset,
+        uint64_t in_dim, uint64_t out0_dim, uint64_t out1_dim,
+        const ds4_gpu_tensor *x, uint64_t n_tok) {
+    const bool exact_tokens =
+        n_tok >= 32u && n_tok <= DS4_METAL_Q4_PAIR_RHS_MAX_TOKENS &&
+        (n_tok % 32u) == 0u;
+    if (!out0 || !out1 || !model_map || !x || !exact_tokens ||
+        !ds4_gpu_device_is_pre_m5_apple_silicon() ||
+        g_batch_encoder_concurrent ||
+        g_ds4_stream < 0 || g_ds4_stream >= DS4_GPU_MAX_STREAMS ||
+        in_dim == 0u || in_dim > DS4_METAL_Q4_PAIR_RHS_MAX_IN ||
+        (in_dim % 256u) != 0u ||
+        out0_dim == 0u || out1_dim == 0u ||
+        out0_dim > INT32_MAX || out1_dim > INT32_MAX ||
+        (out0_dim % 64u) != 0u || (out1_dim % 64u) != 0u ||
+        ds4_gpu_env_bool("DS4_METAL_DISABLE_CONTIG_F32_F16_COPY") == 1) {
+        return 0;
+    }
+
+    const uint64_t row_bytes = (in_dim / 256u) * 144u;
+    if (out0_dim > UINT64_MAX / row_bytes ||
+        out1_dim > UINT64_MAX / row_bytes ||
+        in_dim > UINT64_MAX / n_tok / sizeof(float) ||
+        out0_dim > UINT64_MAX / n_tok / sizeof(float) ||
+        out1_dim > UINT64_MAX / n_tok / sizeof(float)) {
+        return 0;
+    }
+    const uint64_t w0_bytes = out0_dim * row_bytes;
+    const uint64_t w1_bytes = out1_dim * row_bytes;
+    const uint64_t x_bytes = in_dim * n_tok * sizeof(float);
+    const uint64_t o0_bytes = out0_dim * n_tok * sizeof(float);
+    const uint64_t o1_bytes = out1_dim * n_tok * sizeof(float);
+    if (weight0_offset > model_size || w0_bytes > model_size - weight0_offset ||
+        weight1_offset > model_size || w1_bytes > model_size - weight1_offset ||
+        ds4_gpu_tensor_bytes(x) < x_bytes ||
+        ds4_gpu_tensor_bytes(out0) < o0_bytes ||
+        ds4_gpu_tensor_bytes(out1) < o1_bytes ||
+        ds4_gpu_tensor_prefixes_overlap(x, x_bytes, out0, o0_bytes) ||
+        ds4_gpu_tensor_prefixes_overlap(x, x_bytes, out1, o1_bytes) ||
+        ds4_gpu_tensor_prefixes_overlap(out0, o0_bytes, out1, o1_bytes)) {
+        return 0;
+    }
+
+    @autoreleasepool {
+        uint64_t inner0 = 0u;
+        uint64_t inner1 = 0u;
+        id<MTLBuffer> w0 = ds4_gpu_wrap_model_range(
+            model_map, model_size, weight0_offset, w0_bytes, &inner0);
+        id<MTLBuffer> w1 = ds4_gpu_wrap_model_range(
+            model_map, model_size, weight1_offset, w1_bytes, &inner1);
+        id<MTLBuffer> xb = ds4_gpu_tensor_buffer(x);
+        id<MTLBuffer> o0 = ds4_gpu_tensor_buffer(out0);
+        id<MTLBuffer> o1 = ds4_gpu_tensor_buffer(out1);
+        id<MTLComputePipelineState> mm_pipeline =
+            ds4_gpu_get_mul_mm_pipeline(
+                "kernel_mul_mm_q4_K_f16_rhs", false, false);
+        if (!w0 || !w1 || !xb || !o0 || !o1 ||
+            inner0 > NSUIntegerMax || inner1 > NSUIntegerMax ||
+            !g_cpy_contig_f32_f16_pipeline || !mm_pipeline ||
+            mm_pipeline.threadExecutionWidth != 32u ||
+            mm_pipeline.maxTotalThreadsPerThreadgroup < 128u ||
+            g_cpy_contig_f32_f16_pipeline.maxTotalThreadsPerThreadgroup == 0u ||
+            !ds4_gpu_ensure_scratch_buffer(
+                &g_q4_pair_rhs_f16_buffer,
+                &g_q4_pair_rhs_f16_bytes,
+                DS4_METAL_Q4_PAIR_RHS_BYTES,
+                "ds4_q4_pair_rhs_f16")) {
+            return 0;
+        }
+        id<MTLBuffer> rhs_f16 = g_q4_pair_rhs_f16_buffer;
+        if (!rhs_f16) return 0;
+
+        ds4_gpu_mul_mm_args args0 =
+            ds4_gpu_make_mm_args(in_dim, out0_dim, n_tok, row_bytes);
+        ds4_gpu_mul_mm_args args1 =
+            ds4_gpu_make_mm_args(in_dim, out1_dim, n_tok, row_bytes);
+        const uint64_t rhs_row_bytes = in_dim * sizeof(uint16_t);
+        const uint64_t rhs_bytes = rhs_row_bytes * n_tok;
+        args0.nb10 = args1.nb10 = sizeof(uint16_t);
+        args0.nb11 = args1.nb11 = rhs_row_bytes;
+        args0.nb12 = args0.nb13 = rhs_bytes;
+        args1.nb12 = args1.nb13 = rhs_bytes;
+
+        int owned = 0;
+        id<MTLCommandBuffer> cb = ds4_gpu_command_buffer(&owned);
+        if (!cb) return 0;
+
+        /* All fallible resource/PSO work above happens before this first
+         * dispatch.  The copy touches only scratch, so the graph can still use
+         * its normal output fallback if encoder creation fails. */
+        if (!ds4_gpu_encode_cpy_f32_f16_1d(
+                cb, xb, ds4_gpu_tensor_offset(x), rhs_f16, 0u,
+                (uint32_t)(in_dim * n_tok))) {
+            return 0;
+        }
+        if (!owned) ds4_gpu_close_batch_encoder();
+
+        /* Ending the copy encoder establishes an explicit producer/consumer
+         * boundary.  Both output dispatches are then encoded without another
+         * fallible host operation between them. */
+        id<MTLComputeCommandEncoder> enc = ds4_gpu_compute_encoder(cb);
+        if (!enc) return 0;
+        [enc setComputePipelineState:mm_pipeline];
+        [enc setThreadgroupMemoryLength:6144u atIndex:0];
+
+        [enc setBytes:&args0 length:sizeof(args0) atIndex:0];
+        [enc setBuffer:w0 offset:(NSUInteger)inner0 atIndex:1];
+        [enc setBuffer:rhs_f16 offset:0 atIndex:2];
+        [enc setBuffer:o0 offset:ds4_gpu_tensor_offset(out0) atIndex:3];
+        [enc dispatchThreadgroups:MTLSizeMake(
+                (NSUInteger)n_tok / 32u,
+                (NSUInteger)out0_dim / 64u,
+                1u)
+             threadsPerThreadgroup:MTLSizeMake(128u, 1u, 1u)];
+
+        [enc setBytes:&args1 length:sizeof(args1) atIndex:0];
+        [enc setBuffer:w1 offset:(NSUInteger)inner1 atIndex:1];
+        [enc setBuffer:o1 offset:ds4_gpu_tensor_offset(out1) atIndex:3];
+        [enc dispatchThreadgroups:MTLSizeMake(
+                (NSUInteger)n_tok / 32u,
+                (NSUInteger)out1_dim / 64u,
+                1u)
+             threadsPerThreadgroup:MTLSizeMake(128u, 1u, 1u)];
+        ds4_gpu_end_compute_encoder(cb, enc);
+
+        return ds4_gpu_finish_command_buffer(
+            cb, owned, "paired Q4_K prefill F16 RHS") ? 1 : -1;
+    }
+}
+
+int ds4_gpu_matmul_q4_K_pair_tensor(
+        ds4_gpu_tensor *out0, ds4_gpu_tensor *out1,
+        const void *model_map, uint64_t model_size,
+        uint64_t weight0_offset, uint64_t weight1_offset,
+        uint64_t in_dim, uint64_t out0_dim, uint64_t out1_dim,
+        const ds4_gpu_tensor *x, uint64_t n_tok) {
+    const int pair_enable =
+        ds4_gpu_env_bool("DS4_METAL_ENABLE_Q4_PREFILL_PAIR_F16_RHS");
+    const int pair_disable =
+        ds4_gpu_env_bool("DS4_METAL_DISABLE_Q4_PREFILL_PAIR_F16_RHS");
+    const int pair_require =
+        ds4_gpu_env_bool("DS4_METAL_REQUIRE_Q4_PREFILL_PAIR_F16_RHS");
+    const bool prefill_pair_call = n_tok >= 32u;
+    const bool global_pair_disabled =
+        getenv("DS4_METAL_DISABLE_Q4_DENSE_PAIR") != NULL;
+    const bool pair_requested =
+        !global_pair_disabled && pair_disable != 1 &&
+        (pair_enable == 1 || pair_require == 1);
+
+    if (!g_initialized && !ds4_gpu_init()) {
+        return pair_require == 1 && prefill_pair_call ? -1 : 0;
+    }
+    if (pair_requested) {
+        const int candidate = ds4_gpu_try_q4_K_prefill_pair_f16_rhs(
+            out0, out1, model_map, model_size,
+            weight0_offset, weight1_offset,
+            in_dim, out0_dim, out1_dim, x, n_tok);
+        if (candidate > 0) return 1;
+        if (candidate < 0) return -1;
+    }
+    if (pair_require == 1 && prefill_pair_call) {
+        fprintf(stderr,
+                "ds4: Metal Q4 prefill pair F16-RHS required but unavailable "
+                "(disabled=%d global_pair_disabled=%d pre_m5=%d "
+                "in=%llu out0=%llu out1=%llu tokens=%llu)\n",
+                pair_disable == 1,
+                global_pair_disabled,
+                ds4_gpu_device_is_pre_m5_apple_silicon(),
+                (unsigned long long)in_dim,
+                (unsigned long long)out0_dim,
+                (unsigned long long)out1_dim,
+                (unsigned long long)n_tok);
+        return -1;
+    }
+    if (!out0 || !out1 || !model_map || !x || n_tok == 0 || n_tok > 8u ||
+        in_dim == 0 || (in_dim % 256u) != 0 ||
+        in_dim > UINT32_MAX || out0_dim == 0 || out1_dim == 0 ||
+        out0_dim > UINT32_MAX || out1_dim > UINT32_MAX ||
+        global_pair_disabled) {
+        return 0;
+    }
+    @autoreleasepool {
+        const uint64_t row_bytes = (in_dim / 256u) * 144u;
+        const uint64_t w0_bytes = out0_dim * row_bytes;
+        const uint64_t w1_bytes = out1_dim * row_bytes;
+        if (weight0_offset > model_size || w0_bytes > model_size - weight0_offset ||
+            weight1_offset > model_size || w1_bytes > model_size - weight1_offset ||
+            ds4_gpu_tensor_bytes(x) < n_tok * in_dim * sizeof(float) ||
+            ds4_gpu_tensor_bytes(out0) < n_tok * out0_dim * sizeof(float) ||
+            ds4_gpu_tensor_bytes(out1) < n_tok * out1_dim * sizeof(float)) {
+            return 0;
+        }
+        uint64_t inner0 = 0, inner1 = 0;
+        id<MTLBuffer> w0 = ds4_gpu_wrap_model_range(
+            model_map, model_size, weight0_offset, w0_bytes, &inner0);
+        id<MTLBuffer> w1 = ds4_gpu_wrap_model_range(
+            model_map, model_size, weight1_offset, w1_bytes, &inner1);
+        id<MTLBuffer> xb = ds4_gpu_tensor_buffer(x);
+        id<MTLBuffer> o0 = ds4_gpu_tensor_buffer(out0);
+        id<MTLBuffer> o1 = ds4_gpu_tensor_buffer(out1);
+        if (!w0 || !w1 || !xb || !o0 || !o1) return 0;
+
+        const int16_t nsg = 2;
+        id<MTLComputePipelineState> pipeline = ds4_gpu_get_mul_mv_ext_pipeline(
+            "kernel_mul_mv_q4_K_dense_pair_f32", nsg, 8);
+        if (!pipeline) return 0;
+        ds4_gpu_q8_0_matvec_args args0 = {
+            .ne00=(int32_t)in_dim, .ne01=(int32_t)out0_dim, .ne02=1,
+            .nb00=1, .nb01=row_bytes, .nb02=row_bytes*out0_dim, .nb03=row_bytes*out0_dim,
+            .ne10=(int32_t)in_dim, .ne11=(int32_t)n_tok, .ne12=1,
+            .nb10=sizeof(float), .nb11=in_dim*sizeof(float),
+            .nb12=in_dim*n_tok*sizeof(float), .nb13=in_dim*n_tok*sizeof(float),
+            .ne0=(int32_t)out0_dim, .ne1=(int32_t)n_tok, .nr0=2, .r2=1, .r3=1,
+        };
+        ds4_gpu_q8_0_matvec_args args1 = args0;
+        args1.ne01 = args1.ne0 = (int32_t)out1_dim;
+        args1.nb02 = args1.nb03 = row_bytes * out1_dim;
+
+        int owned = 0;
+        id<MTLCommandBuffer> cb = ds4_gpu_command_buffer(&owned);
+        if (!cb) return 0;
+        id<MTLComputeCommandEncoder> enc = ds4_gpu_compute_encoder(cb);
+        if (!enc) return 0;
+        [enc setComputePipelineState:pipeline];
+        [enc setBytes:&args0 length:sizeof(args0) atIndex:0];
+        [enc setBytes:&args1 length:sizeof(args1) atIndex:1];
+        [enc setBuffer:w0 offset:(NSUInteger)inner0 atIndex:2];
+        [enc setBuffer:w1 offset:(NSUInteger)inner1 atIndex:3];
+        [enc setBuffer:xb offset:ds4_gpu_tensor_offset(x) atIndex:4];
+        [enc setBuffer:o0 offset:ds4_gpu_tensor_offset(out0) atIndex:5];
+        [enc setBuffer:o1 offset:ds4_gpu_tensor_offset(out1) atIndex:6];
+        [enc setThreadgroupMemoryLength:32 atIndex:0];
+        const uint64_t max_out = out0_dim > out1_dim ? out0_dim : out1_dim;
+        [enc dispatchThreadgroups:MTLSizeMake((max_out + 3u) / 4u, n_tok, 1)
+             threadsPerThreadgroup:MTLSizeMake(32, nsg, 1)];
+        ds4_gpu_end_compute_encoder(cb, enc);
+        return ds4_gpu_finish_command_buffer(
+            cb, owned, "paired Q4_K matvec") ? 1 : -1;
+    }
+}
+
+int ds4_gpu_q4_K_pair_quad_compressor_store_tensor(
+        ds4_gpu_tensor       *qr,
+        ds4_gpu_tensor       *kv_raw,
+        ds4_gpu_tensor       *out0_kv,
+        ds4_gpu_tensor       *out0_score,
+        ds4_gpu_tensor       *out1_kv,
+        ds4_gpu_tensor       *out1_score,
+        ds4_gpu_tensor       *state0_kv,
+        ds4_gpu_tensor       *state0_score,
+        ds4_gpu_tensor       *state1_kv,
+        ds4_gpu_tensor       *state1_score,
+        const void           *model_map,
+        uint64_t              model_size,
+        uint64_t              q_a_offset,
+        uint64_t              kv_offset,
+        uint64_t              weight0_kv_offset,
+        uint64_t              weight0_score_offset,
+        uint64_t              weight1_kv_offset,
+        uint64_t              weight1_score_offset,
+        uint64_t              ape0_offset,
+        uint32_t              ape0_type,
+        uint64_t              ape1_offset,
+        uint32_t              ape1_type,
+        uint32_t              in_dim,
+        uint32_t              q_rank,
+        uint32_t              kv_dim,
+        uint32_t              width0,
+        uint32_t              width1,
+        const ds4_gpu_tensor *x,
+        uint32_t              ratio,
+        uint32_t              pos) {
+    if (!g_initialized && !ds4_gpu_init()) return -1;
+    if (getenv("DS4_METAL_DISABLE_Q4_QKV_COMPRESSOR_FUSE") != NULL) return 0;
+    if (!qr || !kv_raw || !out0_kv || !out0_score ||
+        !state0_kv || !state0_score || !model_map || !x ||
+        in_dim == 0u || (in_dim % 256u) != 0u ||
+        q_rank == 0u || kv_dim == 0u || (q_rank & 1u) != 0u ||
+        (kv_dim & 1u) != 0u || width0 == 0u ||
+        (width0 & 1u) != 0u || (width1 & 1u) != 0u ||
+        (ratio != 4u && ratio != 128u) ||
+        (ape0_type != 0u && ape0_type != 1u) ||
+        (ape1_type != 0u && ape1_type != 1u)) {
+        return 0;
+    }
+    if (width1 != 0u &&
+        (!out1_kv || !out1_score || !state1_kv || !state1_score)) {
+        return -1;
+    }
+    @autoreleasepool {
+        const uint64_t q4_row_bytes = ((uint64_t)in_dim / 256u) * 144u;
+        const uint64_t f16_row_bytes = (uint64_t)in_dim * sizeof(uint16_t);
+        if ((uint64_t)q_rank > UINT64_MAX / q4_row_bytes ||
+            (uint64_t)kv_dim > UINT64_MAX / q4_row_bytes ||
+            (uint64_t)width0 > UINT64_MAX / f16_row_bytes ||
+            (width1 != 0u &&
+             (uint64_t)width1 > UINT64_MAX / f16_row_bytes)) {
+            return -1;
+        }
+        const uint64_t q_a_bytes = (uint64_t)q_rank * q4_row_bytes;
+        const uint64_t kv_bytes = (uint64_t)kv_dim * q4_row_bytes;
+        const uint64_t weight0_bytes = (uint64_t)width0 * f16_row_bytes;
+        const uint64_t weight1_bytes = (uint64_t)width1 * f16_row_bytes;
+        const uint64_t state_rows = ratio == 4u ? 2u * ratio : ratio;
+        const uint64_t ape0_elem = ape0_type == 1u ? sizeof(uint16_t) : sizeof(float);
+        const uint64_t ape1_elem = ape1_type == 1u ? sizeof(uint16_t) : sizeof(float);
+        const uint64_t ape0_bytes = (uint64_t)width0 * ratio * ape0_elem;
+        const uint64_t ape1_bytes = (uint64_t)width1 * ratio * ape1_elem;
+        if (q_a_offset > model_size || q_a_bytes > model_size - q_a_offset ||
+            kv_offset > model_size || kv_bytes > model_size - kv_offset ||
+            weight0_kv_offset > model_size ||
+            weight0_bytes > model_size - weight0_kv_offset ||
+            weight0_score_offset > model_size ||
+            weight0_bytes > model_size - weight0_score_offset ||
+            ape0_offset > model_size || ape0_bytes > model_size - ape0_offset ||
+            (width1 != 0u &&
+             (weight1_kv_offset > model_size ||
+              weight1_bytes > model_size - weight1_kv_offset ||
+              weight1_score_offset > model_size ||
+              weight1_bytes > model_size - weight1_score_offset ||
+              ape1_offset > model_size ||
+              ape1_bytes > model_size - ape1_offset))) {
+            return -1;
+        }
+
+        const uint64_t state0_bytes = state_rows * width0 * sizeof(float);
+        const uint64_t state1_bytes = state_rows * width1 * sizeof(float);
+        if (ds4_gpu_tensor_bytes(qr) < (uint64_t)q_rank * sizeof(float) ||
+            ds4_gpu_tensor_bytes(kv_raw) < (uint64_t)kv_dim * sizeof(float) ||
+            ds4_gpu_tensor_bytes(out0_kv) < (uint64_t)width0 * sizeof(float) ||
+            ds4_gpu_tensor_bytes(out0_score) < (uint64_t)width0 * sizeof(float) ||
+            ds4_gpu_tensor_bytes(state0_kv) < state0_bytes ||
+            ds4_gpu_tensor_bytes(state0_score) < state0_bytes ||
+            ds4_gpu_tensor_bytes(x) < (uint64_t)in_dim * sizeof(float) ||
+            (width1 != 0u &&
+             (ds4_gpu_tensor_bytes(out1_kv) < (uint64_t)width1 * sizeof(float) ||
+              ds4_gpu_tensor_bytes(out1_score) < (uint64_t)width1 * sizeof(float) ||
+              ds4_gpu_tensor_bytes(state1_kv) < state1_bytes ||
+              ds4_gpu_tensor_bytes(state1_score) < state1_bytes))) {
+            return -1;
+        }
+
+        uint64_t q_a_inner = 0, kv_inner = 0;
+        uint64_t w0kv_inner = 0, w0sc_inner = 0;
+        uint64_t w1kv_inner = 0, w1sc_inner = 0;
+        uint64_t ape0_inner = 0, ape1_inner = 0;
+        id<MTLBuffer> qw0buf = ds4_gpu_wrap_model_range(
+            model_map, model_size, q_a_offset, q_a_bytes, &q_a_inner);
+        id<MTLBuffer> qw1buf = ds4_gpu_wrap_model_range(
+            model_map, model_size, kv_offset, kv_bytes, &kv_inner);
+        id<MTLBuffer> w0kvbuf = ds4_gpu_wrap_model_range(
+            model_map, model_size, weight0_kv_offset, weight0_bytes, &w0kv_inner);
+        id<MTLBuffer> w0scbuf = ds4_gpu_wrap_model_range(
+            model_map, model_size, weight0_score_offset, weight0_bytes, &w0sc_inner);
+        id<MTLBuffer> ape0buf = ds4_gpu_wrap_model_range(
+            model_map, model_size, ape0_offset, ape0_bytes, &ape0_inner);
+        id<MTLBuffer> w1kvbuf = width1 != 0u
+            ? ds4_gpu_wrap_model_range(model_map, model_size,
+                                       weight1_kv_offset, weight1_bytes,
+                                       &w1kv_inner)
+            : w0kvbuf;
+        id<MTLBuffer> w1scbuf = width1 != 0u
+            ? ds4_gpu_wrap_model_range(model_map, model_size,
+                                       weight1_score_offset, weight1_bytes,
+                                       &w1sc_inner)
+            : w0scbuf;
+        id<MTLBuffer> ape1buf = width1 != 0u
+            ? ds4_gpu_wrap_model_range(model_map, model_size,
+                                       ape1_offset, ape1_bytes, &ape1_inner)
+            : ape0buf;
+        if (width1 == 0u) {
+            w1kv_inner = w0kv_inner;
+            w1sc_inner = w0sc_inner;
+            ape1_inner = ape0_inner;
+            out1_kv = out0_kv;
+            out1_score = out0_score;
+            state1_kv = state0_kv;
+            state1_score = state0_score;
+        }
+        if (!qw0buf || !qw1buf || !w0kvbuf || !w0scbuf ||
+            !w1kvbuf || !w1scbuf || !ape0buf || !ape1buf) return -1;
+
+        id<MTLBuffer> xbuf = ds4_gpu_tensor_buffer(x);
+        id<MTLBuffer> qrbuf = ds4_gpu_tensor_buffer(qr);
+        id<MTLBuffer> kvbuf = ds4_gpu_tensor_buffer(kv_raw);
+        id<MTLBuffer> out0kvbuf = ds4_gpu_tensor_buffer(out0_kv);
+        id<MTLBuffer> out0scbuf = ds4_gpu_tensor_buffer(out0_score);
+        id<MTLBuffer> out1kvbuf = ds4_gpu_tensor_buffer(out1_kv);
+        id<MTLBuffer> out1scbuf = ds4_gpu_tensor_buffer(out1_score);
+        id<MTLBuffer> state0kvbuf = ds4_gpu_tensor_buffer(state0_kv);
+        id<MTLBuffer> state0scbuf = ds4_gpu_tensor_buffer(state0_score);
+        id<MTLBuffer> state1kvbuf = ds4_gpu_tensor_buffer(state1_kv);
+        id<MTLBuffer> state1scbuf = ds4_gpu_tensor_buffer(state1_score);
+        if (!xbuf || !qrbuf || !kvbuf || !out0kvbuf || !out0scbuf ||
+            !out1kvbuf || !out1scbuf || !state0kvbuf || !state0scbuf ||
+            !state1kvbuf || !state1scbuf) return -1;
+
+        ds4_gpu_q8_0_matvec_args args0 = {
+            .ne00 = (int32_t)in_dim, .ne01 = (int32_t)q_rank, .ne02 = 1,
+            .nb00 = 1, .nb01 = q4_row_bytes,
+            .nb02 = q4_row_bytes * q_rank, .nb03 = q4_row_bytes * q_rank,
+            .ne10 = (int32_t)in_dim, .ne11 = 1, .ne12 = 1,
+            .nb10 = sizeof(float), .nb11 = (uint64_t)in_dim * sizeof(float),
+            .nb12 = (uint64_t)in_dim * sizeof(float),
+            .nb13 = (uint64_t)in_dim * sizeof(float),
+            .ne0 = (int32_t)q_rank, .ne1 = 1, .nr0 = 2, .r2 = 1, .r3 = 1,
+        };
+        ds4_gpu_q8_0_matvec_args args1 = args0;
+        args1.ne01 = args1.ne0 = (int32_t)kv_dim;
+        args1.nb02 = args1.nb03 = q4_row_bytes * kv_dim;
+        ds4_gpu_f16_matvec_args cargs =
+            ds4_gpu_make_f16_mv_args(in_dim, width0);
+        cargs.nr0 = 2;
+        ds4_gpu_dsv4_compressor_store_one_args store0_args = {
+            .width = width0, .ratio = ratio, .pos = pos,
+            .ape_type = ape0_type,
+        };
+        ds4_gpu_dsv4_compressor_store_one_args store1_args = {
+            .width = width1, .ratio = ratio, .pos = pos,
+            .ape_type = ape1_type,
+        };
+        const uint32_t max_out = q_rank > kv_dim ? q_rank : kv_dim;
+        const uint32_t pair_tgs = (max_out + 15u) / 16u;
+        id<MTLComputePipelineState> pipeline = ds4_gpu_get_mul_mv_pipeline(
+            "kernel_dsv4_q4_K_qkv_pair_quad_compressor_store", 8);
+        if (!pipeline) return 0;
+
+        int owned = 0;
+        id<MTLCommandBuffer> cb = ds4_gpu_command_buffer(&owned);
+        if (!cb) return -1;
+        id<MTLComputeCommandEncoder> enc = ds4_gpu_compute_encoder(cb);
+        [enc setComputePipelineState:pipeline];
+        [enc setBytes:&args0 length:sizeof(args0) atIndex:0];
+        [enc setBytes:&args1 length:sizeof(args1) atIndex:1];
+        [enc setBytes:&cargs length:sizeof(cargs) atIndex:2];
+        [enc setBytes:&store0_args length:sizeof(store0_args) atIndex:3];
+        [enc setBytes:&store1_args length:sizeof(store1_args) atIndex:4];
+        [enc setBytes:&pair_tgs length:sizeof(pair_tgs) atIndex:5];
+        [enc setBuffer:qw0buf offset:(NSUInteger)q_a_inner atIndex:6];
+        [enc setBuffer:qw1buf offset:(NSUInteger)kv_inner atIndex:7];
+        [enc setBuffer:w0kvbuf offset:(NSUInteger)w0kv_inner atIndex:8];
+        [enc setBuffer:w0scbuf offset:(NSUInteger)w0sc_inner atIndex:9];
+        [enc setBuffer:w1kvbuf offset:(NSUInteger)w1kv_inner atIndex:10];
+        [enc setBuffer:w1scbuf offset:(NSUInteger)w1sc_inner atIndex:11];
+        [enc setBuffer:xbuf offset:ds4_gpu_tensor_offset(x) atIndex:12];
+        [enc setBuffer:qrbuf offset:ds4_gpu_tensor_offset(qr) atIndex:13];
+        [enc setBuffer:kvbuf offset:ds4_gpu_tensor_offset(kv_raw) atIndex:14];
+        [enc setBuffer:out0kvbuf offset:ds4_gpu_tensor_offset(out0_kv) atIndex:15];
+        [enc setBuffer:out0scbuf offset:ds4_gpu_tensor_offset(out0_score) atIndex:16];
+        [enc setBuffer:out1kvbuf offset:ds4_gpu_tensor_offset(out1_kv) atIndex:17];
+        [enc setBuffer:out1scbuf offset:ds4_gpu_tensor_offset(out1_score) atIndex:18];
+        [enc setBuffer:ape0buf offset:(NSUInteger)ape0_inner atIndex:19];
+        [enc setBuffer:ape1buf offset:(NSUInteger)ape1_inner atIndex:20];
+        [enc setBuffer:state0kvbuf offset:ds4_gpu_tensor_offset(state0_kv) atIndex:21];
+        [enc setBuffer:state0scbuf offset:ds4_gpu_tensor_offset(state0_score) atIndex:22];
+        [enc setBuffer:state1kvbuf offset:ds4_gpu_tensor_offset(state1_kv) atIndex:23];
+        [enc setBuffer:state1scbuf offset:ds4_gpu_tensor_offset(state1_score) atIndex:24];
+        [enc setThreadgroupMemoryLength:32u * 2u * sizeof(float) atIndex:0];
+        [enc dispatchThreadgroups:MTLSizeMake(
+                (NSUInteger)pair_tgs +
+                    ((NSUInteger)width0 + 1u) / 2u +
+                    ((NSUInteger)width1 + 1u) / 2u,
+                1, 1)
+             threadsPerThreadgroup:MTLSizeMake(32, 8, 1)];
+        ds4_gpu_end_compute_encoder(cb, enc);
+
+        if (!ds4_gpu_finish_command_buffer(
+                cb, owned, "Q4_K QKV pair + compressor store")) {
+            return -1;
+        }
+    }
+
+    return 1;
+}
+
 int ds4_gpu_matmul_q8_0_f16_out_tensor(
         ds4_gpu_tensor       *out_h,
         const void             *model_map,
@@ -20233,18 +24550,39 @@ static int ds4_gpu_shared_gate_up_swiglu_q8_0_impl(
             return 0;
         }
 
+        const bool exact_decode_views =
+            ds4_gpu_support_q8_decode_exact_views_enabled(model_map,
+                                                          model_size);
         uint64_t gate_inner = 0;
         uint64_t up_inner = 0;
-        id<MTLBuffer> gate_wbuf = ds4_gpu_wrap_model_range(model_map,
-                                                           model_size,
-                                                           gate_offset,
-                                                           weight_bytes,
-                                                           &gate_inner);
-        id<MTLBuffer> up_wbuf = ds4_gpu_wrap_model_range(model_map,
-                                                         model_size,
-                                                         up_offset,
-                                                         weight_bytes,
-                                                         &up_inner);
+        id<MTLBuffer> gate_wbuf = nil;
+        id<MTLBuffer> up_wbuf = nil;
+        if (exact_decode_views) {
+            gate_wbuf = ds4_gpu_wrap_model_exact_range(model_map,
+                                                       model_size,
+                                                       gate_offset,
+                                                       weight_bytes,
+                                                       &gate_inner);
+            up_wbuf = ds4_gpu_wrap_model_exact_range(model_map,
+                                                     model_size,
+                                                     up_offset,
+                                                     weight_bytes,
+                                                     &up_inner);
+        }
+        if (!gate_wbuf || !up_wbuf) {
+            gate_inner = 0;
+            up_inner = 0;
+            gate_wbuf = ds4_gpu_wrap_model_range(model_map,
+                                                 model_size,
+                                                 gate_offset,
+                                                 weight_bytes,
+                                                 &gate_inner);
+            up_wbuf = ds4_gpu_wrap_model_range(model_map,
+                                               model_size,
+                                               up_offset,
+                                                 weight_bytes,
+                                                 &up_inner);
+        }
         if (!gate_wbuf || !up_wbuf) return 0;
 
         ds4_gpu_q8_0_matvec_args args = ds4_gpu_make_q8_0_mv_args(in_dim, out_dim);
@@ -20958,7 +25296,7 @@ int ds4_gpu_matmul_f16_tensor(
                 [enc setBuffer:wbuf offset:(NSUInteger)inner_offset atIndex:1];
                 [enc setBuffer:xbuf offset:ds4_gpu_tensor_offset(x) atIndex:2];
                 [enc setBuffer:outbuf offset:ds4_gpu_tensor_offset(out) atIndex:3];
-                [enc setThreadgroupMemoryLength:2u * 64u * 32u * sizeof(uint16_t) atIndex:0];
+                [enc setThreadgroupMemoryLength:DS4_METAL_MPP_DIRECT_RHS_SMEM atIndex:0];
                 [enc dispatchThreadgroups:MTLSizeMake((NSUInteger)(n_tok / nax_tile_n),
                                                       (NSUInteger)out_dim / 64u,
                                                       1)
@@ -21221,6 +25559,15 @@ int ds4_gpu_matmul_f16_pair_compressor_store_tensor(
     return 1;
 }
 
+int ds4_gpu_f16_quad_compressor_store_auto_available(void) {
+    if (!g_initialized && !ds4_gpu_init()) return 0;
+    return strncmp(g_metal_device_name, "Apple M", 7) == 0 &&
+           g_metal_device_name[7] >= '1' &&
+           g_metal_device_name[7] <= '5' &&
+           (g_metal_device_name[8] == '\0' ||
+            g_metal_device_name[8] == ' ');
+}
+
 /* Quad variant of the paired compressor projection: the attention compressor
  * and indexer compressor pairs share the input activation and F16 matvec
  * shape, so one dispatch covers all four matrices.  Bit-exact by
@@ -21533,51 +25880,95 @@ int ds4_gpu_qkv_pair_quad_compressor_store_tensor(
         const ds4_gpu_tensor *x,
         uint32_t              ratio,
         uint32_t              pos) {
-    if (!g_initialized && !ds4_gpu_init()) return 0;
-    if (!qr || !kv_raw || !out0_kv || !out0_score || !out1_kv || !out1_score ||
-        !state0_kv || !state0_score || !state1_kv || !state1_score ||
-        !model_map || !x || ratio == 0u || (in_dim & 31u) != 0 ||
-        (width0 & 1u) != 0 || (width1 & 1u) != 0) {
+    if (!g_initialized && !ds4_gpu_init()) return -1;
+    if (!qr || !kv_raw || !out0_kv || !out0_score ||
+        !state0_kv || !state0_score || !model_map || !x ||
+        in_dim == 0u || (in_dim & 31u) != 0u ||
+        q_rank == 0u || kv_dim == 0u || width0 == 0u ||
+        q_rank > INT32_MAX || kv_dim > INT32_MAX ||
+        width0 > INT32_MAX || width1 > INT32_MAX ||
+        (width0 & 1u) != 0u || (width1 & 1u) != 0u ||
+        (ratio != 4u && ratio != 128u) ||
+        (ratio == 4u && width1 == 0u) ||
+        (ratio == 128u && width1 != 0u) ||
+        (ape0_type != 0u && ape0_type != 1u) ||
+        (ape1_type != 0u && ape1_type != 1u)) {
+        return 0;
+    }
+    if (width1 != 0u &&
+        (!out1_kv || !out1_score || !state1_kv || !state1_score)) {
+        return -1;
+    }
+    /* The compound kernel embeds the canonical NSG=4 Q8 pair and NR0=2 F16
+     * reduction trees.  Respect diagnostic schedule overrides by selecting
+     * the separate dispatches instead of silently changing their arithmetic. */
+    if (ds4_gpu_make_q8_0_mv_dispatch().nsg != 4 ||
+        ds4_gpu_use_compressor_pair_nr4()) {
         return 0;
     }
 
     @autoreleasepool {
         const uint64_t q8_row_bytes = (in_dim / 32u) * 34u;
         const uint64_t f16_row_bytes = (uint64_t)in_dim * sizeof(uint16_t);
+        if ((uint64_t)q_rank > UINT64_MAX / q8_row_bytes ||
+            (uint64_t)kv_dim > UINT64_MAX / q8_row_bytes ||
+            (uint64_t)width0 > UINT64_MAX / f16_row_bytes ||
+            (width1 != 0u &&
+             (uint64_t)width1 > UINT64_MAX / f16_row_bytes)) {
+            return -1;
+        }
         const uint64_t q_a_bytes = (uint64_t)q_rank * q8_row_bytes;
         const uint64_t kv_bytes = (uint64_t)kv_dim * q8_row_bytes;
         const uint64_t weight0_bytes = (uint64_t)width0 * f16_row_bytes;
         const uint64_t weight1_bytes = (uint64_t)width1 * f16_row_bytes;
         const uint64_t state_rows = ratio == 4u ? 2u * ratio : ratio;
-        const uint64_t ape0_bytes = (uint64_t)width0 * ratio *
-                                    (ape0_type == 1u ? 2u : 4u);
-        const uint64_t ape1_bytes = (uint64_t)width1 * ratio *
-                                    (ape1_type == 1u ? 2u : 4u);
+        const uint64_t ape0_elem =
+            ape0_type == 1u ? sizeof(uint16_t) : sizeof(float);
+        const uint64_t ape1_elem =
+            ape1_type == 1u ? sizeof(uint16_t) : sizeof(float);
+        if ((uint64_t)width0 > UINT64_MAX / ratio / ape0_elem ||
+            (width1 != 0u &&
+             (uint64_t)width1 > UINT64_MAX / ratio / ape1_elem)) {
+            return -1;
+        }
+        const uint64_t ape0_bytes =
+            (uint64_t)width0 * ratio * ape0_elem;
+        const uint64_t ape1_bytes =
+            (uint64_t)width1 * ratio * ape1_elem;
         if (q_a_offset > model_size || q_a_bytes > model_size - q_a_offset ||
             kv_offset > model_size || kv_bytes > model_size - kv_offset ||
             weight0_kv_offset > model_size ||
             weight0_bytes > model_size - weight0_kv_offset ||
             weight0_score_offset > model_size ||
             weight0_bytes > model_size - weight0_score_offset ||
-            weight1_kv_offset > model_size ||
-            weight1_bytes > model_size - weight1_kv_offset ||
-            weight1_score_offset > model_size ||
-            weight1_bytes > model_size - weight1_score_offset ||
             ape0_offset > model_size || ape0_bytes > model_size - ape0_offset ||
-            ape1_offset > model_size || ape1_bytes > model_size - ape1_offset) {
+            (width1 != 0u &&
+             (weight1_kv_offset > model_size ||
+              weight1_bytes > model_size - weight1_kv_offset ||
+              weight1_score_offset > model_size ||
+              weight1_bytes > model_size - weight1_score_offset ||
+              ape1_offset > model_size ||
+              ape1_bytes > model_size - ape1_offset))) {
             return -1;
         }
+        const uint64_t state0_bytes =
+            state_rows * (uint64_t)width0 * sizeof(float);
+        const uint64_t state1_bytes =
+            state_rows * (uint64_t)width1 * sizeof(float);
         if (ds4_gpu_tensor_bytes(qr) < (uint64_t)q_rank * sizeof(float) ||
             ds4_gpu_tensor_bytes(kv_raw) < (uint64_t)kv_dim * sizeof(float) ||
             ds4_gpu_tensor_bytes(out0_kv) < (uint64_t)width0 * sizeof(float) ||
             ds4_gpu_tensor_bytes(out0_score) < (uint64_t)width0 * sizeof(float) ||
-            ds4_gpu_tensor_bytes(out1_kv) < (uint64_t)width1 * sizeof(float) ||
-            ds4_gpu_tensor_bytes(out1_score) < (uint64_t)width1 * sizeof(float) ||
             ds4_gpu_tensor_bytes(x) < (uint64_t)in_dim * sizeof(float) ||
-            ds4_gpu_tensor_bytes(state0_kv) < state_rows * width0 * sizeof(float) ||
-            ds4_gpu_tensor_bytes(state0_score) < state_rows * width0 * sizeof(float) ||
-            ds4_gpu_tensor_bytes(state1_kv) < state_rows * width1 * sizeof(float) ||
-            ds4_gpu_tensor_bytes(state1_score) < state_rows * width1 * sizeof(float)) {
+            ds4_gpu_tensor_bytes(state0_kv) < state0_bytes ||
+            ds4_gpu_tensor_bytes(state0_score) < state0_bytes ||
+            (width1 != 0u &&
+             (ds4_gpu_tensor_bytes(out1_kv) <
+                  (uint64_t)width1 * sizeof(float) ||
+              ds4_gpu_tensor_bytes(out1_score) <
+                  (uint64_t)width1 * sizeof(float) ||
+              ds4_gpu_tensor_bytes(state1_kv) < state1_bytes ||
+              ds4_gpu_tensor_bytes(state1_score) < state1_bytes))) {
             return -1;
         }
 
@@ -21605,6 +25996,10 @@ int ds4_gpu_qkv_pair_quad_compressor_store_tensor(
             w1kv_inner = w0kv_inner;
             w1sc_inner = w0sc_inner;
             ape1_inner = ape0_inner;
+            out1_kv = out0_kv;
+            out1_score = out0_score;
+            state1_kv = state0_kv;
+            state1_score = state0_score;
         }
         if (!qw0buf || !qw1buf || !w0kvbuf || !w0scbuf ||
             !w1kvbuf || !w1scbuf || !ape0buf || !ape1buf) return -1;
@@ -22741,6 +27136,14 @@ int ds4_gpu_head_rms_norm_rope_tail_tensor(
         if (!cb) return 0;
 
         id<MTLComputeCommandEncoder> enc = ds4_gpu_compute_encoder(cb);
+        if (!enc) {
+            if (owned) {
+                (void)ds4_gpu_finish_command_buffer(
+                    cb, owned,
+                    "fused head norm/RoPE encoder allocation");
+            }
+            return 0;
+        }
         [enc setComputePipelineState:pipeline];
         [enc setBytes:&args length:sizeof(args) atIndex:0];
         [enc setBuffer:xbuf offset:ds4_gpu_tensor_offset(x) atIndex:1];
@@ -22761,12 +27164,1589 @@ int ds4_gpu_head_rms_norm_rope_tail_tensor(
     return 1;
 }
 
+static int ds4_gpu_q4_attn_q_b_f16_cache_key_equal(
+        const ds4_gpu_q4_attn_q_b_f16_cache_entry *entry,
+        const void                                  *model_map,
+        uint64_t                                     model_size,
+        uint64_t                                     weight_offset,
+        uint64_t                                     weight_bytes,
+        uint64_t                                     in_dim,
+        uint64_t                                     out_dim) {
+    return entry->valid &&
+           entry->model_map == model_map &&
+           entry->model_size == model_size &&
+           entry->weight_offset == weight_offset &&
+           entry->weight_bytes == weight_bytes &&
+           entry->in_dim == in_dim &&
+           entry->out_dim == out_dim;
+}
+
+static int ds4_gpu_q4_attn_q_b_f16_cache_lookup_impl(
+        const void                   *model_map,
+        uint64_t                      model_size,
+        uint64_t                      weight_offset,
+        uint64_t                      weight_bytes,
+        uint64_t                      in_dim,
+        uint64_t                      out_dim,
+        id<MTLBuffer> __strong       *buffer,
+        int                            count_stats) {
+    int found = 0;
+    pthread_mutex_lock(&g_q4_attn_q_b_f16_cache_mu);
+    if (count_stats) g_q4_attn_q_b_f16_cache_lookups++;
+    for (uint32_t i = 0;
+         i < DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES;
+         i++) {
+        ds4_gpu_q4_attn_q_b_f16_cache_entry *entry =
+            &g_q4_attn_q_b_f16_cache[i];
+        if (!ds4_gpu_q4_attn_q_b_f16_cache_key_equal(
+                entry, model_map, model_size, weight_offset, weight_bytes,
+                in_dim, out_dim)) {
+            continue;
+        }
+        *buffer = entry->buffer;
+        if (count_stats) g_q4_attn_q_b_f16_cache_hits++;
+        found = 1;
+        break;
+    }
+    if (count_stats && !found) g_q4_attn_q_b_f16_cache_misses++;
+    pthread_mutex_unlock(&g_q4_attn_q_b_f16_cache_mu);
+    return found;
+}
+
+static int ds4_gpu_q4_attn_q_b_f16_cache_lookup(
+        const void                   *model_map,
+        uint64_t                      model_size,
+        uint64_t                      weight_offset,
+        uint64_t                      weight_bytes,
+        uint64_t                      in_dim,
+        uint64_t                      out_dim,
+        id<MTLBuffer> __strong       *buffer) {
+    return ds4_gpu_q4_attn_q_b_f16_cache_lookup_impl(
+        model_map, model_size, weight_offset, weight_bytes, in_dim,
+        out_dim, buffer, 1);
+}
+
+static int ds4_gpu_q4_attn_q_b_f16_cache_peek(
+        const void                   *model_map,
+        uint64_t                      model_size,
+        uint64_t                      weight_offset,
+        uint64_t                      weight_bytes,
+        uint64_t                      in_dim,
+        uint64_t                      out_dim,
+        id<MTLBuffer> __strong       *buffer) {
+    return ds4_gpu_q4_attn_q_b_f16_cache_lookup_impl(
+        model_map, model_size, weight_offset, weight_bytes, in_dim,
+        out_dim, buffer, 0);
+}
+
+static int ds4_gpu_q4_attn_q_b_f16_cache_insert(
+        const void          *model_map,
+        uint64_t             model_size,
+        uint64_t             weight_offset,
+        uint64_t             weight_bytes,
+        uint64_t             in_dim,
+        uint64_t             out_dim,
+        uint64_t             f16_bytes,
+        uint64_t             budget_bytes,
+        id<MTLBuffer>        buffer) {
+    int inserted = 0;
+    pthread_mutex_lock(&g_q4_attn_q_b_f16_cache_mu);
+
+    /* Normal graph encoding is serialized.  Recheck nevertheless so an
+     * unusual concurrent caller cannot publish a duplicate key. */
+    for (uint32_t i = 0;
+         i < DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES;
+         i++) {
+        if (ds4_gpu_q4_attn_q_b_f16_cache_key_equal(
+                &g_q4_attn_q_b_f16_cache[i], model_map, model_size,
+                weight_offset, weight_bytes, in_dim, out_dim)) {
+            pthread_mutex_unlock(&g_q4_attn_q_b_f16_cache_mu);
+            return -1;
+        }
+    }
+
+    if (f16_bytes <= budget_bytes &&
+        g_q4_attn_q_b_f16_cache_bytes <= budget_bytes - f16_bytes) {
+        for (uint32_t i = 0;
+             i < DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES;
+             i++) {
+            ds4_gpu_q4_attn_q_b_f16_cache_entry *entry =
+                &g_q4_attn_q_b_f16_cache[i];
+            if (entry->valid) continue;
+            entry->model_map = model_map;
+            entry->model_size = model_size;
+            entry->weight_offset = weight_offset;
+            entry->weight_bytes = weight_bytes;
+            entry->in_dim = in_dim;
+            entry->out_dim = out_dim;
+            entry->f16_bytes = f16_bytes;
+            entry->buffer = buffer;
+            entry->valid = 1;
+            const int first_entry =
+                g_q4_attn_q_b_f16_cache_entries == 0u;
+            g_q4_attn_q_b_f16_cache_bytes += f16_bytes;
+            g_q4_attn_q_b_f16_cache_entries++;
+            g_q4_attn_q_b_f16_cache_builds++;
+            inserted = first_entry ? 2 : 1;
+            break;
+        }
+    }
+    pthread_mutex_unlock(&g_q4_attn_q_b_f16_cache_mu);
+    return inserted;
+}
+
+static int ds4_gpu_q4_attn_q_b_f16_cache_insert_batch(
+        const void *model_map,
+        uint64_t model_size,
+        const ds4_gpu_q4_attn_q_b_f16_sidecar_desc *descs,
+        const uint32_t *miss_indices,
+        id<MTLBuffer> __strong *buffers,
+        uint32_t miss_count,
+        uint64_t budget_bytes) {
+    uint32_t free_slots[DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES];
+    uint64_t new_bytes = 0;
+    uint32_t free_count = 0;
+    int inserted = 0;
+
+    if (!descs || !miss_indices || !buffers || miss_count == 0u ||
+        miss_count > DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES) {
+        return 0;
+    }
+
+    pthread_mutex_lock(&g_q4_attn_q_b_f16_cache_mu);
+    for (uint32_t i = 0;
+         i < DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES;
+         i++) {
+        if (!g_q4_attn_q_b_f16_cache[i].valid) {
+            free_slots[free_count++] = i;
+        }
+    }
+
+    for (uint32_t mi = 0; mi < miss_count; mi++) {
+        const ds4_gpu_q4_attn_q_b_f16_sidecar_desc *desc =
+            &descs[miss_indices[mi]];
+        const uint64_t f16_bytes =
+            desc->in_dim * desc->out_dim * sizeof(uint16_t);
+        if (!buffers[mi] || UINT64_MAX - new_bytes < f16_bytes) {
+            goto done;
+        }
+        new_bytes += f16_bytes;
+
+        for (uint32_t i = 0;
+             i < DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES;
+             i++) {
+            if (ds4_gpu_q4_attn_q_b_f16_cache_key_equal(
+                    &g_q4_attn_q_b_f16_cache[i], model_map, model_size,
+                    desc->weight_offset, desc->weight_bytes,
+                    desc->in_dim, desc->out_dim)) {
+                goto done;
+            }
+        }
+        for (uint32_t mj = 0; mj < mi; mj++) {
+            const ds4_gpu_q4_attn_q_b_f16_sidecar_desc *prior =
+                &descs[miss_indices[mj]];
+            if (prior->weight_offset == desc->weight_offset &&
+                prior->weight_bytes == desc->weight_bytes &&
+                prior->in_dim == desc->in_dim &&
+                prior->out_dim == desc->out_dim) {
+                goto done;
+            }
+        }
+    }
+
+    if (free_count < miss_count || new_bytes > budget_bytes ||
+        g_q4_attn_q_b_f16_cache_bytes > budget_bytes - new_bytes) {
+        goto done;
+    }
+
+    const int first_entry = g_q4_attn_q_b_f16_cache_entries == 0u;
+    for (uint32_t mi = 0; mi < miss_count; mi++) {
+        const ds4_gpu_q4_attn_q_b_f16_sidecar_desc *desc =
+            &descs[miss_indices[mi]];
+        ds4_gpu_q4_attn_q_b_f16_cache_entry *entry =
+            &g_q4_attn_q_b_f16_cache[free_slots[mi]];
+        entry->model_map = model_map;
+        entry->model_size = model_size;
+        entry->weight_offset = desc->weight_offset;
+        entry->weight_bytes = desc->weight_bytes;
+        entry->in_dim = desc->in_dim;
+        entry->out_dim = desc->out_dim;
+        entry->f16_bytes =
+            desc->in_dim * desc->out_dim * sizeof(uint16_t);
+        entry->buffer = buffers[mi];
+        entry->valid = 1;
+    }
+    g_q4_attn_q_b_f16_cache_bytes += new_bytes;
+    g_q4_attn_q_b_f16_cache_entries += miss_count;
+    g_q4_attn_q_b_f16_cache_builds += miss_count;
+    inserted = first_entry ? 2 : 1;
+
+done:
+    pthread_mutex_unlock(&g_q4_attn_q_b_f16_cache_mu);
+    return inserted;
+}
+
+static int ds4_gpu_q4_attn_q_b_f16_cache_has_room(
+        uint64_t f16_bytes,
+        uint64_t budget_bytes) {
+    int has_room = 0;
+    pthread_mutex_lock(&g_q4_attn_q_b_f16_cache_mu);
+    has_room =
+        g_q4_attn_q_b_f16_cache_entries <
+            DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES &&
+        f16_bytes <= budget_bytes &&
+        g_q4_attn_q_b_f16_cache_bytes <= budget_bytes - f16_bytes;
+    pthread_mutex_unlock(&g_q4_attn_q_b_f16_cache_mu);
+    return has_room;
+}
+
+static int ds4_gpu_q4_attn_q_b_f16_cache_has_room_batch(
+        uint32_t new_entries,
+        uint64_t new_bytes,
+        uint64_t budget_bytes) {
+    int has_room = 0;
+    pthread_mutex_lock(&g_q4_attn_q_b_f16_cache_mu);
+    has_room =
+        new_entries <=
+            DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES -
+                g_q4_attn_q_b_f16_cache_entries &&
+        new_bytes <= budget_bytes &&
+        g_q4_attn_q_b_f16_cache_bytes <= budget_bytes - new_bytes;
+    pthread_mutex_unlock(&g_q4_attn_q_b_f16_cache_mu);
+    return has_room;
+}
+
+static int ds4_gpu_q4_attn_q_b_f16_builds_suppressed(void) {
+    int suppressed = 0;
+    pthread_mutex_lock(&g_q4_attn_q_b_f16_cache_mu);
+    suppressed =
+        g_q4_attn_q_b_f16_build_circuit_state !=
+        DS4_Q4_ATTN_Q_B_F16_CIRCUIT_CLOSED;
+    pthread_mutex_unlock(&g_q4_attn_q_b_f16_cache_mu);
+    return suppressed;
+}
+
+/* SSD streaming normally rejects this resident expansion: 43 production
+ * attn_q_b matrices occupy about 2.69 GiB as F16. Keep the hybrid mode
+ * explicit until hardware A/B data shows that its compute saving outweighs
+ * the smaller unified-memory budget left for streamed experts. The regular
+ * cache DISABLE flag is checked separately and always wins. */
+static int ds4_gpu_q4_attn_q_b_f16_streaming_allowed(void) {
+    if (!g_ssd_streaming_mode) return 1;
+    return
+        __atomic_load_n(&g_q4_attn_q_b_f16_ssd_admission_blocked,
+                        __ATOMIC_ACQUIRE) == 0 &&
+        ds4_gpu_env_bool(
+            "DS4_METAL_ENABLE_Q4_ATTN_Q_B_F16_CACHE_WITH_SSD_STREAMING") == 1;
+}
+
+/* currentAllocatedSize already contains streaming expert buffers that have
+ * been materialized by an earlier session.  Credit those bytes against only
+ * the dynamic-cache portion of the host's future working-set reserve; graph,
+ * full-layer and prefill headroom must remain reserved.  Slabs are counted by
+ * their full allocation rather than by occupied slots because Metal accounts
+ * the entire MTLBuffer in currentAllocatedSize.  Deliberately do not credit
+ * standalone fallback buffers: unlike slabs, ordinary cache rotation may
+ * release them between this snapshot and admission.  Double-counting that
+ * uncommon path is conservative; stale credit would not be. */
+static uint64_t ds4_gpu_stream_expert_cache_materialized_bytes(void) {
+    return __atomic_load_n(
+        &g_stream_expert_cache_slab_allocated_bytes,
+        __ATOMIC_ACQUIRE);
+}
+
+static uint64_t ds4_gpu_q4_attn_q_b_f16_effective_working_set_reserve(
+        uint64_t planned_reserve,
+        uint64_t *materialized_credit) {
+    if (materialized_credit) *materialized_credit = 0;
+    if (!g_ssd_streaming_mode || planned_reserve == 0u) {
+        return planned_reserve;
+    }
+
+    const uint32_t budget =
+        ds4_gpu_stream_expert_cache_configured_budget();
+    if (budget == 0u || g_stream_expert_cache_expert_bytes == 0u) {
+        return planned_reserve;
+    }
+    const uint64_t dynamic_target =
+        budget > UINT64_MAX / g_stream_expert_cache_expert_bytes
+            ? UINT64_MAX
+            : (uint64_t)budget * g_stream_expert_cache_expert_bytes;
+    uint64_t credit = ds4_gpu_stream_expert_cache_materialized_bytes();
+    if (credit > dynamic_target) credit = dynamic_target;
+    if (credit > planned_reserve) credit = planned_reserve;
+    if (materialized_credit) *materialized_credit = credit;
+    return planned_reserve - credit;
+}
+
+/* The initial SSD model map intentionally covers token/static spans only.
+ * Wrap each dense q_b source as a short-lived exact no-copy view while its
+ * sidecar is built; retaining those ~18 MiB views in the general exact-view
+ * cache would pin another ~0.77 GiB of model mappings for no steady-state
+ * benefit. The caller keeps the returned owned buffer alive through command
+ * completion. */
+static id<MTLBuffer> ds4_gpu_q4_attn_q_b_f16_wrap_source(
+        const void *model_map,
+        uint64_t    model_size,
+        uint64_t    weight_offset,
+        uint64_t    weight_bytes,
+        uint64_t   *inner_offset) {
+    if (g_ssd_streaming_mode) {
+        return ds4_gpu_wrap_model_exact_range_owned(
+            model_map, model_size, weight_offset, weight_bytes,
+            inner_offset);
+    }
+    return ds4_gpu_wrap_model_range(
+        model_map, model_size, weight_offset, weight_bytes,
+        inner_offset);
+}
+
+static uint64_t ds4_gpu_q4_attn_q_b_f16_exact_source_bytes(
+        uint64_t model_size,
+        uint64_t weight_offset,
+        uint64_t weight_bytes) {
+    if (model_size == 0u || weight_offset > model_size ||
+        weight_bytes > model_size - weight_offset) {
+        return UINT64_MAX;
+    }
+    const uint64_t page = (uint64_t)getpagesize();
+    const uint64_t page_offset = weight_offset & ~(page - 1u);
+    const uint64_t leading = weight_offset - page_offset;
+    if (weight_bytes > UINT64_MAX - leading ||
+        leading + weight_bytes > UINT64_MAX - (page - 1u)) {
+        return UINT64_MAX;
+    }
+    uint64_t view_bytes = round_up_u64(leading + weight_bytes, page);
+    const uint64_t remaining = model_size - page_offset;
+    if (view_bytes > remaining) view_bytes = remaining;
+    return view_bytes;
+}
+
+static void ds4_gpu_q4_attn_q_b_f16_suppress_builds(
+        ds4_gpu_q4_attn_q_b_f16_circuit_state state) {
+    if (state == DS4_Q4_ATTN_Q_B_F16_CIRCUIT_CLOSED) return;
+    pthread_mutex_lock(&g_q4_attn_q_b_f16_cache_mu);
+    if (state == DS4_Q4_ATTN_Q_B_F16_CIRCUIT_HARD ||
+        g_q4_attn_q_b_f16_build_circuit_state ==
+            DS4_Q4_ATTN_Q_B_F16_CIRCUIT_CLOSED) {
+        g_q4_attn_q_b_f16_build_circuit_state = state;
+    }
+    pthread_mutex_unlock(&g_q4_attn_q_b_f16_cache_mu);
+}
+
+/* A working-set rejection describes the allocations present at one instant,
+ * unlike a cache-budget or build failure.  A later session can have a smaller
+ * graph or follow cleanup of a failed strict session, so let its explicit
+ * prewarm re-evaluate pressure.  Lazy per-layer misses stay suppressed to
+ * avoid retry storms.  Called only while the cold-build mutex is held. */
+static int ds4_gpu_q4_attn_q_b_f16_prewarm_hard_suppressed(void) {
+    int hard = 0;
+    pthread_mutex_lock(&g_q4_attn_q_b_f16_cache_mu);
+    hard = g_q4_attn_q_b_f16_build_circuit_state ==
+        DS4_Q4_ATTN_Q_B_F16_CIRCUIT_HARD;
+    if (g_q4_attn_q_b_f16_build_circuit_state ==
+        DS4_Q4_ATTN_Q_B_F16_CIRCUIT_PRESSURE) {
+        g_q4_attn_q_b_f16_build_circuit_state =
+            DS4_Q4_ATTN_Q_B_F16_CIRCUIT_CLOSED;
+    }
+    pthread_mutex_unlock(&g_q4_attn_q_b_f16_cache_mu);
+    return hard;
+}
+
+static int ds4_gpu_q4_attn_q_b_f16_working_set_policy(
+        uint64_t recommended,
+        uint64_t allocated,
+        uint64_t additional_bytes) {
+    if (recommended == 0u) return 0;
+
+    const uint64_t reserve = recommended / 8u;
+    const uint64_t limit = recommended - reserve;
+    return allocated <= limit && additional_bytes <= limit - allocated;
+}
+
+int ds4_gpu_test_q4_attn_q_b_f16_working_set_policy(
+        uint64_t recommended,
+        uint64_t allocated,
+        uint64_t additional) {
+    return ds4_gpu_q4_attn_q_b_f16_working_set_policy(
+        recommended, allocated, additional);
+}
+
+/* Keep one eighth of Metal's recommended working set free for graph
+ * activations, KV growth, command-buffer transients, and the OS.  Metal
+ * accounts both mmap-backed no-copy model buffers and private sidecars in
+ * currentAllocatedSize, so this conservative gate covers the combined Metal
+ * resource footprint rather than just the logical sidecar budget. */
+static int ds4_gpu_q4_attn_q_b_f16_working_set_has_room(
+        uint64_t additional_bytes) {
+    if (!g_device || additional_bytes > NSUIntegerMax) return 0;
+    return ds4_gpu_q4_attn_q_b_f16_working_set_policy(
+        (uint64_t)[g_device recommendedMaxWorkingSetSize],
+        (uint64_t)[g_device currentAllocatedSize],
+        additional_bytes);
+}
+
+int ds4_gpu_make_room_for_q4_attn_q_b_f16_session(void) {
+    pthread_mutex_lock(&g_q4_attn_q_b_f16_cache_mu);
+    const uint32_t entries = g_q4_attn_q_b_f16_cache_entries;
+    const uint64_t bytes = g_q4_attn_q_b_f16_cache_bytes;
+    pthread_mutex_unlock(&g_q4_attn_q_b_f16_cache_mu);
+    if (entries == 0u) return 1;
+
+    fprintf(stderr,
+            "ds4: evicting %.2f GiB of resident Q4 attn_q_b F16 "
+            "sidecars before allocating another live session\n",
+            ds4_gpu_gib(bytes));
+    return ds4_gpu_release_q4_attn_q_b_f16_sidecars();
+}
+
+static int ds4_gpu_q4_attn_q_b_f16_fallback(int required,
+                                              int rejected,
+                                              int build_failure) {
+    pthread_mutex_lock(&g_q4_attn_q_b_f16_cache_mu);
+    g_q4_attn_q_b_f16_cache_fallbacks++;
+    if (rejected) g_q4_attn_q_b_f16_cache_rejects++;
+    if (build_failure) g_q4_attn_q_b_f16_cache_build_failures++;
+    pthread_mutex_unlock(&g_q4_attn_q_b_f16_cache_mu);
+    return required ? -1 : 0;
+}
+
+/* Called only while the cold-build mutex is held.  Existing READY entries
+ * remain usable; only future misses are suppressed.  Hard failures persist
+ * until cache reset, while transient working-set pressure may be re-evaluated
+ * by the next explicit session prewarm. */
+static int ds4_gpu_q4_attn_q_b_f16_abort_cold_build(
+        int required,
+        int rejected,
+        int build_failure,
+        ds4_gpu_q4_attn_q_b_f16_circuit_state circuit_state) {
+    ds4_gpu_q4_attn_q_b_f16_suppress_builds(circuit_state);
+    pthread_mutex_unlock(&g_q4_attn_q_b_f16_build_mu);
+    return ds4_gpu_q4_attn_q_b_f16_fallback(
+        required, rejected, build_failure);
+}
+
+static int ds4_gpu_q4_attn_q_b_f16_abort_prewarm(
+        int required,
+        int rejected,
+        int build_failure,
+        ds4_gpu_q4_attn_q_b_f16_circuit_state circuit_state) {
+    ds4_gpu_q4_attn_q_b_f16_suppress_builds(circuit_state);
+    pthread_mutex_unlock(&g_q4_attn_q_b_f16_build_mu);
+    pthread_mutex_lock(&g_q4_attn_q_b_f16_cache_mu);
+    if (rejected) g_q4_attn_q_b_f16_cache_rejects++;
+    if (build_failure) g_q4_attn_q_b_f16_cache_build_failures++;
+    pthread_mutex_unlock(&g_q4_attn_q_b_f16_cache_mu);
+    return required ? -1 : 0;
+}
+
+static int ds4_gpu_q4_qb_transient_f16_scratch_ensure_locked(
+        int      stream,
+        uint64_t required_bytes);
+
+/* Resolve the complete transient path and reserve its one stream-local
+ * expansion buffer before prefill timing starts.  Per-layer Q4->F16 work is
+ * intentionally not performed here: unlike PSO creation and allocation, that
+ * dequantization is a real recurring cost of the optimized projection.  SSD
+ * streaming additionally reserves room for one short-lived exact Q4 source
+ * view; the view itself is created only when its layer is encoded. */
+static int ds4_gpu_prepare_q4_attn_q_b_transient_f16(
+        const void *model_map,
+        uint64_t model_size,
+        const ds4_gpu_q4_attn_q_b_f16_sidecar_desc *descs,
+        uint32_t count,
+        uint32_t max_prefill_rows,
+        uint64_t working_set_reserve_bytes,
+        uint64_t *prepared_bytes) {
+    const double prepare_t0 = ds4_gpu_now_ms();
+    const uint64_t min_tokens = ds4_gpu_env_u64(
+        "DS4_METAL_Q4_ATTN_Q_B_TRANSIENT_F16_MIN_TOKENS",
+        4096u, 32u, UINT32_MAX);
+    if (max_prefill_rows < min_tokens ||
+        g_quality_mode ||
+        g_batch_encoder_concurrent ||
+        !ds4_gpu_device_is_pre_m5_apple_silicon() ||
+        ds4_gpu_mpp_available() ||
+        g_ds4_stream < 0 || g_ds4_stream >= DS4_GPU_MAX_STREAMS) {
+        return 0;
+    }
+
+    uint64_t f16_bytes = 0u;
+    uint64_t max_source_view_bytes = 0u;
+    for (uint32_t i = 0; i < count; i++) {
+        const ds4_gpu_q4_attn_q_b_f16_sidecar_desc *desc = &descs[i];
+        if (desc->weight_type != DS4_METAL_TENSOR_Q4_K ||
+            desc->in_dim != 1024u || desc->out_dim != 32768u) {
+            return 0;
+        }
+        const uint64_t row_bytes =
+            (desc->in_dim / 256u) * 144u;
+        const uint64_t expected_weight_bytes =
+            desc->out_dim * row_bytes;
+        const uint64_t desc_f16_bytes =
+            desc->in_dim * desc->out_dim * sizeof(uint16_t);
+        if (desc->weight_bytes != expected_weight_bytes ||
+            desc->weight_offset > model_size ||
+            desc->weight_bytes > model_size - desc->weight_offset ||
+            (f16_bytes != 0u && f16_bytes != desc_f16_bytes)) {
+            return 0;
+        }
+        f16_bytes = desc_f16_bytes;
+        if (g_ssd_streaming_mode) {
+            const uint64_t source_view_bytes =
+                ds4_gpu_q4_attn_q_b_f16_exact_source_bytes(
+                    model_size, desc->weight_offset, desc->weight_bytes);
+            if (source_view_bytes > max_source_view_bytes) {
+                max_source_view_bytes = source_view_bytes;
+            }
+        }
+    }
+    if (!model_map || f16_bytes == 0u || f16_bytes > NSUIntegerMax) {
+        return 0;
+    }
+
+    id<MTLComputePipelineState> dequant_pipeline =
+        ds4_gpu_get_pipeline("kernel_dequantize_q4_K_f16");
+    id<MTLComputePipelineState> mm_aligned_pipeline =
+        ds4_gpu_get_mul_mm_pipeline(
+            "kernel_mul_mm_f16_f16_rhs", false, false);
+    id<MTLComputePipelineState> mm_boundary_pipeline =
+        ds4_gpu_get_mul_mm_pipeline(
+            "kernel_mul_mm_f16_f16_rhs", false, true);
+    const int use_contiguous_copy =
+        ds4_gpu_env_bool(
+            "DS4_METAL_DISABLE_CONTIG_F32_F16_COPY") <= 0;
+    id<MTLComputePipelineState> copy_pipeline = use_contiguous_copy
+        ? g_cpy_contig_f32_f16_pipeline
+        : g_cpy_f32_f16_pipeline;
+    if (!dequant_pipeline || !mm_aligned_pipeline ||
+        !mm_boundary_pipeline || !copy_pipeline ||
+        dequant_pipeline.maxTotalThreadsPerThreadgroup < 64u ||
+        mm_aligned_pipeline.threadExecutionWidth != 32u ||
+        mm_aligned_pipeline.maxTotalThreadsPerThreadgroup < 128u ||
+        mm_boundary_pipeline.threadExecutionWidth != 32u ||
+        mm_boundary_pipeline.maxTotalThreadsPerThreadgroup < 128u) {
+        return 0;
+    }
+
+    const int stream = g_ds4_stream;
+    const uint64_t effective_working_set_reserve_bytes =
+        ds4_gpu_q4_attn_q_b_f16_effective_working_set_reserve(
+            working_set_reserve_bytes, NULL);
+    bool allocated = false;
+    pthread_mutex_lock(&g_q4_qb_transient_f16_scratch_mu);
+    const bool needs_allocation =
+        !g_q4_qb_transient_f16_scratch[stream] ||
+        g_q4_qb_transient_f16_scratch_capacity[stream] <
+            (NSUInteger)f16_bytes;
+    uint64_t admission_bytes = needs_allocation ? f16_bytes : 0u;
+    const uint64_t source_view_bytes =
+        g_ssd_streaming_mode ? max_source_view_bytes : 0u;
+    if (UINT64_MAX - admission_bytes < source_view_bytes) {
+        admission_bytes = UINT64_MAX;
+    } else {
+        admission_bytes += source_view_bytes;
+    }
+    if (UINT64_MAX - admission_bytes <
+            effective_working_set_reserve_bytes) {
+        admission_bytes = UINT64_MAX;
+    } else {
+        admission_bytes += effective_working_set_reserve_bytes;
+    }
+    if (((needs_allocation || g_ssd_streaming_mode) &&
+         !ds4_gpu_q4_attn_q_b_f16_working_set_has_room(
+             admission_bytes)) ||
+        !ds4_gpu_q4_qb_transient_f16_scratch_ensure_locked(
+            stream, f16_bytes)) {
+        pthread_mutex_unlock(&g_q4_qb_transient_f16_scratch_mu);
+        return 0;
+    }
+    allocated = needs_allocation;
+    pthread_mutex_unlock(&g_q4_qb_transient_f16_scratch_mu);
+
+    if (allocated) {
+        if (prepared_bytes) *prepared_bytes = f16_bytes;
+        fprintf(stderr,
+                "ds4: Metal prepared %.2f MiB stream-local Q4 attn_q_b "
+                "transient F16 scratch and pipelines before prefill "
+                "in %.3f ms\n",
+                (double)f16_bytes / 1048576.0,
+                ds4_gpu_now_ms() - prepare_t0);
+    }
+    return 1;
+}
+
+int ds4_gpu_prepare_q4_attn_q_b_f16_sidecars(
+        const void *model_map,
+        uint64_t model_size,
+        const ds4_gpu_q4_attn_q_b_f16_sidecar_desc *descs,
+        uint32_t count,
+        uint32_t max_prefill_rows,
+        uint64_t working_set_reserve_bytes,
+        uint64_t *prepared_bytes) {
+    if (prepared_bytes) *prepared_bytes = 0;
+    if (!model_map || !descs || count == 0u ||
+        count > DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES ||
+        max_prefill_rows < 32u) {
+        return 0;
+    }
+    if (!g_initialized && !ds4_gpu_init()) return 0;
+
+    const uint64_t transient_effective_reserve_bytes =
+        ds4_gpu_q4_attn_q_b_f16_effective_working_set_reserve(
+            working_set_reserve_bytes, NULL);
+    bool ssd_transient_same_contract = false;
+    const bool ssd_transient_invalidated = g_ssd_streaming_mode &&
+        ds4_gpu_q4_qb_transient_f16_admission_begin(
+            g_ds4_stream, model_map, model_size,
+            transient_effective_reserve_bytes,
+            &ssd_transient_same_contract);
+
+    const int required =
+        ds4_gpu_env_bool("DS4_METAL_REQUIRE_Q4_ATTN_Q_B_F16_CACHE") == 1;
+    const int transient_disabled =
+        ds4_gpu_env_bool(
+            "DS4_METAL_DISABLE_Q4_ATTN_Q_B_TRANSIENT_F16") == 1;
+    const int ssd_sidecar_opt_in =
+        g_ssd_streaming_mode &&
+        ds4_gpu_env_bool(
+            "DS4_METAL_ENABLE_Q4_ATTN_Q_B_F16_CACHE_WITH_SSD_STREAMING") == 1;
+    /* The 64 MiB transient production path is the resident pre-M5 default.
+     * Do not prewarm 43 persistent layer sidecars (~2.69 GiB) unless the
+     * operator explicitly selects the legacy path, makes it mandatory, or
+     * opts into the established SSD-streaming hybrid. */
+    if (!required && !transient_disabled && !ssd_sidecar_opt_in) {
+        const int rc = ds4_gpu_prepare_q4_attn_q_b_transient_f16(
+            model_map, model_size, descs, count, max_prefill_rows,
+            working_set_reserve_bytes, prepared_bytes);
+        if (ssd_transient_invalidated &&
+            (rc != 1 || !ssd_transient_same_contract)) {
+            /* The stream slot is shared by sessions.  If an admitted slot is
+             * rejected or repurposed, invalidate every session that cached
+             * the old generation so it cannot skip its next preflight. */
+            ds4_gpu_q4_attn_q_b_f16_advance_generation();
+        }
+        if (rc == 1 && g_ssd_streaming_mode) {
+            ds4_gpu_q4_qb_transient_f16_admission_commit(
+                g_ds4_stream, model_map, model_size,
+                ds4_gpu_q4_attn_q_b_f16_cache_generation(),
+                transient_effective_reserve_bytes);
+        }
+        return rc;
+    }
+    if (ssd_transient_invalidated) {
+        ds4_gpu_q4_attn_q_b_f16_advance_generation();
+    }
+    const int disabled =
+        ds4_gpu_env_bool("DS4_METAL_DISABLE_Q4_ATTN_Q_B_F16_CACHE") == 1;
+    const int disable_f16_rhs =
+        ds4_gpu_env_bool("DS4_METAL_DISABLE_Q4_ATTN_Q_B_F16_RHS") == 1;
+    const uint64_t min_tokens = ds4_gpu_env_u64(
+        "DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MIN_TOKENS", 512u, 32u,
+        UINT32_MAX);
+
+    /* A session whose largest possible chunk is below the configured
+     * threshold can never be a cache candidate, even in strict mode. */
+    if (max_prefill_rows < min_tokens) return 0;
+    if (disabled ||
+        !ds4_gpu_q4_attn_q_b_f16_streaming_allowed() ||
+        g_quality_mode ||
+        !ds4_gpu_device_is_pre_m5_apple_silicon() ||
+        ds4_gpu_mpp_available()) {
+        return required ? -1 : 0;
+    }
+
+    uint64_t total_f16_bytes = 0;
+    for (uint32_t i = 0; i < count; i++) {
+        const ds4_gpu_q4_attn_q_b_f16_sidecar_desc *desc = &descs[i];
+        if (desc->weight_type != DS4_METAL_TENSOR_Q4_K ||
+            desc->in_dim != 1024u || desc->out_dim != 32768u ||
+            desc->in_dim > UINT64_MAX / desc->out_dim /
+                sizeof(uint16_t)) {
+            return required ? -1 : 0;
+        }
+        const uint64_t blocks_per_row = desc->in_dim / 256u;
+        const uint64_t expected_weight_bytes =
+            desc->out_dim * blocks_per_row * 144u;
+        const uint64_t f16_bytes =
+            desc->in_dim * desc->out_dim * sizeof(uint16_t);
+        if (desc->weight_bytes != expected_weight_bytes ||
+            desc->weight_offset > model_size ||
+            desc->weight_bytes > model_size - desc->weight_offset ||
+            UINT64_MAX - total_f16_bytes < f16_bytes) {
+            return required ? -1 : 0;
+        }
+        total_f16_bytes += f16_bytes;
+    }
+
+    const uint64_t budget_mib = ds4_gpu_env_u64(
+        "DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MB", 3072u, 1u, 65536u);
+    const uint64_t budget_bytes = budget_mib * 1024u * 1024u;
+    const double build_t0 = ds4_gpu_now_ms();
+
+    pthread_mutex_lock(&g_q4_attn_q_b_f16_build_mu);
+
+    uint32_t miss_indices[DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES];
+    uint32_t miss_count = 0;
+    uint64_t missing_bytes = 0;
+    uint64_t missing_source_bytes = 0;
+    for (uint32_t i = 0; i < count; i++) {
+        id<MTLBuffer> hit_buffer = nil;
+        const ds4_gpu_q4_attn_q_b_f16_sidecar_desc *desc = &descs[i];
+        if (ds4_gpu_q4_attn_q_b_f16_cache_peek(
+                model_map, model_size, desc->weight_offset,
+                desc->weight_bytes, desc->in_dim, desc->out_dim,
+                &hit_buffer)) {
+            continue;
+        }
+        miss_indices[miss_count++] = i;
+        missing_bytes +=
+            desc->in_dim * desc->out_dim * sizeof(uint16_t);
+        if (UINT64_MAX - missing_source_bytes < desc->weight_bytes) {
+            return ds4_gpu_q4_attn_q_b_f16_abort_prewarm(
+                required, 1, 0, DS4_Q4_ATTN_Q_B_F16_CIRCUIT_HARD);
+        }
+        missing_source_bytes += desc->weight_bytes;
+    }
+    if (miss_count == 0u) {
+        pthread_mutex_unlock(&g_q4_attn_q_b_f16_build_mu);
+        return 1;
+    }
+
+    if (ds4_gpu_q4_attn_q_b_f16_prewarm_hard_suppressed()) {
+        return ds4_gpu_q4_attn_q_b_f16_abort_prewarm(
+            required, 1, 0, DS4_Q4_ATTN_Q_B_F16_CIRCUIT_CLOSED);
+    }
+    if (!ds4_gpu_q4_attn_q_b_f16_cache_has_room_batch(
+            miss_count, missing_bytes, budget_bytes)) {
+        fprintf(stderr,
+                "ds4: Metal Q4 attn_q_b F16 prewarm skipped: "
+                "%u missing sidecars need %.2f GiB, cache budget is "
+                "%llu MiB\n",
+                miss_count,
+                (double)missing_bytes / 1073741824.0,
+                (unsigned long long)budget_mib);
+        return ds4_gpu_q4_attn_q_b_f16_abort_prewarm(
+            required, 1, 0, DS4_Q4_ATTN_Q_B_F16_CIRCUIT_HARD);
+    }
+    const uint64_t temporary_source_reserve_bytes =
+        g_ssd_streaming_mode ? missing_source_bytes : 0u;
+    uint64_t materialized_streaming_credit_bytes = 0;
+    const uint64_t effective_working_set_reserve_bytes =
+        ds4_gpu_q4_attn_q_b_f16_effective_working_set_reserve(
+            working_set_reserve_bytes,
+            &materialized_streaming_credit_bytes);
+    if (UINT64_MAX - effective_working_set_reserve_bytes <
+            temporary_source_reserve_bytes ||
+        UINT64_MAX - missing_bytes <
+            effective_working_set_reserve_bytes +
+                temporary_source_reserve_bytes ||
+        !ds4_gpu_q4_attn_q_b_f16_working_set_has_room(
+            missing_bytes + effective_working_set_reserve_bytes +
+                temporary_source_reserve_bytes)) {
+        const uint64_t allocated =
+            (uint64_t)[g_device currentAllocatedSize];
+        const uint64_t recommended =
+            (uint64_t)[g_device recommendedMaxWorkingSetSize];
+        fprintf(stderr,
+                "ds4: Metal Q4 attn_q_b F16 prewarm skipped for "
+                "working-set headroom: allocated %.2f GiB + sidecars "
+                "%.2f GiB + reserved working set %.2f GiB (planned %.2f, "
+                "%.2f already materialized) + temporary "
+                "SSD sources %.2f GiB, 7/8 safety limit %.2f GiB\n",
+                (double)allocated / 1073741824.0,
+                (double)missing_bytes / 1073741824.0,
+                (double)effective_working_set_reserve_bytes /
+                    1073741824.0,
+                (double)working_set_reserve_bytes / 1073741824.0,
+                (double)materialized_streaming_credit_bytes /
+                    1073741824.0,
+                (double)temporary_source_reserve_bytes / 1073741824.0,
+                (double)(recommended - recommended / 8u) /
+                    1073741824.0);
+        return ds4_gpu_q4_attn_q_b_f16_abort_prewarm(
+            required, 1, 0, DS4_Q4_ATTN_Q_B_F16_CIRCUIT_PRESSURE);
+    }
+
+    id<MTLComputePipelineState> dequant_pipeline =
+        ds4_gpu_get_pipeline("kernel_dequantize_q4_K_f16");
+    /* Resolve both consumers before publishing any READY entry.  Exact
+     * multiples of 32 and boundary batches use distinct function-constant
+     * specializations; compiling them here keeps first-prefill PSO cost out
+     * of the measured path and makes REQUIRE genuinely fail at preparation. */
+    id<MTLComputePipelineState> mm_aligned_pipeline =
+        ds4_gpu_get_mul_mm_pipeline(
+            "kernel_mul_mm_f16_f32", false, false);
+    id<MTLComputePipelineState> mm_boundary_pipeline =
+        ds4_gpu_get_mul_mm_pipeline(
+            "kernel_mul_mm_f16_f32", false, true);
+    /* The compact-RHS specialization is nested under the sidecar and must
+     * never make the established F16-weight/F32-RHS cache unavailable. */
+    if (!disable_f16_rhs && !g_ssd_streaming_mode) {
+        (void)ds4_gpu_get_mul_mm_pipeline(
+            "kernel_mul_mm_f16_f16_rhs", false, false);
+        (void)ds4_gpu_get_mul_mm_pipeline(
+            "kernel_mul_mm_f16_f16_rhs", false, true);
+    }
+    id<MTLCommandBuffer> build_cb =
+        [ds4_gpu_active_queue() commandBuffer];
+    if (!dequant_pipeline || !mm_aligned_pipeline ||
+        !mm_boundary_pipeline || !build_cb) {
+        return ds4_gpu_q4_attn_q_b_f16_abort_prewarm(
+            required, 0, 1, DS4_Q4_ATTN_Q_B_F16_CIRCUIT_HARD);
+    }
+    build_cb.label = @"ds4 Q4 attn_q_b F16 sidecar prewarm";
+
+    __strong id<MTLBuffer>
+        sidecars[DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES] = {nil};
+    __strong id<MTLBuffer>
+        weights[DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES] = {nil};
+    uint64_t weight_inner[DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES] = {0};
+
+    for (uint32_t mi = 0; mi < miss_count; mi++) {
+        const ds4_gpu_q4_attn_q_b_f16_sidecar_desc *desc =
+            &descs[miss_indices[mi]];
+        const uint64_t f16_bytes =
+            desc->in_dim * desc->out_dim * sizeof(uint16_t);
+        weights[mi] = ds4_gpu_q4_attn_q_b_f16_wrap_source(
+            model_map, model_size, desc->weight_offset,
+            desc->weight_bytes, &weight_inner[mi]);
+        sidecars[mi] = [g_device
+            newBufferWithLength:(NSUInteger)f16_bytes
+                        options:MTLResourceStorageModePrivate];
+        if (!weights[mi] || !sidecars[mi]) {
+            return ds4_gpu_q4_attn_q_b_f16_abort_prewarm(
+                required, 0, 1, DS4_Q4_ATTN_Q_B_F16_CIRCUIT_HARD);
+        }
+        sidecars[mi].label = [NSString stringWithFormat:
+            @"ds4_attn_q_b_q4_f16_cache_layer_%u", desc->layer];
+    }
+
+    id<MTLComputeCommandEncoder> enc = [build_cb computeCommandEncoder];
+    if (!enc) {
+        return ds4_gpu_q4_attn_q_b_f16_abort_prewarm(
+            required, 0, 1, DS4_Q4_ATTN_Q_B_F16_CIRCUIT_HARD);
+    }
+    [enc setComputePipelineState:dequant_pipeline];
+    for (uint32_t mi = 0; mi < miss_count; mi++) {
+        const ds4_gpu_q4_attn_q_b_f16_sidecar_desc *desc =
+            &descs[miss_indices[mi]];
+        const uint32_t chunks_per_row = (uint32_t)(desc->in_dim / 16u);
+        const uint32_t row_count = (uint32_t)desc->out_dim;
+        [enc setBuffer:weights[mi]
+                offset:(NSUInteger)weight_inner[mi]
+               atIndex:0];
+        [enc setBuffer:sidecars[mi] offset:0 atIndex:1];
+        [enc setBytes:&chunks_per_row
+               length:sizeof(chunks_per_row)
+              atIndex:2];
+        [enc setBytes:&row_count length:sizeof(row_count) atIndex:3];
+        [enc dispatchThreadgroups:MTLSizeMake(
+                ((NSUInteger)chunks_per_row + 63u) / 64u,
+                (NSUInteger)row_count,
+                1)
+             threadsPerThreadgroup:MTLSizeMake(64, 1, 1)];
+    }
+    [enc endEncoding];
+    [build_cb commit];
+    if (!ds4_gpu_wait_command_buffer(
+            build_cb, "Q4 attn_q_b F16 sidecar prewarm")) {
+        return ds4_gpu_q4_attn_q_b_f16_abort_prewarm(
+            required, 0, 1, DS4_Q4_ATTN_Q_B_F16_CIRCUIT_HARD);
+    }
+
+    const int insert_rc = ds4_gpu_q4_attn_q_b_f16_cache_insert_batch(
+        model_map, model_size, descs, miss_indices, sidecars,
+        miss_count, budget_bytes);
+    if (insert_rc <= 0) {
+        return ds4_gpu_q4_attn_q_b_f16_abort_prewarm(
+            required, insert_rc == 0, insert_rc < 0,
+            DS4_Q4_ATTN_Q_B_F16_CIRCUIT_HARD);
+    }
+    pthread_mutex_unlock(&g_q4_attn_q_b_f16_build_mu);
+
+    if (prepared_bytes) *prepared_bytes = missing_bytes;
+    const uint64_t allocated_after_prewarm =
+        (uint64_t)[g_device currentAllocatedSize];
+    const uint64_t recommended_working_set =
+        (uint64_t)[g_device recommendedMaxWorkingSetSize];
+    fprintf(stderr,
+            "ds4: Metal%s prewarmed %u resident Q4 attn_q_b F16 "
+            "sidecars (%.2f GiB) in %.3f ms; cache budget %llu MiB; "
+            "reserved working set %.2f GiB (planned %.2f, %.2f already "
+            "materialized); temporary SSD sources "
+            "%.2f GiB; Metal allocated %.2f / %.2f GiB recommended\n",
+            g_ssd_streaming_mode ? " SSD-streaming hybrid" : "",
+            miss_count,
+            (double)missing_bytes / 1073741824.0,
+            ds4_gpu_now_ms() - build_t0,
+            (unsigned long long)budget_mib,
+            (double)effective_working_set_reserve_bytes / 1073741824.0,
+            (double)working_set_reserve_bytes / 1073741824.0,
+            (double)materialized_streaming_credit_bytes / 1073741824.0,
+            (double)temporary_source_reserve_bytes / 1073741824.0,
+            (double)allocated_after_prewarm / 1073741824.0,
+            (double)recommended_working_set / 1073741824.0);
+    return 1;
+}
+
+static bool ds4_gpu_tensor_prefixes_overlap(
+        const ds4_gpu_tensor *a,
+        uint64_t              a_bytes,
+        const ds4_gpu_tensor *b,
+        uint64_t              b_bytes);
+static int ds4_gpu_encode_f16_rhs_mm(
+        id<MTLCommandBuffer>        cb,
+        id<MTLComputePipelineState> pipeline,
+        id<MTLBuffer>               weights,
+        NSUInteger                  weights_offset,
+        const ds4_gpu_tensor       *rhs_f16,
+        ds4_gpu_tensor             *out,
+        uint64_t                    in_dim,
+        uint64_t                    out_dim,
+        uint64_t                    n_tok,
+        uint64_t                    row_bytes,
+        bool                        bc_out);
+static int ds4_gpu_encode_q4_K_transient_f16(
+        id<MTLCommandBuffer>        cb,
+        id<MTLComputePipelineState> pipeline,
+        id<MTLBuffer>               q4_weights,
+        NSUInteger                  q4_weights_offset,
+        id<MTLBuffer>               f16_scratch,
+        uint64_t                    in_dim,
+        uint64_t                    out_dim);
+
+static int ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor_impl(
+        ds4_gpu_tensor       *out,
+        ds4_gpu_tensor       *q_half,
+        const void           *model_map,
+        uint64_t              model_size,
+        uint64_t              weight_offset,
+        uint32_t              weight_type,
+        uint64_t              in_dim,
+        uint64_t              out_dim,
+        const ds4_gpu_tensor *x,
+        uint32_t              n_tok,
+        uint32_t              n_head,
+        uint32_t              head_dim,
+        uint32_t              n_rot,
+        uint32_t              pos0,
+        uint32_t              n_ctx_orig,
+        bool                  inverse,
+        float                 freq_base,
+        float                 freq_scale,
+        float                 ext_factor,
+        float                 attn_factor,
+        float                 beta_fast,
+        float                 beta_slow,
+        float                 eps,
+        int                   apply_norm_rope) {
+    if (weight_type != DS4_METAL_TENSOR_Q4_K) return 0;
+    /* This path is prefill-only.  Keep decode and tiny batches off the
+     * environment parser, cache mutex, and pipeline lookup entirely. */
+    if (n_tok < 32u) return 0;
+    if (!g_initialized && !ds4_gpu_init()) return 0;
+
+    const int required =
+        ds4_gpu_env_bool("DS4_METAL_REQUIRE_Q4_ATTN_Q_B_F16_CACHE") == 1;
+    const int disabled =
+        ds4_gpu_env_bool("DS4_METAL_DISABLE_Q4_ATTN_Q_B_F16_CACHE") == 1;
+    bool use_f16_rhs =
+        q_half != NULL &&
+        !g_ssd_streaming_mode &&
+        !g_batch_encoder_concurrent &&
+        ds4_gpu_env_bool("DS4_METAL_DISABLE_Q4_ATTN_Q_B_F16_RHS") != 1;
+    const uint64_t min_tokens = ds4_gpu_env_u64(
+        "DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MIN_TOKENS", 512u, 32u,
+        UINT32_MAX);
+
+    /* REQUIRE applies only to actual cache candidates.  In particular, a
+     * short final prefill chunk below the configured threshold must retain
+     * the native Q4 fallback just like decode and tiny batches. */
+    if ((uint64_t)n_tok < min_tokens) return 0;
+
+    pthread_mutex_lock(&g_q4_attn_q_b_f16_cache_mu);
+    g_q4_attn_q_b_f16_cache_candidate_calls++;
+    pthread_mutex_unlock(&g_q4_attn_q_b_f16_cache_mu);
+
+    /* The initial production arm targets DeepSeek-V4 Flash q_b exactly.
+     * Larger variants need more than 10 GiB of sidecar storage and require a
+     * separate memory/performance admission study. */
+    if (disabled ||
+        !ds4_gpu_q4_attn_q_b_f16_streaming_allowed() ||
+        g_quality_mode ||
+        !ds4_gpu_device_is_pre_m5_apple_silicon() ||
+        ds4_gpu_mpp_available() ||
+        in_dim != 1024u || out_dim != 32768u ||
+        n_head == 0u || head_dim == 0u ||
+        out_dim != (uint64_t)n_head * head_dim ||
+        n_rot > head_dim || (n_rot & 1u) != 0u ||
+        n_tok > (uint32_t)INT32_MAX ||
+        pos0 > (uint32_t)INT32_MAX - n_tok) {
+        return ds4_gpu_q4_attn_q_b_f16_fallback(required, 1, 0);
+    }
+
+    if (!out || !x || !model_map ||
+        n_tok > UINT64_MAX / in_dim / sizeof(float) ||
+        n_tok > UINT64_MAX / out_dim / sizeof(float) ||
+        ds4_gpu_tensor_bytes(x) <
+            (uint64_t)n_tok * in_dim * sizeof(float) ||
+        ds4_gpu_tensor_bytes(out) <
+            (uint64_t)n_tok * out_dim * sizeof(float)) {
+        return ds4_gpu_q4_attn_q_b_f16_fallback(required, 1, 0);
+    }
+
+    const uint64_t x_f32_bytes =
+        (uint64_t)n_tok * in_dim * sizeof(float);
+    const uint64_t x_f16_bytes =
+        (uint64_t)n_tok * in_dim * sizeof(uint16_t);
+    const uint64_t out_f32_bytes =
+        (uint64_t)n_tok * out_dim * sizeof(float);
+    if (use_f16_rhs &&
+        ((uint64_t)n_tok * in_dim > UINT32_MAX ||
+         ds4_gpu_tensor_bytes(q_half) < x_f16_bytes ||
+         !ds4_gpu_tensor_buffer(q_half) ||
+         ds4_gpu_tensor_prefixes_overlap(
+             q_half, x_f16_bytes, x, x_f32_bytes) ||
+         ds4_gpu_tensor_prefixes_overlap(
+             q_half, x_f16_bytes, out, out_f32_bytes))) {
+        /* Compact F16 RHS staging is a nested optimization.  An unusable
+         * scratch view must not disable the already-valid resident
+         * F16-weight/F32-RHS sidecar path (or make REQUIRE reject it). */
+        use_f16_rhs = false;
+    }
+
+    const uint64_t blocks_per_row = in_dim / 256u;
+    const uint64_t row_bytes = blocks_per_row * 144u;
+    if (out_dim > UINT64_MAX / row_bytes) {
+        return ds4_gpu_q4_attn_q_b_f16_fallback(required, 1, 0);
+    }
+    const uint64_t weight_bytes = out_dim * row_bytes;
+    if (weight_offset > model_size ||
+        weight_bytes > model_size - weight_offset ||
+        out_dim > UINT64_MAX / in_dim / sizeof(uint16_t)) {
+        return ds4_gpu_q4_attn_q_b_f16_fallback(required, 1, 0);
+    }
+    const uint64_t f16_bytes = out_dim * in_dim * sizeof(uint16_t);
+    const uint64_t budget_mib = ds4_gpu_env_u64(
+        "DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MB", 3072u, 1u, 65536u);
+    const uint64_t budget_bytes = budget_mib * 1024u * 1024u;
+
+    @autoreleasepool {
+        id<MTLBuffer> f16_buffer = nil;
+        int hit = ds4_gpu_q4_attn_q_b_f16_cache_lookup(
+            model_map, model_size, weight_offset, weight_bytes, in_dim,
+            out_dim, &f16_buffer);
+
+        /* Resolve every resource used by the hot matmul before starting a
+         * cold build.  A failure can then take the native-Q4 fallback without
+         * abandoning an owned command buffer or publishing partial state. */
+        id<MTLBuffer> xbuf = ds4_gpu_tensor_buffer(x);
+        id<MTLBuffer> outbuf = ds4_gpu_tensor_buffer(out);
+        const bool mm_bc_out = (n_tok & 31u) != 0u;
+        id<MTLComputePipelineState> mm_f32_pipeline =
+            ds4_gpu_get_mul_mm_pipeline(
+                "kernel_mul_mm_f16_f32", false, mm_bc_out);
+        id<MTLComputePipelineState> mm_f16_pipeline = nil;
+        if (use_f16_rhs) {
+            mm_f16_pipeline = ds4_gpu_get_mul_mm_pipeline(
+                "kernel_mul_mm_f16_f16_rhs", false, mm_bc_out);
+            const int use_contiguous_copy =
+                ds4_gpu_env_bool(
+                    "DS4_METAL_DISABLE_CONTIG_F32_F16_COPY") <= 0;
+            id<MTLComputePipelineState> copy_pipeline =
+                use_contiguous_copy
+                    ? g_cpy_contig_f32_f16_pipeline
+                    : g_cpy_f32_f16_pipeline;
+            if (!mm_f16_pipeline || !copy_pipeline) {
+                use_f16_rhs = false;
+            }
+        }
+        if (!xbuf || !outbuf || !mm_f32_pipeline) {
+            return ds4_gpu_q4_attn_q_b_f16_fallback(required, 0, 1);
+        }
+
+        int build_lock_held = 0;
+        if (!hit) {
+            /* Serialize cold builders.  Recheck after taking the lock so
+             * another stream's completed build becomes a hit instead of a
+            * duplicate 64 MiB allocation and dequant dispatch. */
+            pthread_mutex_lock(&g_q4_attn_q_b_f16_build_mu);
+            build_lock_held = 1;
+            f16_buffer = nil;
+            hit = ds4_gpu_q4_attn_q_b_f16_cache_peek(
+                model_map, model_size, weight_offset, weight_bytes, in_dim,
+                out_dim, &f16_buffer);
+        }
+
+        if (!hit) {
+            if (ds4_gpu_q4_attn_q_b_f16_builds_suppressed()) {
+                return ds4_gpu_q4_attn_q_b_f16_abort_cold_build(
+                    required, 1, 0,
+                    DS4_Q4_ATTN_Q_B_F16_CIRCUIT_CLOSED);
+            }
+            if (!ds4_gpu_q4_attn_q_b_f16_cache_has_room(
+                    f16_bytes, budget_bytes)) {
+                return ds4_gpu_q4_attn_q_b_f16_abort_cold_build(
+                    required, 1, 0,
+                    DS4_Q4_ATTN_Q_B_F16_CIRCUIT_HARD);
+            }
+            const uint64_t temporary_source_bytes =
+                g_ssd_streaming_mode ? weight_bytes : 0u;
+            if (UINT64_MAX - f16_bytes < temporary_source_bytes ||
+                !ds4_gpu_q4_attn_q_b_f16_working_set_has_room(
+                    f16_bytes + temporary_source_bytes)) {
+                return ds4_gpu_q4_attn_q_b_f16_abort_cold_build(
+                    required, 1, 0,
+                    DS4_Q4_ATTN_Q_B_F16_CIRCUIT_PRESSURE);
+            }
+
+            id<MTLComputePipelineState> dequant_pipeline =
+                ds4_gpu_get_pipeline("kernel_dequantize_q4_K_f16");
+            if (!dequant_pipeline) {
+                return ds4_gpu_q4_attn_q_b_f16_abort_cold_build(
+                    required, 0, 1,
+                    DS4_Q4_ATTN_Q_B_F16_CIRCUIT_HARD);
+            }
+
+            uint64_t weight_inner = 0;
+            id<MTLBuffer> weight_buffer =
+                ds4_gpu_q4_attn_q_b_f16_wrap_source(
+                model_map, model_size, weight_offset, weight_bytes,
+                &weight_inner);
+            if (!weight_buffer || f16_bytes > NSUIntegerMax) {
+                return ds4_gpu_q4_attn_q_b_f16_abort_cold_build(
+                    required, 0, 1,
+                    DS4_Q4_ATTN_Q_B_F16_CIRCUIT_HARD);
+            }
+
+            f16_buffer = [g_device
+                newBufferWithLength:(NSUInteger)f16_bytes
+                            options:MTLResourceStorageModePrivate];
+            if (!f16_buffer) {
+                return ds4_gpu_q4_attn_q_b_f16_abort_cold_build(
+                    required, 0, 1,
+                    DS4_Q4_ATTN_Q_B_F16_CIRCUIT_HARD);
+            }
+            f16_buffer.label = @"ds4_attn_q_b_q4_f16_cache";
+
+            /* The sidecar build is deliberately isolated from g_batch_cb.
+             * It has no activation dependency, and a retained command buffer
+             * lets us verify completion before making the entry visible to
+             * any stream.  Cache entries are therefore READY by definition. */
+            id<MTLCommandBuffer> build_cb =
+                [ds4_gpu_active_queue() commandBuffer];
+            if (!build_cb) {
+                return ds4_gpu_q4_attn_q_b_f16_abort_cold_build(
+                    required, 0, 1,
+                    DS4_Q4_ATTN_Q_B_F16_CIRCUIT_HARD);
+            }
+            build_cb.label = @"ds4 Q4 attn_q_b F16 sidecar build";
+
+            const uint32_t chunks_per_row = (uint32_t)(in_dim / 16u);
+            const uint32_t row_count = (uint32_t)out_dim;
+            id<MTLComputeCommandEncoder> enc =
+                [build_cb computeCommandEncoder];
+            if (!enc) {
+                return ds4_gpu_q4_attn_q_b_f16_abort_cold_build(
+                    required, 0, 1,
+                    DS4_Q4_ATTN_Q_B_F16_CIRCUIT_HARD);
+            }
+            [enc setComputePipelineState:dequant_pipeline];
+            [enc setBuffer:weight_buffer
+                    offset:(NSUInteger)weight_inner
+                   atIndex:0];
+            [enc setBuffer:f16_buffer offset:0 atIndex:1];
+            [enc setBytes:&chunks_per_row
+                   length:sizeof(chunks_per_row)
+                  atIndex:2];
+            [enc setBytes:&row_count length:sizeof(row_count) atIndex:3];
+            [enc dispatchThreadgroups:MTLSizeMake(
+                    ((NSUInteger)chunks_per_row + 63u) / 64u,
+                    (NSUInteger)row_count,
+                    1)
+                 threadsPerThreadgroup:MTLSizeMake(64, 1, 1)];
+            [enc endEncoding];
+            [build_cb commit];
+            if (!ds4_gpu_wait_command_buffer(
+                    build_cb, "Q4 attn_q_b F16 sidecar build")) {
+                return ds4_gpu_q4_attn_q_b_f16_abort_cold_build(
+                    required, 0, 1,
+                    DS4_Q4_ATTN_Q_B_F16_CIRCUIT_HARD);
+            }
+
+            const int insert_rc = ds4_gpu_q4_attn_q_b_f16_cache_insert(
+                    model_map, model_size, weight_offset, weight_bytes,
+                    in_dim, out_dim, f16_bytes, budget_bytes, f16_buffer);
+            if (insert_rc == 0) {
+                /* Another key may have consumed the last budget slot while
+                 * this build was in flight.  Nothing has touched out yet, so
+                 * the native Q4 path remains a clean fallback. */
+                return ds4_gpu_q4_attn_q_b_f16_abort_cold_build(
+                    required, 1, 0,
+                    DS4_Q4_ATTN_Q_B_F16_CIRCUIT_HARD);
+            }
+
+            if (insert_rc < 0) {
+                /* A concurrent builder won.  Consume the published winner
+                 * rather than reporting a false build failure. */
+                f16_buffer = nil;
+                hit = ds4_gpu_q4_attn_q_b_f16_cache_lookup(
+                    model_map, model_size, weight_offset, weight_bytes,
+                    in_dim, out_dim, &f16_buffer);
+                if (!hit || !f16_buffer) {
+                    return ds4_gpu_q4_attn_q_b_f16_abort_cold_build(
+                        required, 0, 1,
+                        DS4_Q4_ATTN_Q_B_F16_CIRCUIT_HARD);
+                }
+            }
+
+            if (insert_rc == 2) {
+                fprintf(stderr,
+                        "ds4: Metal resident Q4 attn_q_b F16 cache enabled "
+                        "(budget %llu MiB, min batch %llu tokens)\n",
+                        (unsigned long long)budget_mib,
+                        (unsigned long long)min_tokens);
+            }
+        }
+
+        if (build_lock_held) {
+            pthread_mutex_unlock(&g_q4_attn_q_b_f16_build_mu);
+        }
+
+        if (!f16_buffer) {
+            return ds4_gpu_q4_attn_q_b_f16_fallback(required, 0, 1);
+        }
+
+        int owned = 0;
+        id<MTLCommandBuffer> cb = ds4_gpu_command_buffer(&owned);
+        if (!cb) {
+            return ds4_gpu_q4_attn_q_b_f16_fallback(required, 0, 1);
+        }
+
+        bool mm_encoded = false;
+        if (use_f16_rhs) {
+            const bool rhs_encoded =
+                ds4_gpu_encode_cpy_f32_f16_1d(
+                    cb,
+                    xbuf,
+                    ds4_gpu_tensor_offset(x),
+                    ds4_gpu_tensor_buffer(q_half),
+                    ds4_gpu_tensor_offset(q_half),
+                    (uint32_t)((uint64_t)n_tok * in_dim)) != 0;
+            mm_encoded =
+                rhs_encoded &&
+                ds4_gpu_encode_f16_rhs_mm(
+                    cb,
+                    mm_f16_pipeline,
+                    f16_buffer,
+                    0,
+                    q_half,
+                    out,
+                    in_dim,
+                    out_dim,
+                    n_tok,
+                    in_dim * sizeof(uint16_t),
+                    mm_bc_out) != 0;
+            if (!mm_encoded) {
+                /* Copy/encoder setup failed before any output writer was
+                 * dispatched.  Keep the sidecar useful by degrading inside
+                 * this command buffer to its established F16/F32 consumer. */
+                use_f16_rhs = false;
+            }
+        }
+        if (!use_f16_rhs) {
+            ds4_gpu_mul_mm_args mm_args = ds4_gpu_make_mm_args(
+                in_dim, out_dim, n_tok, in_dim * sizeof(uint16_t));
+            id<MTLComputeCommandEncoder> enc =
+                ds4_gpu_compute_encoder(cb);
+            if (enc) {
+                [enc setComputePipelineState:mm_f32_pipeline];
+                [enc setBytes:&mm_args length:sizeof(mm_args) atIndex:0];
+                [enc setBuffer:f16_buffer offset:0 atIndex:1];
+                [enc setBuffer:xbuf
+                        offset:ds4_gpu_tensor_offset(x)
+                       atIndex:2];
+                [enc setBuffer:outbuf
+                        offset:ds4_gpu_tensor_offset(out)
+                       atIndex:3];
+                [enc setThreadgroupMemoryLength:
+                        (mm_bc_out ? 8192u : 6144u)
+                                            atIndex:0];
+                [enc dispatchThreadgroups:MTLSizeMake(
+                        ((NSUInteger)n_tok + 31u) / 32u,
+                        (NSUInteger)out_dim / 64u,
+                        1)
+                     threadsPerThreadgroup:MTLSizeMake(128, 1, 1)];
+                ds4_gpu_end_compute_encoder(cb, enc);
+                mm_encoded = true;
+            }
+        }
+        if (!mm_encoded) {
+            if (!ds4_gpu_finish_command_buffer(
+                    cb, owned,
+                    "Q4 attn_q_b cached F16 RHS encoder allocation")) {
+                return ds4_gpu_q4_attn_q_b_f16_fallback(1, 0, 1);
+            }
+            return ds4_gpu_q4_attn_q_b_f16_fallback(required, 0, 1);
+        }
+
+        if (!ds4_gpu_finish_command_buffer(
+                cb, owned,
+                use_f16_rhs
+                    ? "Q4 attn_q_b cached F16/F16 matmul"
+                    : "Q4 attn_q_b cached F16/F32 matmul")) {
+            /* out has an encoded writer (and may already be committed).
+             * Returning the optional-path sentinel would replay native Q4
+             * over partial/failed work, so this failure is always fatal. */
+            return ds4_gpu_q4_attn_q_b_f16_fallback(1, 0, 1);
+        }
+    }
+
+    if (!apply_norm_rope) return 1;
+    if (!ds4_gpu_head_rms_norm_rope_tail_tensor(
+            out, n_tok, n_head, head_dim, n_rot, pos0, n_ctx_orig,
+            inverse, freq_base, freq_scale, ext_factor, attn_factor,
+            beta_fast, beta_slow, eps)) {
+        /* The q_b matmul is already encoded.  Do not authorize the caller to
+         * replay the native path after a downstream encode/submit failure. */
+        return ds4_gpu_q4_attn_q_b_f16_fallback(1, 0, 1);
+    }
+    return 1;
+}
+
+/* Pre-M5 production path: rebuild one Q4_K q_b matrix into a
+ * stream-local 64 MiB F16 buffer, consume it immediately with the compact
+ * F16 RHS, and reuse that allocation for the next layer.  Under SSD streaming
+ * the current Q4 source is a short-lived exact no-copy view retained by its
+ * command buffer.  This removes the native kernel's per-output-tile
+ * dequantization without retaining a 64 MiB sidecar for every layer. */
+static int ds4_gpu_attn_q_b_transient_f16_head_rms_rope_tail_tensor(
+        ds4_gpu_tensor       *out,
+        ds4_gpu_tensor       *q_half,
+        const void           *model_map,
+        uint64_t              model_size,
+        uint64_t              weight_offset,
+        uint32_t              weight_type,
+        uint64_t              in_dim,
+        uint64_t              out_dim,
+        const ds4_gpu_tensor *x,
+        uint32_t              n_tok,
+        uint32_t              n_head,
+        uint32_t              head_dim,
+        uint32_t              n_rot,
+        uint32_t              pos0,
+        uint32_t              n_ctx_orig,
+        bool                  inverse,
+        float                 freq_base,
+        float                 freq_scale,
+        float                 ext_factor,
+        float                 attn_factor,
+        float                 beta_fast,
+        float                 beta_slow,
+        float                 eps) {
+    if (weight_type != DS4_METAL_TENSOR_Q4_K) return 0;
+    if (!g_initialized && !ds4_gpu_init()) return 0;
+
+    const uint64_t min_tokens = ds4_gpu_env_u64(
+        "DS4_METAL_Q4_ATTN_Q_B_TRANSIENT_F16_MIN_TOKENS",
+        4096u, 32u, UINT32_MAX);
+    if ((uint64_t)n_tok < min_tokens ||
+        g_quality_mode ||
+        g_batch_encoder_concurrent ||
+        !ds4_gpu_device_is_pre_m5_apple_silicon() ||
+        ds4_gpu_mpp_available() ||
+        in_dim != 1024u || out_dim != 32768u ||
+        !out || !q_half || !x || !model_map ||
+        n_head == 0u || head_dim == 0u ||
+        out_dim != (uint64_t)n_head * head_dim ||
+        n_rot > head_dim || (n_rot & 1u) != 0u ||
+        n_tok > (uint32_t)INT32_MAX ||
+        pos0 > (uint32_t)INT32_MAX - n_tok ||
+        (uint64_t)n_tok * in_dim > UINT32_MAX) {
+        return 0;
+    }
+
+    const uint64_t x_f32_bytes =
+        (uint64_t)n_tok * in_dim * sizeof(float);
+    const uint64_t x_f16_bytes =
+        (uint64_t)n_tok * in_dim * sizeof(uint16_t);
+    const uint64_t out_f32_bytes =
+        (uint64_t)n_tok * out_dim * sizeof(float);
+    id<MTLBuffer> xbuf = ds4_gpu_tensor_buffer(x);
+    id<MTLBuffer> qbuf = ds4_gpu_tensor_buffer(q_half);
+    id<MTLBuffer> outbuf = ds4_gpu_tensor_buffer(out);
+    if (!xbuf || !qbuf || !outbuf ||
+        ds4_gpu_tensor_bytes(x) < x_f32_bytes ||
+        ds4_gpu_tensor_bytes(q_half) < x_f16_bytes ||
+        ds4_gpu_tensor_bytes(out) < out_f32_bytes ||
+        ds4_gpu_tensor_prefixes_overlap(
+            q_half, x_f16_bytes, x, x_f32_bytes) ||
+        ds4_gpu_tensor_prefixes_overlap(
+            q_half, x_f16_bytes, out, out_f32_bytes)) {
+        return 0;
+    }
+
+    const uint64_t row_bytes = (in_dim / 256u) * 144u;
+    const uint64_t weight_bytes = out_dim * row_bytes;
+    const uint64_t f16_bytes =
+        in_dim * out_dim * sizeof(uint16_t);
+    if (weight_offset > model_size ||
+        weight_bytes > model_size - weight_offset ||
+        f16_bytes > NSUIntegerMax ||
+        g_ds4_stream < 0 || g_ds4_stream >= DS4_GPU_MAX_STREAMS) {
+        return 0;
+    }
+    if (g_ssd_streaming_mode &&
+        !ds4_gpu_q4_qb_transient_f16_admission_allows(
+            g_ds4_stream, model_map, model_size,
+            ds4_gpu_q4_attn_q_b_f16_cache_generation())) {
+        return 0;
+    }
+
+    const bool mm_bc_out = (n_tok & 31u) != 0u;
+    id<MTLComputePipelineState> dequant_pipeline =
+        ds4_gpu_get_pipeline("kernel_dequantize_q4_K_f16");
+    id<MTLComputePipelineState> mm_pipeline =
+        ds4_gpu_get_mul_mm_pipeline(
+            "kernel_mul_mm_f16_f16_rhs", false, mm_bc_out);
+    const int use_contiguous_copy =
+        ds4_gpu_env_bool(
+            "DS4_METAL_DISABLE_CONTIG_F32_F16_COPY") <= 0;
+    id<MTLComputePipelineState> copy_pipeline = use_contiguous_copy
+        ? g_cpy_contig_f32_f16_pipeline
+        : g_cpy_f32_f16_pipeline;
+    if (!dequant_pipeline || !mm_pipeline || !copy_pipeline ||
+        dequant_pipeline.maxTotalThreadsPerThreadgroup < 64u ||
+        mm_pipeline.threadExecutionWidth != 32u ||
+        mm_pipeline.maxTotalThreadsPerThreadgroup < 128u) {
+        return 0;
+    }
+
+    const int stream = g_ds4_stream;
+    id<MTLBuffer> scratch = nil;
+    pthread_mutex_lock(&g_q4_qb_transient_f16_scratch_mu);
+    const bool needs_allocation =
+        !g_q4_qb_transient_f16_scratch[stream] ||
+        g_q4_qb_transient_f16_scratch_capacity[stream] <
+            (NSUInteger)f16_bytes;
+    uint64_t admission_bytes = needs_allocation ? f16_bytes : 0u;
+    if (g_ssd_streaming_mode) {
+        const uint64_t source_view_bytes =
+            ds4_gpu_q4_attn_q_b_f16_exact_source_bytes(
+                model_size, weight_offset, weight_bytes);
+        admission_bytes = UINT64_MAX - admission_bytes < source_view_bytes
+            ? UINT64_MAX
+            : admission_bytes + source_view_bytes;
+    }
+    if (((needs_allocation || g_ssd_streaming_mode) &&
+         !ds4_gpu_q4_attn_q_b_f16_working_set_has_room(admission_bytes)) ||
+        !ds4_gpu_q4_qb_transient_f16_scratch_ensure_locked(
+            stream, f16_bytes)) {
+        pthread_mutex_unlock(&g_q4_qb_transient_f16_scratch_mu);
+        return 0;
+    }
+    scratch = g_q4_qb_transient_f16_scratch[stream];
+    pthread_mutex_unlock(&g_q4_qb_transient_f16_scratch_mu);
+
+    uint64_t weight_inner = 0u;
+    id<MTLBuffer> weight_buffer =
+        ds4_gpu_q4_attn_q_b_f16_wrap_source(
+            model_map, model_size, weight_offset, weight_bytes,
+            &weight_inner);
+    if (!weight_buffer || weight_inner > NSUIntegerMax) return 0;
+
+    int owned = 0;
+    id<MTLCommandBuffer> cb = ds4_gpu_command_buffer(&owned);
+    if (!cb) return 0;
+    if (g_ssd_streaming_mode) {
+        /* Retained command buffers normally own every bound resource.  Keep
+         * the exact mmap view alive explicitly as well so the diagnostic
+         * unretained-command-buffer mode and asynchronous progress flushes
+         * obey the same lifetime contract.  The completion block releases the
+         * ~18 MiB view immediately instead of accumulating one per layer. */
+        id<MTLBuffer> retained_weight_buffer = weight_buffer;
+        dispatch_group_enter(g_progress_completion_group);
+        __atomic_add_fetch(
+            &g_q4_qb_transient_f16_exact_views_created,
+            1u,
+            __ATOMIC_RELEASE);
+        __atomic_add_fetch(
+            &g_q4_qb_transient_f16_exact_views_live,
+            1u,
+            __ATOMIC_RELEASE);
+        [cb addCompletedHandler:^(id<MTLCommandBuffer> completed) {
+            (void)completed;
+            (void)[retained_weight_buffer length];
+            __atomic_sub_fetch(
+                &g_q4_qb_transient_f16_exact_views_live,
+                1u,
+                __ATOMIC_RELEASE);
+            dispatch_group_leave(g_progress_completion_group);
+        }];
+    }
+
+    /* Copy may share the caller's current serial batch encoder.  Close it
+     * before dequantization, then close the dequant encoder as well: each
+     * following layer can overwrite the same scratch only after the previous
+     * F16/F16 consumer has completed in command-buffer order. */
+    if (!ds4_gpu_encode_cpy_f32_f16_1d(
+            cb, xbuf, ds4_gpu_tensor_offset(x),
+            qbuf, ds4_gpu_tensor_offset(q_half),
+            (uint32_t)((uint64_t)n_tok * in_dim))) {
+        if (owned) {
+            (void)ds4_gpu_finish_command_buffer(
+                cb, owned, "Q4 q_b transient F16 RHS encode failure");
+        }
+        return 0;
+    }
+    if (!owned) ds4_gpu_close_batch_encoder();
+
+    if (!ds4_gpu_encode_q4_K_transient_f16(
+            cb, dequant_pipeline, weight_buffer,
+            (NSUInteger)weight_inner, scratch, in_dim, out_dim)) {
+        if (owned) {
+            (void)ds4_gpu_finish_command_buffer(
+                cb, owned, "Q4 q_b transient dequant encode failure");
+        }
+        return 0;
+    }
+    if (!owned) ds4_gpu_close_batch_encoder();
+
+    /* A true return from the helper means the output-writing matmul dispatch
+     * has been encoded.  From this point onward the native Q4 path must not
+     * replay the projection, even if submission or the downstream tail fails. */
+    if (!ds4_gpu_encode_f16_rhs_mm(
+            cb, mm_pipeline, scratch, 0u, q_half, out,
+            in_dim, out_dim, n_tok,
+            in_dim * sizeof(uint16_t), mm_bc_out)) {
+        if (owned) {
+            (void)ds4_gpu_finish_command_buffer(
+                cb, owned, "Q4 q_b transient F16/F16 encode failure");
+        }
+        return 0;
+    }
+    if (!ds4_gpu_finish_command_buffer(
+            cb, owned, "Q4 q_b transient F16/F16 matmul")) {
+        return -1;
+    }
+
+    if (!ds4_gpu_head_rms_norm_rope_tail_tensor(
+            out, n_tok, n_head, head_dim, n_rot, pos0, n_ctx_orig,
+            inverse, freq_base, freq_scale, ext_factor, attn_factor,
+            beta_fast, beta_slow, eps)) {
+        return -1;
+    }
+    return 1;
+}
+
 int ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor(
         ds4_gpu_tensor       *out,
         ds4_gpu_tensor       *q_half,
         const void           *model_map,
         uint64_t              model_size,
         uint64_t              weight_offset,
+        uint32_t              weight_type,
         uint64_t              in_dim,
         uint64_t              out_dim,
         const ds4_gpu_tensor *x,
@@ -22784,13 +28764,357 @@ int ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor(
         float                 beta_fast,
         float                 beta_slow,
         float                 eps) {
-    (void)out; (void)q_half; (void)model_map; (void)model_size;
-    (void)weight_offset; (void)in_dim; (void)out_dim; (void)x;
-    (void)n_tok; (void)n_head; (void)head_dim; (void)n_rot; (void)pos0;
-    (void)n_ctx_orig; (void)inverse; (void)freq_base; (void)freq_scale;
-    (void)ext_factor; (void)attn_factor; (void)beta_fast; (void)beta_slow;
-    (void)eps;
-    return 0;
+    const int required =
+        ds4_gpu_env_bool(
+            "DS4_METAL_REQUIRE_Q4_ATTN_Q_B_F16_CACHE") == 1;
+    const int transient_disabled =
+        ds4_gpu_env_bool(
+            "DS4_METAL_DISABLE_Q4_ATTN_Q_B_TRANSIENT_F16") == 1;
+    const int ssd_sidecar_opt_in =
+        g_ssd_streaming_mode &&
+        ds4_gpu_env_bool(
+            "DS4_METAL_ENABLE_Q4_ATTN_Q_B_F16_CACHE_WITH_SSD_STREAMING") == 1;
+    if (!required && !transient_disabled && !ssd_sidecar_opt_in) {
+        /* Default mode never falls through to the multi-GiB sidecar cache.
+         * A non-candidate or any failure before the output writer returns the
+         * native-Q4 fallback sentinel directly.  The explicit SSD hybrid
+         * remains on its persistent sidecar path; ordinary SSD streaming uses
+         * one exact Q4 view and the same stream-local transient scratch. */
+        return ds4_gpu_attn_q_b_transient_f16_head_rms_rope_tail_tensor(
+            out, q_half, model_map, model_size, weight_offset, weight_type,
+            in_dim, out_dim, x, n_tok, n_head, head_dim, n_rot, pos0,
+            n_ctx_orig, inverse, freq_base, freq_scale, ext_factor,
+            attn_factor, beta_fast, beta_slow, eps);
+    }
+    return ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor_impl(
+        out, q_half, model_map, model_size, weight_offset, weight_type,
+        in_dim, out_dim, x, n_tok, n_head, head_dim, n_rot, pos0,
+        n_ctx_orig, inverse, freq_base, freq_scale, ext_factor,
+        attn_factor, beta_fast, beta_slow, eps, 1);
+}
+
+int ds4_gpu_test_q4_attn_q_b_f16_projection_tensor(
+        ds4_gpu_tensor       *out,
+        const void           *model_map,
+        uint64_t              model_size,
+        uint64_t              weight_offset,
+        uint64_t              in_dim,
+        uint64_t              out_dim,
+        const ds4_gpu_tensor *x,
+        uint32_t              n_tok) {
+    return ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor_impl(
+        out, NULL, model_map, model_size, weight_offset,
+        DS4_METAL_TENSOR_Q4_K, in_dim, out_dim, x, n_tok,
+        64u, 512u, 0u, 0u, 0u, false,
+        10000.0f, 1.0f, 0.0f, 1.0f, 32.0f, 1.0f, 1.0e-6f, 0);
+}
+
+/* Allocate or grow one stream's private transient buffer.  The caller holds
+ * the metadata mutex only through publication and captures a strong reference
+ * before unlocking; GPU execution itself never serializes on this mutex. */
+static int ds4_gpu_q4_qb_transient_f16_scratch_ensure_locked(
+        int      stream,
+        uint64_t required_bytes) {
+    if (!g_device || stream < 0 || stream >= DS4_GPU_MAX_STREAMS ||
+        required_bytes == 0u ||
+        required_bytes > NSUIntegerMax) {
+        return 0;
+    }
+    if (g_q4_qb_transient_f16_scratch[stream] &&
+        g_q4_qb_transient_f16_scratch_capacity[stream] >=
+            (NSUInteger)required_bytes) {
+        return 1;
+    }
+
+    id<MTLBuffer> scratch = [g_device
+        newBufferWithLength:(NSUInteger)required_bytes
+                    options:MTLResourceStorageModePrivate];
+    if (!scratch) return 0;
+    scratch.label = [NSString stringWithFormat:
+        @"ds4 Q4 q_b transient F16 scratch stream %d", stream];
+    g_q4_qb_transient_f16_scratch[stream] = scratch;
+    g_q4_qb_transient_f16_scratch_capacity[stream] =
+        (NSUInteger)required_bytes;
+    return 1;
+}
+
+/* Expand Q4_K using the exact production sidecar kernel.  Ending this encoder
+ * before opening the F16/F16 matmul encoder establishes the required Metal
+ * dependency in one command buffer without a CPU wait or an intermediate
+ * allocation. */
+static int ds4_gpu_encode_q4_K_transient_f16(
+        id<MTLCommandBuffer>        cb,
+        id<MTLComputePipelineState> pipeline,
+        id<MTLBuffer>               q4_weights,
+        NSUInteger                  q4_weights_offset,
+        id<MTLBuffer>               f16_scratch,
+        uint64_t                    in_dim,
+        uint64_t                    out_dim) {
+    if (!cb || !pipeline || !q4_weights || !f16_scratch ||
+        in_dim == 0u || out_dim == 0u ||
+        (in_dim % 256u) != 0u ||
+        in_dim > UINT32_MAX || out_dim > UINT32_MAX) {
+        return 0;
+    }
+
+    const uint32_t chunks_per_row = (uint32_t)(in_dim / 16u);
+    const uint32_t row_count = (uint32_t)out_dim;
+    id<MTLComputeCommandEncoder> enc = ds4_gpu_compute_encoder(cb);
+    if (!enc) return 0;
+    [enc setComputePipelineState:pipeline];
+    [enc setBuffer:q4_weights offset:q4_weights_offset atIndex:0];
+    [enc setBuffer:f16_scratch offset:0 atIndex:1];
+    [enc setBytes:&chunks_per_row
+           length:sizeof(chunks_per_row)
+          atIndex:2];
+    [enc setBytes:&row_count length:sizeof(row_count) atIndex:3];
+    [enc dispatchThreadgroups:MTLSizeMake(
+            ((NSUInteger)chunks_per_row + 63u) / 64u,
+            (NSUInteger)row_count,
+            1)
+         threadsPerThreadgroup:MTLSizeMake(64, 1, 1)];
+    ds4_gpu_end_compute_encoder(cb, enc);
+    return 1;
+}
+
+int ds4_gpu_test_q4_attn_q_b_mm_arm_supported(
+        ds4_gpu_test_q4_qb_mm_arm arm) {
+    if (!g_initialized && !ds4_gpu_init()) return 0;
+    if (arm != DS4_GPU_TEST_Q4_QB_MM_Q4_TRANSIENT_F16_F16 ||
+        g_batch_cb || g_batch_encoder_concurrent) {
+        return 0;
+    }
+    id<MTLComputePipelineState> dequant_pipeline =
+        ds4_gpu_get_pipeline("kernel_dequantize_q4_K_f16");
+    id<MTLComputePipelineState> mm_pipeline =
+        ds4_gpu_get_mul_mm_pipeline(
+            "kernel_mul_mm_f16_f16_rhs", false, false);
+    return dequant_pipeline && mm_pipeline &&
+           dequant_pipeline.maxTotalThreadsPerThreadgroup >= 64u &&
+           mm_pipeline.threadExecutionWidth == 32u &&
+           mm_pipeline.maxTotalThreadsPerThreadgroup >= 128u;
+}
+
+int ds4_gpu_test_q4_attn_q_b_mm_variant_tensor(
+        ds4_gpu_tensor              *out_f32,
+        ds4_gpu_tensor              *rhs_f16,
+        const void                  *model_map,
+        uint64_t                     model_size,
+        uint64_t                     weight_offset,
+        uint64_t                     in_dim,
+        uint64_t                     out_dim,
+        const ds4_gpu_tensor        *x_f32,
+        uint32_t                     n_tok,
+        ds4_gpu_test_q4_qb_mm_arm    arm,
+        bool                         materialize_rhs) {
+    if (!g_initialized && !ds4_gpu_init()) return 0;
+    if (!out_f32 || !x_f32 || !model_map || n_tok == 0u ||
+        arm < DS4_GPU_TEST_Q4_QB_MM_Q4_F32 ||
+        arm >= DS4_GPU_TEST_Q4_QB_MM_ARM_COUNT ||
+        in_dim == 0u || out_dim == 0u ||
+        in_dim > INT32_MAX || out_dim > INT32_MAX ||
+        n_tok > (uint32_t)INT32_MAX ||
+        (in_dim % 256u) != 0u ||
+        n_tok > UINT32_MAX / in_dim ||
+        out_dim > UINT64_MAX / n_tok / sizeof(float)) {
+        return 0;
+    }
+
+    const bool weight_f16 =
+        arm == DS4_GPU_TEST_Q4_QB_MM_F16_F32 ||
+        arm == DS4_GPU_TEST_Q4_QB_MM_F16_F16;
+    const bool rhs_is_f16 =
+        arm == DS4_GPU_TEST_Q4_QB_MM_Q4_F16 ||
+        arm == DS4_GPU_TEST_Q4_QB_MM_F16_F16 ||
+        arm == DS4_GPU_TEST_Q4_QB_MM_Q4_TRANSIENT_F16_F16;
+    const bool transient_f16 =
+        arm == DS4_GPU_TEST_Q4_QB_MM_Q4_TRANSIENT_F16_F16;
+    const uint64_t q4_row_bytes = (in_dim / 256u) * 144u;
+    if (out_dim > UINT64_MAX / q4_row_bytes) return 0;
+    const uint64_t q4_weight_bytes = out_dim * q4_row_bytes;
+    if (weight_offset > model_size ||
+        q4_weight_bytes > model_size - weight_offset) {
+        return 0;
+    }
+
+    const uint64_t x_f32_bytes =
+        (uint64_t)n_tok * in_dim * sizeof(float);
+    const uint64_t rhs_f16_bytes =
+        (uint64_t)n_tok * in_dim * sizeof(uint16_t);
+    const uint64_t out_f32_bytes =
+        (uint64_t)n_tok * out_dim * sizeof(float);
+    if (ds4_gpu_tensor_bytes(x_f32) < x_f32_bytes ||
+        ds4_gpu_tensor_bytes(out_f32) < out_f32_bytes ||
+        (rhs_is_f16 &&
+         (!rhs_f16 ||
+          ds4_gpu_tensor_bytes(rhs_f16) < rhs_f16_bytes ||
+          ds4_gpu_tensor_prefixes_overlap(
+              rhs_f16, rhs_f16_bytes, x_f32, x_f32_bytes) ||
+          ds4_gpu_tensor_prefixes_overlap(
+              rhs_f16, rhs_f16_bytes, out_f32, out_f32_bytes)))) {
+        return 0;
+    }
+
+    @autoreleasepool {
+        id<MTLBuffer> weight_buffer = nil;
+        uint64_t weight_inner = 0u;
+        uint64_t weight_row_bytes = q4_row_bytes;
+        if (weight_f16) {
+            if (!ds4_gpu_q4_attn_q_b_f16_cache_peek(
+                    model_map, model_size, weight_offset,
+                    q4_weight_bytes, in_dim, out_dim,
+                    &weight_buffer) || !weight_buffer) {
+                return 0;
+            }
+            weight_row_bytes = in_dim * sizeof(uint16_t);
+        } else {
+            weight_buffer = ds4_gpu_wrap_model_range(
+                model_map, model_size, weight_offset, q4_weight_bytes,
+                &weight_inner);
+            if (!weight_buffer || weight_inner > NSUIntegerMax) return 0;
+        }
+
+        const char *pipeline_name = NULL;
+        switch (arm) {
+        case DS4_GPU_TEST_Q4_QB_MM_Q4_F32:
+            pipeline_name = "kernel_mul_mm_q4_K_f32";
+            break;
+        case DS4_GPU_TEST_Q4_QB_MM_Q4_F16:
+            pipeline_name = "kernel_mul_mm_q4_K_f16_rhs";
+            break;
+        case DS4_GPU_TEST_Q4_QB_MM_F16_F32:
+            pipeline_name = "kernel_mul_mm_f16_f32";
+            break;
+        case DS4_GPU_TEST_Q4_QB_MM_F16_F16:
+            pipeline_name = "kernel_mul_mm_f16_f16_rhs";
+            break;
+        case DS4_GPU_TEST_Q4_QB_MM_Q4_TRANSIENT_F16_F16:
+            pipeline_name = "kernel_mul_mm_f16_f16_rhs";
+            break;
+        default:
+            return 0;
+        }
+        const bool bc_out =
+            ((out_dim % 64u) != 0u || (n_tok % 32u) != 0u);
+        id<MTLComputePipelineState> pipeline =
+            ds4_gpu_get_mul_mm_pipeline(pipeline_name, false, bc_out);
+        id<MTLComputePipelineState> dequant_pipeline = transient_f16
+            ? ds4_gpu_get_pipeline("kernel_dequantize_q4_K_f16")
+            : nil;
+        id<MTLBuffer> x_buffer = ds4_gpu_tensor_buffer(x_f32);
+        id<MTLBuffer> out_buffer = ds4_gpu_tensor_buffer(out_f32);
+        if (!pipeline || !x_buffer || !out_buffer ||
+            (transient_f16 &&
+             (!dequant_pipeline ||
+              dequant_pipeline.maxTotalThreadsPerThreadgroup < 64u ||
+              pipeline.threadExecutionWidth != 32u ||
+              pipeline.maxTotalThreadsPerThreadgroup < 128u))) {
+            return 0;
+        }
+
+        uint64_t transient_f16_bytes = 0u;
+        id<MTLBuffer> transient_scratch = nil;
+        if (transient_f16) {
+            if (out_dim > UINT64_MAX / in_dim /
+                              sizeof(uint16_t) ||
+                g_batch_cb || g_batch_encoder_concurrent ||
+                g_ds4_stream < 0 ||
+                g_ds4_stream >= DS4_GPU_MAX_STREAMS) {
+                return 0;
+            }
+            transient_f16_bytes =
+                in_dim * out_dim * sizeof(uint16_t);
+            const int stream = g_ds4_stream;
+            pthread_mutex_lock(&g_q4_qb_transient_f16_scratch_mu);
+            /* Recheck after taking the lock: another test thread may have
+             * waited while a batch was opened. */
+            if (g_batch_cb || g_batch_encoder_concurrent ||
+                !ds4_gpu_q4_qb_transient_f16_scratch_ensure_locked(
+                    stream, transient_f16_bytes)) {
+                pthread_mutex_unlock(
+                    &g_q4_qb_transient_f16_scratch_mu);
+                return 0;
+            }
+            transient_scratch =
+                g_q4_qb_transient_f16_scratch[stream];
+            pthread_mutex_unlock(
+                &g_q4_qb_transient_f16_scratch_mu);
+        }
+
+        int owned = 0;
+        id<MTLCommandBuffer> cb = ds4_gpu_command_buffer(&owned);
+        if (!cb || (transient_f16 && !owned)) {
+            return 0;
+        }
+
+        bool encoded = true;
+        if (rhs_is_f16 && materialize_rhs) {
+            encoded = ds4_gpu_encode_cpy_f32_f16_1d(
+                cb,
+                x_buffer,
+                ds4_gpu_tensor_offset(x_f32),
+                ds4_gpu_tensor_buffer(rhs_f16),
+                ds4_gpu_tensor_offset(rhs_f16),
+                (uint32_t)((uint64_t)n_tok * in_dim)) != 0;
+        }
+        id<MTLBuffer> mm_weight_buffer = weight_buffer;
+        NSUInteger mm_weight_offset = (NSUInteger)weight_inner;
+        uint64_t mm_weight_row_bytes = weight_row_bytes;
+        if (encoded && transient_f16) {
+            encoded = ds4_gpu_encode_q4_K_transient_f16(
+                cb, dequant_pipeline, weight_buffer,
+                (NSUInteger)weight_inner, transient_scratch,
+                in_dim, out_dim) != 0;
+            if (encoded) {
+                mm_weight_buffer = transient_scratch;
+                mm_weight_offset = 0u;
+                mm_weight_row_bytes =
+                    in_dim * sizeof(uint16_t);
+            }
+        }
+        if (encoded && rhs_is_f16) {
+            encoded = ds4_gpu_encode_f16_rhs_mm(
+                cb, pipeline, mm_weight_buffer, mm_weight_offset,
+                rhs_f16, out_f32, in_dim, out_dim, n_tok,
+                mm_weight_row_bytes, bc_out) != 0;
+        } else if (encoded) {
+            ds4_gpu_mul_mm_args args = ds4_gpu_make_mm_args(
+                in_dim, out_dim, n_tok, weight_row_bytes);
+            id<MTLComputeCommandEncoder> enc =
+                ds4_gpu_compute_encoder(cb);
+            if (!enc) {
+                encoded = false;
+            } else {
+                [enc setComputePipelineState:pipeline];
+                [enc setBytes:&args length:sizeof(args) atIndex:0];
+                [enc setBuffer:weight_buffer
+                        offset:(NSUInteger)weight_inner
+                       atIndex:1];
+                [enc setBuffer:x_buffer
+                        offset:ds4_gpu_tensor_offset(x_f32)
+                       atIndex:2];
+                [enc setBuffer:out_buffer
+                        offset:ds4_gpu_tensor_offset(out_f32)
+                       atIndex:3];
+                [enc setThreadgroupMemoryLength:(bc_out ? 8192u : 6144u)
+                                        atIndex:0];
+                [enc dispatchThreadgroups:MTLSizeMake(
+                        ((NSUInteger)n_tok + 31u) / 32u,
+                        ((NSUInteger)out_dim + 63u) / 64u,
+                        1)
+                     threadsPerThreadgroup:MTLSizeMake(128, 1, 1)];
+                ds4_gpu_end_compute_encoder(cb, enc);
+            }
+        }
+
+        if (!encoded) {
+            (void)ds4_gpu_finish_command_buffer(
+                cb, owned, "Q4 q_b four-way benchmark encode failure");
+            return 0;
+        }
+        return ds4_gpu_finish_command_buffer(
+            cb, owned, "Q4 q_b four-way resident benchmark");
+    }
 }
 
 int ds4_gpu_dsv4_fp8_kv_quantize_tensor(
@@ -25616,7 +31940,169 @@ static int ds4_gpu_encode_fill_f32_rows(
     return 1;
 }
 
-int ds4_gpu_attention_output_q8_batch_tensor(
+typedef struct {
+    ds4_gpu_tensor       *out_hc;
+    const ds4_gpu_tensor *residual_hc;
+    const ds4_gpu_tensor *split;
+    id<MTLBuffer> __strong weight_buffer;
+    NSUInteger           weight_offset;
+} ds4_gpu_attn_out_hc_target;
+
+static int ds4_gpu_encode_attn_out_q8_mm_hc(
+        id<MTLCommandBuffer>              cb,
+        const ds4_gpu_attn_out_hc_target *target,
+        const ds4_gpu_tensor             *low,
+        uint32_t                          n_tokens) {
+    if (!cb || !target || !target->out_hc || !target->residual_hc ||
+        !target->split || !target->weight_buffer || !low || n_tokens == 0u) {
+        return 0;
+    }
+
+    id<MTLBuffer> lowbuf = ds4_gpu_tensor_buffer(low);
+    id<MTLBuffer> resbuf = ds4_gpu_tensor_buffer(target->residual_hc);
+    id<MTLBuffer> splitbuf = ds4_gpu_tensor_buffer(target->split);
+    id<MTLBuffer> outbuf = ds4_gpu_tensor_buffer(target->out_hc);
+    if (!lowbuf || !resbuf || !splitbuf || !outbuf ||
+        !g_dsv4_attn_out_q8_mm_hc_expand4_pipeline) {
+        return 0;
+    }
+
+    const uint32_t in_dim = 8192u;
+    const uint32_t out_dim = 4096u;
+    const uint32_t n_hc = 4u;
+    const uint64_t row_bytes = (uint64_t)(in_dim / 32u) * 34u;
+    ds4_gpu_mul_mm_args mm =
+        ds4_gpu_make_mm_args(in_dim, out_dim, n_tokens, row_bytes);
+    const uint64_t mix_hc = 2ull * n_hc + (uint64_t)n_hc * n_hc;
+    ds4_gpu_hc_expand_args hc = {
+        .n_embd = out_dim,
+        .n_hc = n_hc,
+        .n_tokens = (int64_t)n_tokens,
+        .nb_block0 = sizeof(float),
+        .nb_block1 = (uint64_t)out_dim * sizeof(float),
+        .nb_add0 = sizeof(float),
+        .nb_add1 = (uint64_t)out_dim * sizeof(float),
+        .nb_res0 = sizeof(float),
+        .nb_res1 = (uint64_t)out_dim * sizeof(float),
+        .nb_res2 = (uint64_t)n_hc * out_dim * sizeof(float),
+        .nb_post0 = sizeof(float),
+        .nb_post1 = mix_hc * sizeof(float),
+        .nb_comb0 = sizeof(float),
+        .nb_comb1 = (uint64_t)n_hc * sizeof(float),
+        .nb_comb2 = mix_hc * sizeof(float),
+        .nb0 = sizeof(float),
+        .nb1 = (uint64_t)out_dim * sizeof(float),
+        .nb2 = (uint64_t)n_hc * out_dim * sizeof(float),
+        .has_add = 0,
+    };
+
+    id<MTLComputeCommandEncoder> enc = ds4_gpu_compute_encoder(cb);
+    if (!enc) return 0;
+    [enc setComputePipelineState:g_dsv4_attn_out_q8_mm_hc_expand4_pipeline];
+    [enc setBytes:&mm length:sizeof(mm) atIndex:0];
+    [enc setBuffer:target->weight_buffer offset:target->weight_offset atIndex:1];
+    [enc setBuffer:lowbuf offset:ds4_gpu_tensor_offset(low) atIndex:2];
+    [enc setBuffer:resbuf offset:ds4_gpu_tensor_offset(target->residual_hc) atIndex:3];
+    [enc setBuffer:splitbuf
+            offset:ds4_gpu_tensor_offset(target->split) +
+                   (NSUInteger)n_hc * sizeof(float)
+           atIndex:4];
+    [enc setBuffer:splitbuf
+            offset:ds4_gpu_tensor_offset(target->split) +
+                   (NSUInteger)(2u * n_hc) * sizeof(float)
+           atIndex:5];
+    [enc setBuffer:outbuf offset:ds4_gpu_tensor_offset(target->out_hc) atIndex:6];
+    [enc setBytes:&hc length:sizeof(hc) atIndex:7];
+    [enc setThreadgroupMemoryLength:8192u atIndex:0];
+    [enc dispatchThreadgroups:MTLSizeMake((NSUInteger)n_tokens / 32u,
+                                          (NSUInteger)out_dim / 64u,
+                                          1u)
+         threadsPerThreadgroup:MTLSizeMake(128u, 1u, 1u)];
+    ds4_gpu_end_compute_encoder(cb, enc);
+    return 1;
+}
+
+static int ds4_gpu_encode_attn_out_q4_mm_hc(
+        id<MTLCommandBuffer>              cb,
+        const ds4_gpu_attn_out_hc_target *target,
+        const ds4_gpu_tensor             *rhs_f16,
+        uint32_t                          n_tokens) {
+    if (!cb || !target || !target->out_hc || !target->residual_hc ||
+        !target->split || !target->weight_buffer || !rhs_f16 ||
+        n_tokens == 0u || (n_tokens % 32u) != 0u) {
+        return 0;
+    }
+
+    id<MTLBuffer> rhsbuf = ds4_gpu_tensor_buffer(rhs_f16);
+    id<MTLBuffer> resbuf = ds4_gpu_tensor_buffer(target->residual_hc);
+    id<MTLBuffer> splitbuf = ds4_gpu_tensor_buffer(target->split);
+    id<MTLBuffer> outbuf = ds4_gpu_tensor_buffer(target->out_hc);
+    if (!rhsbuf || !resbuf || !splitbuf || !outbuf ||
+        !g_dsv4_attn_out_q4_mm_hc_expand4_pipeline) {
+        return 0;
+    }
+
+    const uint32_t in_dim = 8192u;
+    const uint32_t out_dim = 4096u;
+    const uint32_t n_hc = 4u;
+    const uint64_t row_bytes = (uint64_t)(in_dim / 256u) * 144u;
+    ds4_gpu_mul_mm_args mm =
+        ds4_gpu_make_mm_args(in_dim, out_dim, n_tokens, row_bytes);
+    mm.nb10 = sizeof(uint16_t);
+    mm.nb11 = (uint64_t)in_dim * sizeof(uint16_t);
+    mm.nb12 = (uint64_t)in_dim * n_tokens * sizeof(uint16_t);
+    mm.nb13 = mm.nb12;
+
+    const uint64_t mix_hc = 2ull * n_hc + (uint64_t)n_hc * n_hc;
+    ds4_gpu_hc_expand_args hc = {
+        .n_embd = out_dim,
+        .n_hc = n_hc,
+        .n_tokens = (int64_t)n_tokens,
+        .nb_block0 = sizeof(float),
+        .nb_block1 = (uint64_t)out_dim * sizeof(float),
+        .nb_add0 = sizeof(float),
+        .nb_add1 = (uint64_t)out_dim * sizeof(float),
+        .nb_res0 = sizeof(float),
+        .nb_res1 = (uint64_t)out_dim * sizeof(float),
+        .nb_res2 = (uint64_t)n_hc * out_dim * sizeof(float),
+        .nb_post0 = sizeof(float),
+        .nb_post1 = mix_hc * sizeof(float),
+        .nb_comb0 = sizeof(float),
+        .nb_comb1 = (uint64_t)n_hc * sizeof(float),
+        .nb_comb2 = mix_hc * sizeof(float),
+        .nb0 = sizeof(float),
+        .nb1 = (uint64_t)out_dim * sizeof(float),
+        .nb2 = (uint64_t)n_hc * out_dim * sizeof(float),
+        .has_add = 0,
+    };
+
+    id<MTLComputeCommandEncoder> enc = ds4_gpu_compute_encoder(cb);
+    if (!enc) return 0;
+    [enc setComputePipelineState:g_dsv4_attn_out_q4_mm_hc_expand4_pipeline];
+    [enc setBytes:&mm length:sizeof(mm) atIndex:0];
+    [enc setBuffer:target->weight_buffer offset:target->weight_offset atIndex:1];
+    [enc setBuffer:rhsbuf offset:ds4_gpu_tensor_offset(rhs_f16) atIndex:2];
+    [enc setBuffer:resbuf offset:ds4_gpu_tensor_offset(target->residual_hc) atIndex:3];
+    [enc setBuffer:splitbuf
+            offset:ds4_gpu_tensor_offset(target->split) +
+                   (NSUInteger)n_hc * sizeof(float)
+           atIndex:4];
+    [enc setBuffer:splitbuf
+            offset:ds4_gpu_tensor_offset(target->split) +
+                   (NSUInteger)(2u * n_hc) * sizeof(float)
+           atIndex:5];
+    [enc setBuffer:outbuf offset:ds4_gpu_tensor_offset(target->out_hc) atIndex:6];
+    [enc setBytes:&hc length:sizeof(hc) atIndex:7];
+    [enc setThreadgroupMemoryLength:8192u atIndex:0];
+    [enc dispatchThreadgroups:MTLSizeMake((NSUInteger)n_tokens / 32u,
+                                          (NSUInteger)out_dim / 64u,
+                                          1u)
+         threadsPerThreadgroup:MTLSizeMake(128u, 1u, 1u)];
+    ds4_gpu_end_compute_encoder(cb, enc);
+    return 1;
+}
+
+static int ds4_gpu_attention_output_q8_batch_impl(
         ds4_gpu_tensor       *out,
         ds4_gpu_tensor       *low,
         ds4_gpu_tensor       *group_tmp,
@@ -25630,7 +32116,8 @@ int ds4_gpu_attention_output_q8_batch_tensor(
         uint32_t                n_groups,
         uint64_t                out_dim,
         const ds4_gpu_tensor *heads,
-        uint32_t                n_tokens) {
+        uint32_t                n_tokens,
+        const ds4_gpu_attn_out_hc_target *hc_target) {
     if (!g_initialized && !ds4_gpu_init()) return 0;
     if (!out || !low || !group_tmp || !low_tmp || !heads || !model_map ||
         group_dim == 0 || rank == 0 || n_groups == 0 || out_dim == 0 || n_tokens == 0 ||
@@ -25886,8 +32373,30 @@ int ds4_gpu_attention_output_q8_batch_tensor(
                     .nb1 = (uint64_t)rank * sizeof(float),
                     .nr0 = 2,
                 };
-                id<MTLComputePipelineState> pipeline =
-                    ds4_gpu_get_mul_mv_pipeline("kernel_dsv4_attn_out_low_q8_0_f32", 4);
+                const bool force_flash_decode_static_for_test =
+                    (g_test_flags & DS4_GPU_TEST_ATTN_OUT_LOW_Q8_STATIC) != 0u;
+                const bool use_pre_m5_flash_decode_static =
+                    (ds4_gpu_device_is_pre_m5_apple_silicon() ||
+                     force_flash_decode_static_for_test) &&
+                    (!g_attn_out_low_q8_static_unavailable ||
+                     force_flash_decode_static_for_test) &&
+                    getenv("DS4_METAL_DISABLE_PRE_M5_DECODE_PORTS") == NULL &&
+                    getenv("DS4_METAL_DISABLE_PRE_M5_ATTN_OUT_LOW_Q8_STATIC") == NULL &&
+                    group_dim == 4096u && rank == 1024u && n_groups == 8u &&
+                    low_dim == 8192u && row_a_bytes == 4352u &&
+                    out_a_bytes == 35651584u;
+                id<MTLComputePipelineState> pipeline = ds4_gpu_get_mul_mv_pipeline(
+                    use_pre_m5_flash_decode_static ?
+                        "kernel_dsv4_attn_out_low_q8_0_flash_decode_static_f32" :
+                        "kernel_dsv4_attn_out_low_q8_0_f32",
+                    4);
+                if (!pipeline && use_pre_m5_flash_decode_static) {
+                    g_attn_out_low_q8_static_unavailable = true;
+                    if (!force_flash_decode_static_for_test) {
+                        pipeline = ds4_gpu_get_mul_mv_pipeline(
+                            "kernel_dsv4_attn_out_low_q8_0_f32", 4);
+                    }
+                }
                 ok = ds4_gpu_encode_attn_out_low_q8_direct(cb,
                                                              pipeline,
                                                              &args,
@@ -25944,9 +32453,14 @@ int ds4_gpu_attention_output_q8_batch_tensor(
         DS4_METAL_PROFILE_ATTN_OUT_STAGE("low_proj");
 
         if (ok) {
-            ok = ds4_gpu_matmul_q8_0_tensor(out, model_map, model_size,
-                                              out_b_offset,
-                                              low_dim, out_dim, low, n_tokens) != 0;
+            if (hc_target) {
+                ok = ds4_gpu_encode_attn_out_q8_mm_hc(
+                    cb, hc_target, low, n_tokens) != 0;
+            } else {
+                ok = ds4_gpu_matmul_q8_0_tensor(
+                    out, model_map, model_size, out_b_offset,
+                    low_dim, out_dim, low, n_tokens) != 0;
+            }
         }
         DS4_METAL_PROFILE_ATTN_OUT_STAGE("out_proj");
 
@@ -25958,11 +32472,153 @@ int ds4_gpu_attention_output_q8_batch_tensor(
     }
 }
 
-int ds4_gpu_attention_output_q4_K_batch_tensor(
+int ds4_gpu_attention_output_q8_batch_tensor(
         ds4_gpu_tensor       *out,
         ds4_gpu_tensor       *low,
         ds4_gpu_tensor       *group_tmp,
         ds4_gpu_tensor       *low_tmp,
+        const void           *model_map,
+        uint64_t              model_size,
+        uint64_t              out_a_offset,
+        uint64_t              out_b_offset,
+        uint64_t              group_dim,
+        uint64_t              rank,
+        uint32_t              n_groups,
+        uint64_t              out_dim,
+        const ds4_gpu_tensor *heads,
+        uint32_t              n_tokens) {
+    return ds4_gpu_attention_output_q8_batch_impl(
+        out, low, group_tmp, low_tmp, model_map, model_size,
+        out_a_offset, out_b_offset, group_dim, rank, n_groups, out_dim,
+        heads, n_tokens, NULL);
+}
+
+/* Return 0 before encoding when the exact resident fast path is unavailable,
+ * allowing the caller to run the established output-B + HC fallback.  Once
+ * selected, an encode/submit failure returns -1 so work is never replayed over
+ * a partially written HC destination. */
+int ds4_gpu_attention_output_q8_batch_hc_tensor(
+        ds4_gpu_tensor       *out,
+        ds4_gpu_tensor       *out_hc,
+        const ds4_gpu_tensor *residual_hc,
+        const ds4_gpu_tensor *split,
+        ds4_gpu_tensor       *low,
+        ds4_gpu_tensor       *group_tmp,
+        ds4_gpu_tensor       *low_tmp,
+        const void           *model_map,
+        uint64_t              model_size,
+        uint64_t              out_a_offset,
+        uint64_t              out_b_offset,
+        uint64_t              group_dim,
+        uint64_t              rank,
+        uint32_t              n_groups,
+        uint64_t              out_dim,
+        const ds4_gpu_tensor *heads,
+        uint32_t              n_tokens,
+        uint32_t              n_hc) {
+    if (!g_initialized && !ds4_gpu_init()) return -1;
+    const bool force =
+        (g_test_flags & DS4_GPU_TEST_BATCH_ATTN_OUT_Q8_HC_FUSION) != 0u;
+    const bool require =
+        getenv("DS4_METAL_REQUIRE_PRE_M5_BATCH_ATTN_OUT_HC_FUSION") != NULL;
+    if (group_dim != 4096u || rank != 1024u || n_groups != 8u ||
+        out_dim != 4096u || n_hc != 4u ||
+        n_tokens < 32u || (n_tokens < 512u && !force) ||
+        n_tokens > 4096u || (n_tokens % 32u) != 0u) {
+        return require ? -1 : 0;
+    }
+
+    const NSUInteger dynamic_bytes = 8192u;
+    const NSUInteger static_bytes =
+        g_dsv4_attn_out_q8_mm_hc_expand4_pipeline
+            ? g_dsv4_attn_out_q8_mm_hc_expand4_pipeline
+                  .staticThreadgroupMemoryLength
+            : 0u;
+    const NSUInteger max_tg_bytes =
+        g_device ? g_device.maxThreadgroupMemoryLength : 0u;
+    const bool pipeline_ok =
+        g_dsv4_attn_out_q8_mm_hc_expand4_pipeline != nil &&
+        g_dsv4_attn_out_q8_mm_hc_expand4_pipeline.threadExecutionWidth == 32u &&
+        g_dsv4_attn_out_q8_mm_hc_expand4_pipeline
+                .maxTotalThreadsPerThreadgroup >= 128u &&
+        static_bytes <= max_tg_bytes &&
+        dynamic_bytes <= max_tg_bytes - static_bytes;
+    const uint64_t low_dim = (uint64_t)n_groups * rank;
+    const uint64_t row_a_bytes = group_dim / 32u * 34u;
+    const uint64_t row_b_bytes = low_dim / 32u * 34u;
+    const uint64_t out_a_bytes = (uint64_t)n_groups * rank * row_a_bytes;
+    const uint64_t out_b_bytes = out_dim * row_b_bytes;
+    const uint64_t heads_bytes =
+        (uint64_t)n_tokens * n_groups * group_dim * sizeof(float);
+    const uint64_t low_bytes =
+        (uint64_t)n_tokens * low_dim * sizeof(float);
+    const uint64_t out_bytes =
+        (uint64_t)n_tokens * out_dim * sizeof(float);
+    const uint64_t hc_bytes =
+        (uint64_t)n_tokens * n_hc * out_dim * sizeof(float);
+    const uint64_t split_bytes =
+        (uint64_t)n_tokens * (2ull * n_hc + (uint64_t)n_hc * n_hc) *
+        sizeof(float);
+    bool eligible =
+        out && out_hc && residual_hc && split && low && group_tmp && low_tmp &&
+        heads && model_map &&
+        (ds4_gpu_device_is_pre_m5_apple_silicon() || force) &&
+        (!g_quality_mode || force) &&
+        !g_ssd_streaming_mode && !ds4_gpu_tp_world_is_two() &&
+        getenv("DS4_METAL_DISABLE_PRE_M5_BATCH_ATTN_OUT_HC_FUSION") == NULL &&
+        getenv("DS4_METAL_ATTN_OUT_STAGE_PROFILE") == NULL &&
+        getenv("DS4_METAL_Q8_PREFILL_PROFILE") == NULL &&
+        pipeline_ok && low_dim == 8192u &&
+        out_a_offset <= model_size &&
+        out_a_bytes <= model_size - out_a_offset &&
+        out_b_offset <= model_size &&
+        out_b_bytes <= model_size - out_b_offset &&
+        ds4_gpu_tensor_buffer(out) != nil &&
+        ds4_gpu_tensor_buffer(out_hc) != nil &&
+        ds4_gpu_tensor_buffer(residual_hc) != nil &&
+        ds4_gpu_tensor_buffer(split) != nil &&
+        ds4_gpu_tensor_buffer(low) != nil &&
+        ds4_gpu_tensor_buffer(heads) != nil &&
+        ds4_gpu_tensor_bytes(out) >= out_bytes &&
+        ds4_gpu_tensor_bytes(out_hc) >= hc_bytes &&
+        ds4_gpu_tensor_bytes(residual_hc) >= hc_bytes &&
+        ds4_gpu_tensor_bytes(split) >= split_bytes &&
+        ds4_gpu_tensor_bytes(low) >= low_bytes &&
+        ds4_gpu_tensor_bytes(heads) >= heads_bytes;
+
+    uint64_t out_b_inner = 0u;
+    id<MTLBuffer> out_b_buf = nil;
+    if (eligible) {
+        out_b_buf = ds4_gpu_wrap_model_range(
+            model_map, model_size, out_b_offset, out_b_bytes, &out_b_inner);
+        eligible = out_b_buf != nil && out_b_inner <= NSUIntegerMax;
+    }
+    if (!eligible) {
+        if (require) {
+            fprintf(stderr,
+                    "ds4: required Metal Q8 batch attention-output HC "
+                    "fusion was not selected\n");
+            return -1;
+        }
+        return 0;
+    }
+
+    ds4_gpu_attn_out_hc_target target = {
+        .out_hc = out_hc,
+        .residual_hc = residual_hc,
+        .split = split,
+        .weight_buffer = out_b_buf,
+        .weight_offset = (NSUInteger)out_b_inner,
+    };
+    return ds4_gpu_attention_output_q8_batch_impl(
+        out, low, group_tmp, low_tmp, model_map, model_size,
+        out_a_offset, out_b_offset, group_dim, rank, n_groups, out_dim,
+        heads, n_tokens, &target) ? 1 : -1;
+}
+
+static int ds4_gpu_attention_output_q4_K_ssd_prefill_exactn_tensor(
+        ds4_gpu_tensor       *out,
+        ds4_gpu_tensor       *low,
         const void             *model_map,
         uint64_t                model_size,
         uint64_t                out_a_offset,
@@ -25974,47 +32630,534 @@ int ds4_gpu_attention_output_q4_K_batch_tensor(
         uint64_t                out_dim,
         const ds4_gpu_tensor *heads,
         uint32_t                n_tokens) {
-    if (!g_initialized && !ds4_gpu_init()) return 0;
+    const bool scope = n_tokens >= 6u && n_tokens <= 31u;
+    const bool require = scope &&
+        ds4_gpu_env_bool("DS4_METAL_REQUIRE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN") == 1;
+    const bool require_scale_meta = scope &&
+        ds4_gpu_env_bool(
+            "DS4_METAL_REQUIRE_Q4_SSD_PREFILL_ATTN_OUT_SCALE_META") == 1;
+    const int failure_rc = require || require_scale_meta ? -1 : 0;
+    if (!scope) return 0;
+    if (!g_initialized && !ds4_gpu_init()) return failure_rc;
+
+    const bool disabled =
+        ds4_gpu_env_bool("DS4_METAL_DISABLE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN") == 1;
+    const bool enabled = require || require_scale_meta ||
+        ds4_gpu_env_bool("DS4_METAL_ENABLE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN") == 1;
+    const bool scale_meta_disabled =
+        ds4_gpu_env_bool(
+            "DS4_METAL_DISABLE_Q4_SSD_PREFILL_ATTN_OUT_SCALE_META") == 1;
+    const bool classic_q4 = getenv("DS4_METAL_DISABLE_Q4_MV_CLASSIC") == NULL;
+    const bool platform_ok =
+        g_ssd_streaming_mode &&
+        !g_quality_mode &&
+        ds4_gpu_device_is_pre_m5_apple_silicon();
+    if (!enabled || disabled || !classic_q4 || !platform_ok ||
+        out_b_type != DS4_METAL_TENSOR_Q4_K) {
+        if (failure_rc < 0) {
+            fprintf(stderr,
+                    "ds4: required Metal Q4 SSD-prefill attention exact-N "
+                    "path is ineligible (rows=%u type=%u ssd=%u quality=%u "
+                    "pre_m5=%u disabled=%u classic=%u)\n",
+                    n_tokens,
+                    out_b_type,
+                    g_ssd_streaming_mode ? 1u : 0u,
+                    g_quality_mode ? 1u : 0u,
+                    ds4_gpu_device_is_pre_m5_apple_silicon() ? 1u : 0u,
+                    disabled ? 1u : 0u,
+                    classic_q4 ? 1u : 0u);
+        }
+        return failure_rc;
+    }
+    if (require_scale_meta && scale_meta_disabled) {
+        fprintf(stderr,
+                "ds4: required Metal Q4 SSD-prefill attention exact-N "
+                "scale metadata is disabled\n");
+        return -1;
+    }
+
+    if (!out || !low || !heads || !model_map ||
+        group_dim == 0 || rank == 0 || n_groups == 0 || out_dim == 0 ||
+        group_dim > UINT32_MAX || rank > UINT32_MAX || out_dim > UINT32_MAX) {
+        return failure_rc;
+    }
+
+    @autoreleasepool {
+        uint64_t low_dim = 0;
+        if (!ds4_gpu_u64_mul_checked((uint64_t)n_groups, rank, &low_dim)) {
+            return failure_rc;
+        }
+        if ((group_dim % 256u) != 0 || (low_dim % 256u) != 0 ||
+            low_dim == 0 || low_dim > UINT32_MAX) {
+            if (failure_rc < 0) {
+                fprintf(stderr,
+                        "ds4: required Metal Q4 SSD-prefill attention exact-N "
+                        "path received unaligned dimensions\n");
+            }
+            return failure_rc;
+        }
+
+        uint64_t row_a_bytes = 0;
+        uint64_t row_b_bytes = 0;
+        if (!ds4_gpu_quant_row_bytes(DS4_METAL_TENSOR_Q4_K,
+                                     (uint32_t)group_dim,
+                                     &row_a_bytes) ||
+            !ds4_gpu_quant_row_bytes(DS4_METAL_TENSOR_Q4_K,
+                                     (uint32_t)low_dim,
+                                     &row_b_bytes)) {
+            return failure_rc;
+        }
+
+        uint64_t group_a_bytes = 0;
+        uint64_t out_a_bytes = 0;
+        uint64_t out_b_bytes = 0;
+        uint64_t heads_group_bytes = 0;
+        uint64_t heads_row_bytes = 0;
+        uint64_t heads_bytes = 0;
+        uint64_t low_row_bytes = 0;
+        uint64_t low_bytes = 0;
+        uint64_t out_row_bytes = 0;
+        uint64_t out_bytes = 0;
+        uint64_t rank_bytes = 0;
+        uint64_t stage_a_bytes = 0;
+        uint64_t stage_b_bytes = 0;
+        uint64_t scale_meta_a_bytes = 0;
+        uint64_t scale_meta_b_bytes = 0;
+        const uint64_t scale_meta_bytes_per_block_pair =
+            2u * 2u * sizeof(uint64_t); /* two rows, two iq records */
+        if (!ds4_gpu_u64_mul_checked(rank, row_a_bytes, &group_a_bytes) ||
+            !ds4_gpu_u64_mul_checked((uint64_t)n_groups,
+                                     group_a_bytes,
+                                     &out_a_bytes) ||
+            !ds4_gpu_u64_mul_checked(out_dim, row_b_bytes, &out_b_bytes) ||
+            !ds4_gpu_u64_mul_checked(group_dim,
+                                     sizeof(float),
+                                     &heads_group_bytes) ||
+            !ds4_gpu_u64_mul_checked((uint64_t)n_groups,
+                                     heads_group_bytes,
+                                     &heads_row_bytes) ||
+            !ds4_gpu_u64_mul_checked((uint64_t)n_tokens,
+                                     heads_row_bytes,
+                                     &heads_bytes) ||
+            !ds4_gpu_u64_mul_checked(low_dim,
+                                     sizeof(float),
+                                     &low_row_bytes) ||
+            !ds4_gpu_u64_mul_checked((uint64_t)n_tokens,
+                                     low_row_bytes,
+                                     &low_bytes) ||
+            !ds4_gpu_u64_mul_checked(out_dim,
+                                     sizeof(float),
+                                     &out_row_bytes) ||
+            !ds4_gpu_u64_mul_checked((uint64_t)n_tokens,
+                                     out_row_bytes,
+                                     &out_bytes) ||
+            !ds4_gpu_u64_mul_checked(rank,
+                                     sizeof(float),
+                                     &rank_bytes) ||
+            !ds4_gpu_u64_mul_checked(2u,
+                                     row_a_bytes,
+                                     &stage_a_bytes) ||
+            !ds4_gpu_u64_mul_checked(2u,
+                                     row_b_bytes,
+                                     &stage_b_bytes) ||
+            !ds4_gpu_u64_mul_checked(group_dim / 256u,
+                                     scale_meta_bytes_per_block_pair,
+                                     &scale_meta_a_bytes) ||
+            !ds4_gpu_u64_mul_checked(low_dim / 256u,
+                                     scale_meta_bytes_per_block_pair,
+                                     &scale_meta_b_bytes) ||
+            scale_meta_a_bytes > UINT64_MAX - stage_a_bytes ||
+            scale_meta_b_bytes > UINT64_MAX - stage_b_bytes) {
+            if (failure_rc < 0) {
+                fprintf(stderr,
+                        "ds4: required Metal Q4 SSD-prefill attention exact-N "
+                        "shape overflows byte strides\n");
+            }
+            return failure_rc;
+        }
+        if (out_a_offset > model_size || out_a_bytes > model_size - out_a_offset ||
+            out_b_offset > model_size || out_b_bytes > model_size - out_b_offset) {
+            if (failure_rc < 0) {
+                fprintf(stderr,
+                        "ds4: required Metal Q4 SSD-prefill attention exact-N "
+                        "weights are outside the mapped model\n");
+            }
+            return failure_rc;
+        }
+
+        id<MTLBuffer> heads_buf = ds4_gpu_tensor_buffer(heads);
+        id<MTLBuffer> low_buf = ds4_gpu_tensor_buffer(low);
+        id<MTLBuffer> out_buf = ds4_gpu_tensor_buffer(out);
+        if (!heads_buf || !low_buf || !out_buf ||
+            ds4_gpu_tensor_bytes(heads) < heads_bytes ||
+            ds4_gpu_tensor_bytes(low) < low_bytes ||
+            ds4_gpu_tensor_bytes(out) < out_bytes) {
+            if (failure_rc < 0) {
+                fprintf(stderr,
+                        "ds4: required Metal Q4 SSD-prefill attention exact-N "
+                        "path received undersized tensors\n");
+            }
+            return failure_rc;
+        }
+
+        const uint64_t scale_meta_stage_a_bytes =
+            stage_a_bytes + scale_meta_a_bytes;
+        const uint64_t scale_meta_stage_b_bytes =
+            stage_b_bytes + scale_meta_b_bytes;
+        const uint64_t scale_meta_max_stage_bytes =
+            scale_meta_stage_a_bytes > scale_meta_stage_b_bytes ?
+                scale_meta_stage_a_bytes : scale_meta_stage_b_bytes;
+        bool scale_meta =
+            !scale_meta_disabled &&
+            scale_meta_max_stage_bytes <=
+                (uint64_t)[g_device maxThreadgroupMemoryLength] &&
+            scale_meta_max_stage_bytes <= (uint64_t)NSUIntegerMax;
+        id<MTLComputePipelineState> pipeline = nil;
+        if (scale_meta) {
+            pipeline = ds4_gpu_get_pipeline(
+                "kernel_dsv4_attn_out_q4_K_ssd_prefill_exactn_scale_meta_f32");
+            if (!pipeline || pipeline.threadExecutionWidth != 32u ||
+                pipeline.maxTotalThreadsPerThreadgroup < 512u) {
+                scale_meta = false;
+            }
+        }
+        if (!scale_meta && require_scale_meta) {
+            fprintf(stderr,
+                    "ds4: required Metal Q4 SSD-prefill attention exact-N "
+                    "scale metadata pipeline is unavailable "
+                    "(stage=%llu max_stage=%lu)\n",
+                    (unsigned long long)scale_meta_max_stage_bytes,
+                    (unsigned long)[g_device maxThreadgroupMemoryLength]);
+            return -1;
+        }
+        if (!scale_meta) {
+            pipeline = ds4_gpu_get_pipeline(
+                "kernel_dsv4_attn_out_q4_K_ssd_prefill_exactn_f32");
+        } else {
+            stage_a_bytes = scale_meta_stage_a_bytes;
+            stage_b_bytes = scale_meta_stage_b_bytes;
+        }
+        const uint64_t max_stage_bytes =
+            stage_a_bytes > stage_b_bytes ? stage_a_bytes : stage_b_bytes;
+        if (!pipeline || pipeline.threadExecutionWidth != 32u ||
+            pipeline.maxTotalThreadsPerThreadgroup < 512u ||
+            max_stage_bytes > (uint64_t)[g_device maxThreadgroupMemoryLength] ||
+            max_stage_bytes > (uint64_t)NSUIntegerMax) {
+            if (failure_rc < 0) {
+                fprintf(stderr,
+                        "ds4: required Metal Q4 SSD-prefill attention exact-N "
+                        "pipeline is unavailable (threads=%lu width=%lu "
+                        "stage=%llu max_stage=%lu)\n",
+                        (unsigned long)(pipeline ?
+                            pipeline.maxTotalThreadsPerThreadgroup : 0u),
+                        (unsigned long)(pipeline ?
+                            pipeline.threadExecutionWidth : 0u),
+                        (unsigned long long)max_stage_bytes,
+                        (unsigned long)[g_device maxThreadgroupMemoryLength]);
+            }
+            return failure_rc;
+        }
+
+        uint64_t out_a_inner = 0;
+        uint64_t out_b_inner = 0;
+        id<MTLBuffer> out_a_buf = ds4_gpu_wrap_model_range(
+            model_map, model_size, out_a_offset, out_a_bytes, &out_a_inner);
+        id<MTLBuffer> out_b_buf = ds4_gpu_wrap_model_range(
+            model_map, model_size, out_b_offset, out_b_bytes, &out_b_inner);
+        if (!out_a_buf || !out_b_buf) return failure_rc;
+
+        const ds4_gpu_q4_attn_exactn_args args_a = {
+            .in_dim = (uint32_t)group_dim,
+            .out_rows = (uint32_t)rank,
+            .n_groups = n_groups,
+            .n_tokens = n_tokens,
+            .weight_row_bytes = row_a_bytes,
+            .weight_group_bytes = group_a_bytes,
+            .input_group_bytes = heads_group_bytes,
+            .input_token_bytes = heads_row_bytes,
+            .output_group_bytes = rank_bytes,
+            .output_token_bytes = low_row_bytes,
+        };
+        const ds4_gpu_q4_attn_exactn_args args_b = {
+            .in_dim = (uint32_t)low_dim,
+            .out_rows = (uint32_t)out_dim,
+            .n_groups = 1u,
+            .n_tokens = n_tokens,
+            .weight_row_bytes = row_b_bytes,
+            .weight_group_bytes = out_b_bytes,
+            .input_group_bytes = 0,
+            .input_token_bytes = low_row_bytes,
+            .output_group_bytes = 0,
+            .output_token_bytes = out_row_bytes,
+        };
+
+        const bool had_batch = g_batch_cb != nil;
+        if (!had_batch && ds4_gpu_begin_commands() == 0) return failure_rc;
+
+        int owned = 0;
+        id<MTLCommandBuffer> cb = ds4_gpu_command_buffer(&owned);
+        if (!cb || owned) {
+            if (!had_batch) (void)ds4_gpu_end_commands();
+            return failure_rc;
+        }
+        id<MTLComputeCommandEncoder> enc = ds4_gpu_compute_encoder(cb);
+        if (!enc) {
+            if (!had_batch) (void)ds4_gpu_end_commands();
+            return failure_rc;
+        }
+
+        [enc setComputePipelineState:pipeline];
+        [enc setBytes:&args_a length:sizeof(args_a) atIndex:0];
+        [enc setBuffer:out_a_buf offset:(NSUInteger)out_a_inner atIndex:1];
+        [enc setBuffer:heads_buf offset:ds4_gpu_tensor_offset(heads) atIndex:2];
+        [enc setBuffer:low_buf offset:ds4_gpu_tensor_offset(low) atIndex:3];
+        [enc setThreadgroupMemoryLength:(NSUInteger)stage_a_bytes atIndex:0];
+        [enc dispatchThreadgroups:MTLSizeMake(((NSUInteger)rank + 1u) / 2u,
+                                              ((NSUInteger)n_tokens + 15u) / 16u,
+                                              (NSUInteger)n_groups)
+             threadsPerThreadgroup:MTLSizeMake(32u, 16u, 1u)];
+
+        [enc setBytes:&args_b length:sizeof(args_b) atIndex:0];
+        [enc setBuffer:out_b_buf offset:(NSUInteger)out_b_inner atIndex:1];
+        [enc setBuffer:low_buf offset:ds4_gpu_tensor_offset(low) atIndex:2];
+        [enc setBuffer:out_buf offset:ds4_gpu_tensor_offset(out) atIndex:3];
+        [enc setThreadgroupMemoryLength:(NSUInteger)stage_b_bytes atIndex:0];
+        [enc dispatchThreadgroups:MTLSizeMake(((NSUInteger)out_dim + 1u) / 2u,
+                                              ((NSUInteger)n_tokens + 15u) / 16u,
+                                              1u)
+             threadsPerThreadgroup:MTLSizeMake(32u, 16u, 1u)];
+        ds4_gpu_end_compute_encoder(cb, enc);
+
+        if (!had_batch && ds4_gpu_end_commands() == 0) {
+            // Work was already submitted: never let the caller replay the
+            // row fallback into the same output after a command-buffer error.
+            return -1;
+        }
+        return 1;
+    }
+}
+
+static int ds4_gpu_attention_output_q4_K_batch_impl(
+        ds4_gpu_tensor       *out,
+        ds4_gpu_tensor       *low,
+        ds4_gpu_tensor       *group_tmp,
+        ds4_gpu_tensor       *low_tmp,
+        const void             *model_map,
+        uint64_t                model_size,
+        uint64_t                out_a_offset,
+        uint64_t                out_b_offset,
+        uint32_t                out_b_type,
+        uint64_t                group_dim,
+        uint64_t                rank,
+        uint32_t                n_groups,
+        uint64_t                out_dim,
+        const ds4_gpu_tensor *heads,
+        uint32_t                n_tokens,
+        const ds4_gpu_attn_out_hc_target *hc_target) {
+    const bool force_f16_rhs =
+        (g_test_flags & DS4_GPU_TEST_BATCH_ATTN_OUT_Q4_HC_FUSION) != 0u;
+    const bool require_f16_rhs =
+        hc_target != NULL ||
+        ds4_gpu_env_bool("DS4_METAL_REQUIRE_Q4_ATTN_OUT_B_F16_RHS") == 1;
+    const bool disable_f16_rhs =
+        ds4_gpu_env_bool("DS4_METAL_DISABLE_Q4_ATTN_OUT_B_F16_RHS") == 1;
+    const bool require_direct_low =
+        getenv("DS4_METAL_REQUIRE_Q4_ATTN_OUT_A_DIRECT") != NULL;
+    const bool disable_direct_low =
+        getenv("DS4_METAL_DISABLE_Q4_ATTN_OUT_A_DIRECT") != NULL;
+
+    if (!require_f16_rhs && !require_direct_low &&
+        n_tokens >= 6u && n_tokens <= 31u) {
+        const int exactn_rc =
+            ds4_gpu_attention_output_q4_K_ssd_prefill_exactn_tensor(
+                out,
+                low,
+                model_map,
+                model_size,
+                out_a_offset,
+                out_b_offset,
+                out_b_type,
+                group_dim,
+                rank,
+                n_groups,
+                out_dim,
+                heads,
+                n_tokens);
+        if (exactn_rc != 0) return exactn_rc;
+    }
+
+    const bool tiny_scope = n_tokens >= 2u && n_tokens <= 5u;
+    const bool require_tiny =
+        tiny_scope &&
+        ds4_gpu_env_bool("DS4_METAL_REQUIRE_Q4_ATTN_OUT_TINY_BATCH") == 1;
+    const int failure_rc =
+        (require_tiny || require_f16_rhs || require_direct_low) ? -1 : 0;
+    if (!g_initialized && !ds4_gpu_init()) return failure_rc;
     if (!out || !low || !heads || !model_map ||
         group_dim == 0 || rank == 0 || n_groups == 0 || out_dim == 0 || n_tokens == 0 ||
         group_dim > UINT32_MAX || rank > UINT32_MAX || out_dim > UINT32_MAX) {
-        return 0;
+        return failure_rc;
+    }
+
+    /*
+     * DSpark exact-N only needs two through five rows.  The Q4 attention-A
+     * decode kernel already carries a token index in the same dispatch.  The
+     * Q8 exact-row output projection and the Q4_K classic multi-row matvec
+     * both preserve the canonical one-row arithmetic independently for each
+     * token.  Keep their composition experimental until it wins an end-to-end
+     * A/B: unsupported shapes and either disabled gate normally return zero
+     * to the caller's established row-wise fallback.  The REQUIRE diagnostic
+     * implies the enable gate inside this tiny scope and turns any ineligible
+     * or failed candidate into -1 so a model-backed A/B cannot false-green.
+     * Respect the classic-Q4 kill switch so an explicitly selected alternate
+     * schedule stays canonical.
+     */
+    const bool tiny_disabled =
+        ds4_gpu_env_bool("DS4_METAL_DISABLE_Q4_ATTN_OUT_TINY_BATCH") == 1;
+    const bool tiny_out_b_supported =
+        out_b_type == DS4_METAL_TENSOR_Q8_0 ||
+        (out_b_type == DS4_METAL_TENSOR_Q4_K &&
+         getenv("DS4_METAL_DISABLE_Q4_MV_CLASSIC") == NULL);
+    const bool use_tiny_exact =
+        tiny_scope &&
+        tiny_out_b_supported &&
+        (require_tiny ||
+         ds4_gpu_env_bool("DS4_METAL_ENABLE_Q4_ATTN_OUT_TINY_BATCH") == 1) &&
+        !tiny_disabled;
+    if (require_tiny && !use_tiny_exact) {
+        fprintf(stderr,
+                "ds4: required Metal Q4 attention-output tiny batch is "
+                "ineligible (rows=%u out_b_type=%u disabled=%u)\n",
+                n_tokens, out_b_type, tiny_disabled ? 1u : 0u);
+        return -1;
     }
-    if (n_tokens < 32u) return 0;
+    if (n_tokens < 32u && !use_tiny_exact) return failure_rc;
 
     @autoreleasepool {
-        const uint64_t low_dim = (uint64_t)n_groups * rank;
+        uint64_t low_dim = 0;
+        if (!ds4_gpu_u64_mul_checked((uint64_t)n_groups, rank, &low_dim)) {
+            return failure_rc;
+        }
         if ((group_dim % 256u) != 0 || (low_dim % 256u) != 0 || low_dim > UINT32_MAX) {
-            return 0;
+            return failure_rc;
         }
 
         uint64_t row_a_bytes = 0;
         uint64_t row_b_bytes = 0;
         if (!ds4_gpu_quant_row_bytes(DS4_METAL_TENSOR_Q4_K, (uint32_t)group_dim, &row_a_bytes) ||
             !ds4_gpu_quant_row_bytes(out_b_type, (uint32_t)low_dim, &row_b_bytes)) {
-            return 0;
+            return failure_rc;
         }
 
-        const uint64_t out_a_bytes = (uint64_t)n_groups * rank * row_a_bytes;
-        const uint64_t out_b_bytes = out_dim * row_b_bytes;
+        uint64_t out_a_bytes = 0;
+        uint64_t out_b_bytes = 0;
+        if (!ds4_gpu_u64_mul_checked(low_dim, row_a_bytes, &out_a_bytes) ||
+            !ds4_gpu_u64_mul_checked(out_dim, row_b_bytes, &out_b_bytes)) {
+            return failure_rc;
+        }
         if (out_a_offset > model_size || out_a_bytes > model_size - out_a_offset ||
             out_b_offset > model_size || out_b_bytes > model_size - out_b_offset) {
             fprintf(stderr, "ds4: Metal Q4 attention output batch weights are outside the mapped model\n");
-            return 0;
+            return failure_rc;
         }
 
-        const uint64_t heads_bytes = (uint64_t)n_tokens * n_groups * group_dim * sizeof(float);
-        const uint64_t low_bytes = (uint64_t)n_tokens * low_dim * sizeof(float);
-        const uint64_t out_bytes = (uint64_t)n_tokens * out_dim * sizeof(float);
+        uint64_t heads_row_elements = 0;
+        uint64_t heads_elements = 0;
+        uint64_t heads_bytes = 0;
+        uint64_t low_elements = 0;
+        uint64_t low_bytes = 0;
+        uint64_t out_elements = 0;
+        uint64_t out_bytes = 0;
+        if (!ds4_gpu_u64_mul_checked(
+                (uint64_t)n_groups, group_dim, &heads_row_elements) ||
+            !ds4_gpu_u64_mul_checked(
+                (uint64_t)n_tokens, heads_row_elements, &heads_elements) ||
+            !ds4_gpu_u64_mul_checked(
+                heads_elements, sizeof(float), &heads_bytes) ||
+            !ds4_gpu_u64_mul_checked(
+                (uint64_t)n_tokens, low_dim, &low_elements) ||
+            !ds4_gpu_u64_mul_checked(
+                low_elements, sizeof(float), &low_bytes) ||
+            !ds4_gpu_u64_mul_checked(
+                (uint64_t)n_tokens, out_dim, &out_elements) ||
+            !ds4_gpu_u64_mul_checked(
+                out_elements, sizeof(float), &out_bytes)) {
+            return failure_rc;
+        }
         if (ds4_gpu_tensor_bytes(heads) < heads_bytes ||
             ds4_gpu_tensor_bytes(low) < low_bytes ||
             ds4_gpu_tensor_bytes(out) < out_bytes) {
             fprintf(stderr, "ds4: Metal Q4 attention output batch received undersized buffers\n");
-            return 0;
+            return failure_rc;
         }
-        (void)group_tmp;
         (void)low_tmp;
 
+        const uint64_t f16_rhs_elements = low_elements;
+        uint64_t f16_rhs_bytes = 0;
+        const bool f16_rhs_size_valid =
+            f16_rhs_elements <= UINT32_MAX &&
+            ds4_gpu_u64_mul_checked(
+                f16_rhs_elements, sizeof(uint16_t), &f16_rhs_bytes);
+        const bool f16_rhs_buffers_valid =
+            group_tmp != NULL &&
+            ds4_gpu_tensor_bytes(group_tmp) >= f16_rhs_bytes &&
+            ds4_gpu_tensor_buffer(group_tmp) != nil &&
+            !ds4_gpu_tensor_prefixes_overlap(
+                group_tmp, f16_rhs_bytes, low, low_bytes) &&
+            !ds4_gpu_tensor_prefixes_overlap(
+                group_tmp, f16_rhs_bytes, out, out_bytes) &&
+            !ds4_gpu_tensor_prefixes_overlap(
+                group_tmp, f16_rhs_bytes, heads, heads_bytes);
+        /* Resident pre-M5 models benefit from materializing output-B's RHS
+         * once.  Keep SSD streaming on the established F32 path: storage
+         * latency masks this saving, while the extra copy/cache footprint did
+         * not produce a stable whole-prefill win in that execution mode. */
+        bool use_f16_rhs =
+            !g_ssd_streaming_mode &&
+            !disable_f16_rhs &&
+            (ds4_gpu_device_is_pre_m5_apple_silicon() || force_f16_rhs) &&
+            !g_batch_encoder_concurrent &&
+            out_b_type == DS4_METAL_TENSOR_Q4_K &&
+            (n_tokens % 32u) == 0 &&
+            f16_rhs_size_valid &&
+            f16_rhs_buffers_valid;
+        const bool f16_rhs_bc_out = (out_dim % 64u) != 0;
+        id<MTLComputePipelineState> f16_rhs_pipeline = nil;
+        id<MTLBuffer> out_b_buf = nil;
+        uint64_t out_b_inner = 0;
+        if (use_f16_rhs) {
+            f16_rhs_pipeline = hc_target
+                ? g_dsv4_attn_out_q4_mm_hc_expand4_pipeline
+                : ds4_gpu_get_mul_mm_pipeline(
+                    "kernel_mul_mm_q4_K_f16_rhs", false, f16_rhs_bc_out);
+            if (hc_target) {
+                out_b_buf = hc_target->weight_buffer;
+                out_b_inner = hc_target->weight_offset;
+            } else {
+                out_b_buf = ds4_gpu_wrap_model_range(
+                    model_map, model_size, out_b_offset, out_b_bytes,
+                    &out_b_inner);
+            }
+            if (!f16_rhs_pipeline || !out_b_buf || out_b_inner > NSUIntegerMax) {
+                use_f16_rhs = false;
+            }
+        }
+        if (require_f16_rhs && !use_f16_rhs) {
+            fprintf(stderr,
+                    "ds4: required Metal Q4 attention output-B F16 RHS is "
+                    "ineligible (rows=%u type=%u scratch=%llu/%llu "
+                    "disabled=%u pre_m5=%u ssd=%u)\n",
+                    n_tokens,
+                    out_b_type,
+                    (unsigned long long)(group_tmp
+                        ? ds4_gpu_tensor_bytes(group_tmp) : 0u),
+                    (unsigned long long)f16_rhs_bytes,
+                    disable_f16_rhs ? 1u : 0u,
+                    ds4_gpu_device_is_pre_m5_apple_silicon() ? 1u : 0u,
+                    g_ssd_streaming_mode ? 1u : 0u);
+            return -1;
+        }
+
         const uint64_t padded_n_tokens_u64 =
             ((uint64_t)n_tokens + DS4_METAL_ATTN_OUT_MPP_TILE_N - 1u) /
             DS4_METAL_ATTN_OUT_MPP_TILE_N * DS4_METAL_ATTN_OUT_MPP_TILE_N;
@@ -26047,9 +33190,62 @@ int ds4_gpu_attention_output_q4_K_batch_tensor(
                 use_mpp_low = false;
             }
         }
+
+        /* On pre-M5 GPUs the production attention output-A route is fixed:
+         * slot g always consumes Q4 weight group g.  The generic MoE kernel
+         * nevertheless builds and reads an expert-major route map and work
+         * list before scattering back to the same token/group layout.  The
+         * direct kernel removes only those indirections; its dequantization,
+         * F16 staging, MMA order, and final stores are bit-identical.  Keep
+         * the default to the measured long-prefill range and leave Metal4 on
+         * its cooperative-tensor path. */
+        const bool direct_low_eligible =
+            !use_mpp_low && !use_tiny_exact &&
+            n_tokens >= 512u && n_tokens <= 4096u &&
+            group_dim == 4096u && rank == 1024u && n_groups == 8u &&
+            ds4_gpu_device_is_pre_m5_apple_silicon() &&
+            g_tp_split_world == 1 &&
+            !g_batch_encoder_concurrent;
+        bool use_direct_low =
+            direct_low_eligible && !disable_direct_low;
+        id<MTLComputePipelineState> direct_low_pipeline = nil;
+        if (use_direct_low) {
+            direct_low_pipeline = ds4_gpu_get_pipeline(
+                "kernel_attn_out_low_q4_K_legacy_direct");
+            const NSUInteger max_tgmem =
+                (NSUInteger)[g_device maxThreadgroupMemoryLength];
+            const NSUInteger static_tgmem = direct_low_pipeline
+                ? direct_low_pipeline.staticThreadgroupMemoryLength
+                : 0u;
+            if (!direct_low_pipeline ||
+                direct_low_pipeline.threadExecutionWidth != 32u ||
+                direct_low_pipeline.maxTotalThreadsPerThreadgroup < 128u ||
+                static_tgmem > max_tgmem ||
+                8192u > max_tgmem - static_tgmem) {
+                use_direct_low = false;
+            }
+        }
+        if (require_direct_low && !use_direct_low) {
+            fprintf(stderr,
+                    "ds4: required Metal Q4 attention output-A direct path "
+                    "is ineligible (rows=%u shape=%llux%llux%u disabled=%u "
+                    "pre_m5=%u tp_world=%d concurrent=%u mpp=%u tiny=%u)\n",
+                    n_tokens,
+                    (unsigned long long)group_dim,
+                    (unsigned long long)rank,
+                    n_groups,
+                    disable_direct_low ? 1u : 0u,
+                    ds4_gpu_device_is_pre_m5_apple_silicon() ? 1u : 0u,
+                    g_tp_split_world,
+                    g_batch_encoder_concurrent ? 1u : 0u,
+                    use_mpp_low ? 1u : 0u,
+                    use_tiny_exact ? 1u : 0u);
+            return -1;
+        }
+        const int selected_failure_rc = use_direct_low ? -1 : failure_rc;
         const NSUInteger ids_bytes = (NSUInteger)n_tokens * (NSUInteger)n_groups * sizeof(int32_t);
         id<MTLBuffer> group_ids_buffer = nil;
-        if (!use_mpp_low) {
+        if (!use_mpp_low && !use_direct_low && !use_tiny_exact) {
             if (getenv("DS4_METAL_DISABLE_ATTN_OUT_IDS_CACHE") != NULL) {
                 group_ids_buffer =
                     ds4_gpu_new_transient_buffer(ids_bytes, "attention output Q4 group ids");
@@ -26059,7 +33255,7 @@ int ds4_gpu_attention_output_q4_K_batch_tensor(
                                                      "ds4_attention_output_group_ids")) {
                 group_ids_buffer = g_attn_out_group_ids_buffer;
             }
-            if (!group_ids_buffer) return 0;
+            if (!group_ids_buffer) return failure_rc;
 
             int32_t *ids = (int32_t *)[group_ids_buffer contents];
             for (uint32_t t = 0; t < n_tokens; t++) {
@@ -26074,10 +33270,12 @@ int ds4_gpu_attention_output_q4_K_batch_tensor(
             ds4_gpu_wrap_model_range(model_map, model_size,
                                      out_a_offset, out_a_bytes,
                                      &out_a_inner);
-        if (!out_a_buf) return 0;
+        if (!out_a_buf) return failure_rc;
 
         const bool had_batch = g_batch_cb != nil;
-        if (!had_batch && ds4_gpu_begin_commands() == 0) return 0;
+        if (!had_batch && ds4_gpu_begin_commands() == 0) {
+            return selected_failure_rc;
+        }
 
         bool ok = true;
         int owned = 0;
@@ -26096,7 +33294,47 @@ int ds4_gpu_attention_output_q4_K_batch_tensor(
                                             use_mpp_low && use_mpp_padding
                                                 ? padded_n_tokens
                                                 : n_tokens);
-            if (use_mpp_low) {
+            if (use_tiny_exact) {
+                ds4_gpu_mul_mv_id_args args = {
+                    .nei0 = (int32_t)n_groups,
+                    .nei1 = (int32_t)n_tokens,
+                    .nbi1 = 0,
+                    .ne00 = (int32_t)group_dim,
+                    .ne01 = (int32_t)rank,
+                    .ne02 = (int32_t)n_groups,
+                    .nb00 = 1,
+                    .nb01 = row_a_bytes,
+                    .nb02 = (uint64_t)rank * row_a_bytes,
+                    .ne10 = (int32_t)group_dim,
+                    .ne11 = (int32_t)n_groups,
+                    .ne12 = (int32_t)n_tokens,
+                    .ne13 = 1,
+                    .nb10 = sizeof(float),
+                    .nb11 = (uint64_t)group_dim * sizeof(float),
+                    .nb12 = (uint64_t)n_groups * group_dim * sizeof(float),
+                    .ne0 = (int32_t)rank,
+                    .ne1 = (int32_t)n_groups,
+                    .nb1 = (uint64_t)rank * sizeof(float),
+                    .nr0 = 2,
+                };
+                const NSUInteger nsg = 2u;
+                id<MTLComputePipelineState> pipeline =
+                    ds4_gpu_get_mul_mv_pipeline(
+                        "kernel_dsv4_attn_out_low_q4_K_f32", (int16_t)nsg);
+                ok = ds4_gpu_encode_attn_out_low_q8_direct(
+                        cb,
+                        pipeline,
+                        &args,
+                        out_a_buf,
+                        (NSUInteger)out_a_inner,
+                        ds4_gpu_tensor_buffer(heads),
+                        ds4_gpu_tensor_offset(heads),
+                        ds4_gpu_tensor_buffer(low),
+                        ds4_gpu_tensor_offset(low),
+                        32u,
+                        nsg,
+                        false) != 0;
+            } else if (use_mpp_low) {
                 ok = ds4_gpu_encode_attn_out_low_mpp(
                         cb,
                         mpp_low_pipeline,
@@ -26107,6 +33345,17 @@ int ds4_gpu_attention_output_q4_K_batch_tensor(
                         ds4_gpu_tensor_offset(heads),
                         ds4_gpu_tensor_buffer(low),
                         ds4_gpu_tensor_offset(low)) != 0;
+            } else if (use_direct_low) {
+                ok = ds4_gpu_encode_attn_out_low_q4_direct(
+                        cb,
+                        direct_low_pipeline,
+                        &mm_args,
+                        out_a_buf,
+                        (NSUInteger)out_a_inner,
+                        ds4_gpu_tensor_buffer(heads),
+                        ds4_gpu_tensor_offset(heads),
+                        ds4_gpu_tensor_buffer(low),
+                        ds4_gpu_tensor_offset(low)) != 0;
             } else {
                 ds4_gpu_mul_mm_id_map_args map_args =
                     ds4_gpu_make_mul_mm_id_map_args((uint32_t)group_dim,
@@ -26135,22 +33384,246 @@ int ds4_gpu_attention_output_q4_K_batch_tensor(
         }
 
         if (ok) {
-            ok = ds4_gpu_matmul_quant_tensor(out,
-                                             model_map,
-                                             model_size,
-                                             out_b_offset,
-                                             out_b_type,
-                                             low_dim,
-                                             out_dim,
-                                             low,
-                                             n_tokens) != 0;
+            if (use_f16_rhs) {
+                const bool encoded_f16_rhs =
+                    ds4_gpu_encode_cpy_f32_f16_1d(
+                        cb,
+                        ds4_gpu_tensor_buffer(low),
+                        ds4_gpu_tensor_offset(low),
+                        ds4_gpu_tensor_buffer(group_tmp),
+                        ds4_gpu_tensor_offset(group_tmp),
+                        (uint32_t)f16_rhs_elements) != 0 &&
+                    (hc_target
+                        ? ds4_gpu_encode_attn_out_q4_mm_hc(
+                            cb, hc_target, group_tmp, n_tokens) != 0
+                        : ds4_gpu_encode_f16_rhs_mm(
+                            cb,
+                            f16_rhs_pipeline,
+                            out_b_buf,
+                            (NSUInteger)out_b_inner,
+                            group_tmp,
+                            out,
+                            low_dim,
+                            out_dim,
+                            n_tokens,
+                            row_b_bytes,
+                            f16_rhs_bc_out) != 0);
+                if (!encoded_f16_rhs) {
+                    ok = require_f16_rhs
+                        ? false
+                        : ds4_gpu_matmul_quant_tensor(
+                            out,
+                            model_map,
+                            model_size,
+                            out_b_offset,
+                            out_b_type,
+                            low_dim,
+                            out_dim,
+                            low,
+                            n_tokens) != 0;
+                }
+            } else if (use_tiny_exact && out_b_type == DS4_METAL_TENSOR_Q8_0) {
+                ok = ds4_gpu_matmul_q8_0_decode_rows_exact_tensor(
+                        out,
+                        model_map,
+                        model_size,
+                        out_b_offset,
+                        low_dim,
+                        out_dim,
+                        low,
+                        n_tokens) != 0;
+            } else if (use_tiny_exact) {
+                ok = ds4_gpu_matmul_quant_tensor(out,
+                                                 model_map,
+                                                 model_size,
+                                                 out_b_offset,
+                                                 out_b_type,
+                                                 low_dim,
+                                                 out_dim,
+                                                 low,
+                                                 n_tokens) != 0;
+            } else {
+                ok = ds4_gpu_matmul_quant_tensor(out,
+                                                 model_map,
+                                                 model_size,
+                                                 out_b_offset,
+                                                 out_b_type,
+                                                 low_dim,
+                                                 out_dim,
+                                                 low,
+                                                 n_tokens) != 0;
+            }
         }
 
         if (!had_batch) {
             ok = ds4_gpu_end_commands() != 0 && ok;
         }
-        return ok ? 1 : 0;
+        return ok ? 1 : selected_failure_rc;
+    }
+}
+
+int ds4_gpu_attention_output_q4_K_batch_tensor(
+        ds4_gpu_tensor       *out,
+        ds4_gpu_tensor       *low,
+        ds4_gpu_tensor       *group_tmp,
+        ds4_gpu_tensor       *low_tmp,
+        const void           *model_map,
+        uint64_t              model_size,
+        uint64_t              out_a_offset,
+        uint64_t              out_b_offset,
+        uint32_t              out_b_type,
+        uint64_t              group_dim,
+        uint64_t              rank,
+        uint32_t              n_groups,
+        uint64_t              out_dim,
+        const ds4_gpu_tensor *heads,
+        uint32_t              n_tokens) {
+    return ds4_gpu_attention_output_q4_K_batch_impl(
+        out, low, group_tmp, low_tmp, model_map, model_size,
+        out_a_offset, out_b_offset, out_b_type, group_dim, rank,
+        n_groups, out_dim, heads, n_tokens, NULL);
+}
+
+/* Like the Q8 resident tail, return zero only while the established output-B
+ * plus standalone-HC fallback is still safe to run.  Once selected, failures
+ * are fatal to the candidate because the batch may contain partial writes. */
+int ds4_gpu_attention_output_q4_K_batch_hc_tensor(
+        ds4_gpu_tensor       *out,
+        ds4_gpu_tensor       *out_hc,
+        const ds4_gpu_tensor *residual_hc,
+        const ds4_gpu_tensor *split,
+        ds4_gpu_tensor       *low,
+        ds4_gpu_tensor       *group_tmp,
+        ds4_gpu_tensor       *low_tmp,
+        const void           *model_map,
+        uint64_t              model_size,
+        uint64_t              out_a_offset,
+        uint64_t              out_b_offset,
+        uint32_t              out_b_type,
+        uint64_t              group_dim,
+        uint64_t              rank,
+        uint32_t              n_groups,
+        uint64_t              out_dim,
+        const ds4_gpu_tensor *heads,
+        uint32_t              n_tokens,
+        uint32_t              n_hc) {
+    if (!g_initialized && !ds4_gpu_init()) return -1;
+    const bool force =
+        (g_test_flags & DS4_GPU_TEST_BATCH_ATTN_OUT_Q4_HC_FUSION) != 0u;
+    const bool require =
+        getenv("DS4_METAL_REQUIRE_Q4_BATCH_ATTN_OUT_HC_FUSION") != NULL;
+    if (out_b_type != DS4_METAL_TENSOR_Q4_K || group_dim != 4096u ||
+        rank != 1024u || n_groups != 8u || out_dim != 4096u ||
+        n_hc != 4u || n_tokens < 32u ||
+        (n_tokens < 512u && !force) ||
+        n_tokens > 4096u || (n_tokens % 32u) != 0u) {
+        return require ? -1 : 0;
+    }
+
+    const NSUInteger dynamic_bytes = 8192u;
+    const NSUInteger static_bytes =
+        g_dsv4_attn_out_q4_mm_hc_expand4_pipeline
+            ? g_dsv4_attn_out_q4_mm_hc_expand4_pipeline
+                  .staticThreadgroupMemoryLength
+            : 0u;
+    const NSUInteger max_tg_bytes =
+        g_device ? g_device.maxThreadgroupMemoryLength : 0u;
+    const bool pipeline_ok =
+        g_dsv4_attn_out_q4_mm_hc_expand4_pipeline != nil &&
+        g_dsv4_attn_out_q4_mm_hc_expand4_pipeline.threadExecutionWidth == 32u &&
+        g_dsv4_attn_out_q4_mm_hc_expand4_pipeline
+                .maxTotalThreadsPerThreadgroup >= 128u &&
+        static_bytes <= max_tg_bytes &&
+        dynamic_bytes <= max_tg_bytes - static_bytes;
+    const uint64_t low_dim = (uint64_t)n_groups * rank;
+    const uint64_t row_a_bytes = group_dim / 256u * 144u;
+    const uint64_t row_b_bytes = low_dim / 256u * 144u;
+    const uint64_t out_a_bytes = low_dim * row_a_bytes;
+    const uint64_t out_b_bytes = out_dim * row_b_bytes;
+    const uint64_t heads_bytes =
+        (uint64_t)n_tokens * n_groups * group_dim * sizeof(float);
+    const uint64_t low_bytes =
+        (uint64_t)n_tokens * low_dim * sizeof(float);
+    const uint64_t rhs_f16_bytes =
+        (uint64_t)n_tokens * low_dim * sizeof(uint16_t);
+    const uint64_t out_bytes =
+        (uint64_t)n_tokens * out_dim * sizeof(float);
+    const uint64_t hc_bytes =
+        (uint64_t)n_tokens * n_hc * out_dim * sizeof(float);
+    const uint64_t split_bytes =
+        (uint64_t)n_tokens * (2ull * n_hc + (uint64_t)n_hc * n_hc) *
+        sizeof(float);
+    bool eligible =
+        out && out_hc && residual_hc && split && low && group_tmp && low_tmp &&
+        heads && model_map &&
+        (ds4_gpu_device_is_pre_m5_apple_silicon() || force) &&
+        (!g_quality_mode || force) &&
+        !g_ssd_streaming_mode && !ds4_gpu_tp_world_is_two() &&
+        !g_batch_encoder_concurrent &&
+        getenv("DS4_METAL_DISABLE_PRE_M5_BATCH_ATTN_OUT_HC_FUSION") == NULL &&
+        getenv("DS4_METAL_DISABLE_Q4_ATTN_OUT_B_F16_RHS") == NULL &&
+        getenv("DS4_METAL_ATTN_OUT_STAGE_PROFILE") == NULL &&
+        pipeline_ok && low_dim == 8192u &&
+        out_a_offset <= model_size &&
+        out_a_bytes <= model_size - out_a_offset &&
+        out_b_offset <= model_size &&
+        out_b_bytes <= model_size - out_b_offset &&
+        ds4_gpu_tensor_buffer(out) != nil &&
+        ds4_gpu_tensor_buffer(out_hc) != nil &&
+        ds4_gpu_tensor_buffer(residual_hc) != nil &&
+        ds4_gpu_tensor_buffer(split) != nil &&
+        ds4_gpu_tensor_buffer(low) != nil &&
+        ds4_gpu_tensor_buffer(group_tmp) != nil &&
+        ds4_gpu_tensor_buffer(heads) != nil &&
+        ds4_gpu_tensor_bytes(out) >= out_bytes &&
+        ds4_gpu_tensor_bytes(out_hc) >= hc_bytes &&
+        ds4_gpu_tensor_bytes(residual_hc) >= hc_bytes &&
+        ds4_gpu_tensor_bytes(split) >= split_bytes &&
+        ds4_gpu_tensor_bytes(low) >= low_bytes &&
+        ds4_gpu_tensor_bytes(group_tmp) >= rhs_f16_bytes &&
+        ds4_gpu_tensor_bytes(heads) >= heads_bytes &&
+        !ds4_gpu_tensor_prefixes_overlap(
+            group_tmp, rhs_f16_bytes, out, out_bytes) &&
+        !ds4_gpu_tensor_prefixes_overlap(
+            group_tmp, rhs_f16_bytes, out_hc, hc_bytes) &&
+        !ds4_gpu_tensor_prefixes_overlap(
+            group_tmp, rhs_f16_bytes, residual_hc, hc_bytes) &&
+        !ds4_gpu_tensor_prefixes_overlap(
+            group_tmp, rhs_f16_bytes, split, split_bytes) &&
+        !ds4_gpu_tensor_prefixes_overlap(
+            group_tmp, rhs_f16_bytes, low, low_bytes) &&
+        !ds4_gpu_tensor_prefixes_overlap(
+            group_tmp, rhs_f16_bytes, heads, heads_bytes);
+
+    uint64_t out_b_inner = 0u;
+    id<MTLBuffer> out_b_buf = nil;
+    if (eligible) {
+        out_b_buf = ds4_gpu_wrap_model_range(
+            model_map, model_size, out_b_offset, out_b_bytes, &out_b_inner);
+        eligible = out_b_buf != nil && out_b_inner <= NSUIntegerMax;
+    }
+    if (!eligible) {
+        if (require) {
+            fprintf(stderr,
+                    "ds4: required Metal Q4 batch attention-output HC "
+                    "fusion was not selected\n");
+            return -1;
+        }
+        return 0;
     }
+
+    ds4_gpu_attn_out_hc_target target = {
+        .out_hc = out_hc,
+        .residual_hc = residual_hc,
+        .split = split,
+        .weight_buffer = out_b_buf,
+        .weight_offset = (NSUInteger)out_b_inner,
+    };
+    const int rc = ds4_gpu_attention_output_q4_K_batch_impl(
+        out, low, group_tmp, low_tmp, model_map, model_size,
+        out_a_offset, out_b_offset, out_b_type, group_dim, rank,
+        n_groups, out_dim, heads, n_tokens, &target);
+    return rc == 1 ? 1 : -1;
 }
 
 int ds4_gpu_attention_output_q8_batch_f16_tensor(
@@ -26710,7 +34183,9 @@ int ds4_gpu_attention_output_low_q4_K_slice_tensor(
         uint64_t                rank,
         uint32_t                group0,
         uint32_t                group_cnt,
-        const ds4_gpu_tensor *heads) {
+        const ds4_gpu_tensor *heads,
+        int                    resident_decode) {
+    (void)resident_decode;
     if (!g_initialized && !ds4_gpu_init()) return 0;
     if (!low || !heads || !model_map || group_dim == 0 || rank == 0 ||
         group_cnt == 0 || group_dim > UINT32_MAX || rank > UINT32_MAX) {
@@ -26934,6 +34409,7 @@ static int ds4_gpu_encode_cpy_f32_f16_1d(
     id<MTLComputePipelineState> pipeline = use_contiguous
         ? g_cpy_contig_f32_f16_pipeline
         : g_cpy_f32_f16_pipeline;
+    if (!pipeline) return 0;
     const NSUInteger work_items = use_contiguous
         ? ((NSUInteger)n + 3u) / 4u
         : (NSUInteger)n;
@@ -26941,6 +34417,7 @@ static int ds4_gpu_encode_cpy_f32_f16_1d(
     const NSUInteger groups = (work_items + nth - 1u) / nth;
 
     id<MTLComputeCommandEncoder> enc = ds4_gpu_compute_encoder(cb);
+    if (!enc) return 0;
     [enc setComputePipelineState:pipeline];
     if (use_contiguous) {
         [enc setBytes:&n length:sizeof(n) atIndex:0];
@@ -27260,9 +34737,9 @@ static int ds4_gpu_encode_flash_kv_stage_f16(
         bool                 shared_pad,
         bool                *did_fuse_pad) {
     if (did_fuse_pad) *did_fuse_pad = false;
-    if (!cb || !raw || !comp || !dst || raw_cap == 0 ||
+    if (!cb || !raw || !dst || raw_cap == 0 ||
         raw_start >= raw_cap || n_raw == 0 || n_raw > raw_cap ||
-        n_comp == 0 || head_dim == 0) {
+        (n_comp != 0 && !comp) || head_dim == 0) {
         return 0;
     }
 
@@ -27356,6 +34833,9 @@ static int ds4_gpu_encode_flash_kv_stage_f16(
                                               dst_offset)) {
         return 0;
     }
+    if (n_comp == 0) {
+        return 1;
+    }
     return ds4_gpu_encode_copy_to_f16_1d(
         cb,
         comp,
@@ -28241,8 +35721,11 @@ static int ds4_gpu_encode_flash_attention_prefill_static_mixed_heads_vec(
     }
 
     const uint32_t ncpsg = 32;
-    const uint32_t nwg = 32;
-    const uint32_t nsg = ds4_gpu_flash_attn_vec_nsg(n_keys, nwg, ncpsg);
+    bool direct_output =
+        ds4_gpu_flash_attn_small_prefill_direct(n_keys) &&
+        !ds4_gpu_tensor_prefixes_overlap(heads, q_bytes, q, q_bytes);
+    uint32_t nwg = direct_output ? 1u : 32u;
+    uint32_t nsg = ds4_gpu_flash_attn_vec_nsg(n_keys, nwg, ncpsg);
     const NSUInteger row_bytes = (NSUInteger)head_dim * sizeof(float);
     const NSUInteger row_bytes_f16 = (NSUInteger)head_dim * sizeof(uint16_t);
     const NSUInteger mask_bytes = (NSUInteger)n_keys * (NSUInteger)n_tokens * sizeof(uint16_t);
@@ -28252,8 +35735,9 @@ static int ds4_gpu_encode_flash_attention_prefill_static_mixed_heads_vec(
         ? (NSUInteger)ncpsg * (2u * row_bytes_f16 + (NSUInteger)n_tokens * sizeof(uint16_t))
         : 1u;
     const NSUInteger nrows = (NSUInteger)n_tokens * n_head;
-    const NSUInteger tmp_bytes = nrows * (NSUInteger)head_dim * (NSUInteger)nwg * sizeof(float) +
-                                 nrows * (2u * (NSUInteger)nwg) * sizeof(float);
+    NSUInteger tmp_bytes = direct_output ? 0u :
+        nrows * (NSUInteger)head_dim * (NSUInteger)nwg * sizeof(float) +
+        nrows * (2u * (NSUInteger)nwg) * sizeof(float);
 
     id<MTLBuffer> mask_buffer =
         ds4_gpu_new_transient_buffer(mask_bytes, "ds4_flash_attn_mask");
@@ -28266,10 +35750,11 @@ static int ds4_gpu_encode_flash_attention_prefill_static_mixed_heads_vec(
                                          &g_flash_attn_pad_bytes,
                                          pad_bytes,
                                          "ds4_flash_attn_pad") ||
-        !ds4_gpu_ensure_scratch_buffer(&g_flash_attn_tmp_buffer,
-                                         &g_flash_attn_tmp_bytes,
-                                         tmp_bytes,
-                                         "ds4_flash_attn_tmp")) {
+        (!direct_output &&
+         !ds4_gpu_ensure_scratch_buffer(&g_flash_attn_tmp_buffer,
+                                          &g_flash_attn_tmp_bytes,
+                                          tmp_bytes,
+                                          "ds4_flash_attn_tmp"))) {
         return 0;
     }
 
@@ -28349,16 +35834,39 @@ static int ds4_gpu_encode_flash_attention_prefill_static_mixed_heads_vec(
         if (!pad_pipeline) return 0;
     }
     id<MTLComputePipelineState> vec_pipeline =
-        ds4_gpu_get_flash_attn_vec_pipeline("kernel_flash_attn_ext_vec_f16_dk512_dv512",
-                                              true, true, false, false, has_kvpad,
-                                              false,
-                                              (int32_t)head_dim,
-                                              (int32_t)head_dim,
-                                              (int32_t)nsg,
-                                              (int32_t)nwg);
-    id<MTLComputePipelineState> reduce_pipeline =
-        ds4_gpu_get_flash_attn_reduce_pipeline((int32_t)head_dim, (int32_t)nwg);
-    if (!vec_pipeline || !reduce_pipeline) return 0;
+        direct_output &&
+        (g_test_flags &
+         DS4_GPU_TEST_FLASH_ATTN_SMALL_PREFILL_NWG1_FAILURE) != 0u
+            ? nil
+            : ds4_gpu_get_flash_attn_vec_pipeline(
+                "kernel_flash_attn_ext_vec_f16_dk512_dv512",
+                true, true, false, false, has_kvpad, false,
+                (int32_t)head_dim, (int32_t)head_dim,
+                (int32_t)nsg, (int32_t)nwg);
+    if (direct_output && !vec_pipeline) {
+        direct_output = false;
+        nwg = 32u;
+        nsg = ds4_gpu_flash_attn_vec_nsg(n_keys, nwg, ncpsg);
+        tmp_bytes =
+            nrows * (NSUInteger)head_dim * (NSUInteger)nwg * sizeof(float) +
+            nrows * (2u * (NSUInteger)nwg) * sizeof(float);
+        if (!ds4_gpu_ensure_scratch_buffer(&g_flash_attn_tmp_buffer,
+                                             &g_flash_attn_tmp_bytes,
+                                             tmp_bytes,
+                                             "ds4_flash_attn_tmp")) {
+            return 0;
+        }
+        vec_pipeline = ds4_gpu_get_flash_attn_vec_pipeline(
+            "kernel_flash_attn_ext_vec_f16_dk512_dv512",
+            true, true, false, false, has_kvpad, false,
+            (int32_t)head_dim, (int32_t)head_dim,
+            (int32_t)nsg, (int32_t)nwg);
+    }
+    id<MTLComputePipelineState> reduce_pipeline = direct_output ? nil :
+        ds4_gpu_get_flash_attn_reduce_pipeline(
+            (int32_t)head_dim, (int32_t)nwg);
+    if (!vec_pipeline || (!direct_output && !reduce_pipeline)) return 0;
+    g_test_last_flash_attn_prefill_nwg = nwg;
 
     if (has_kvpad) {
         ds4_gpu_flash_attn_pad_args pad_args = {
@@ -28441,25 +35949,29 @@ static int ds4_gpu_encode_flash_attention_prefill_static_mixed_heads_vec(
     [enc setBuffer:mask_buffer offset:0 atIndex:4];
     [enc setBuffer:sinks_buf offset:sinks_offset atIndex:5];
     [enc setBuffer:g_flash_attn_pad_buffer offset:0 atIndex:6];
-    [enc setBuffer:g_flash_attn_tmp_buffer offset:0 atIndex:7];
+    [enc setBuffer:direct_output ? headsbuf : g_flash_attn_tmp_buffer
+            offset:direct_output ? ds4_gpu_tensor_offset(heads) : 0u
+           atIndex:7];
     [enc setThreadgroupMemoryLength:shared_bytes atIndex:0];
     [enc dispatchThreadgroups:MTLSizeMake(n_tokens, n_head, nwg)
          threadsPerThreadgroup:MTLSizeMake(32, nsg, 1)];
     ds4_gpu_end_compute_encoder(cb, enc);
     DS4_METAL_PROFILE_FLASH_ATTN_STAGE("attention_vec");
 
-    ds4_gpu_flash_attn_reduce_args reduce_args = {
-        .nrows = (int32_t)nrows,
-    };
-    enc = ds4_gpu_compute_encoder(cb);
-    [enc setComputePipelineState:reduce_pipeline];
-    [enc setBytes:&reduce_args length:sizeof(reduce_args) atIndex:0];
-    [enc setBuffer:g_flash_attn_tmp_buffer offset:0 atIndex:1];
-    [enc setBuffer:headsbuf offset:ds4_gpu_tensor_offset(heads) atIndex:2];
-    [enc dispatchThreadgroups:MTLSizeMake(nrows, 1, 1)
-         threadsPerThreadgroup:MTLSizeMake(32u * nwg, 1, 1)];
-    ds4_gpu_end_compute_encoder(cb, enc);
-    DS4_METAL_PROFILE_FLASH_ATTN_STAGE("attention_reduce");
+    if (!direct_output) {
+        ds4_gpu_flash_attn_reduce_args reduce_args = {
+            .nrows = (int32_t)nrows,
+        };
+        enc = ds4_gpu_compute_encoder(cb);
+        [enc setComputePipelineState:reduce_pipeline];
+        [enc setBytes:&reduce_args length:sizeof(reduce_args) atIndex:0];
+        [enc setBuffer:g_flash_attn_tmp_buffer offset:0 atIndex:1];
+        [enc setBuffer:headsbuf offset:ds4_gpu_tensor_offset(heads) atIndex:2];
+        [enc dispatchThreadgroups:MTLSizeMake(nrows, 1, 1)
+             threadsPerThreadgroup:MTLSizeMake(32u * nwg, 1, 1)];
+        ds4_gpu_end_compute_encoder(cb, enc);
+        DS4_METAL_PROFILE_FLASH_ATTN_STAGE("attention_reduce");
+    }
 
 #undef DS4_METAL_PROFILE_FLASH_ATTN_STAGE
     return 1;
@@ -28855,8 +36367,11 @@ static int ds4_gpu_encode_flash_attention_prefill_raw_heads(
     }
 
     const uint32_t ncpsg = 32;
-    const uint32_t nwg = 32;
-    const uint32_t nsg = ds4_gpu_flash_attn_vec_nsg(n_tokens, nwg, ncpsg);
+    bool direct_output =
+        ds4_gpu_flash_attn_small_prefill_direct(n_tokens) &&
+        !ds4_gpu_tensor_prefixes_overlap(heads, q_bytes, q, q_bytes);
+    uint32_t nwg = direct_output ? 1u : 32u;
+    uint32_t nsg = ds4_gpu_flash_attn_vec_nsg(n_tokens, nwg, ncpsg);
     const NSUInteger row_bytes = (NSUInteger)head_dim * sizeof(float);
     const NSUInteger row_bytes_f16 = (NSUInteger)head_dim * sizeof(uint16_t);
     const NSUInteger mask_bytes = (NSUInteger)n_tokens * (NSUInteger)n_tokens * sizeof(uint16_t);
@@ -28865,8 +36380,9 @@ static int ds4_gpu_encode_flash_attention_prefill_raw_heads(
     const NSUInteger pad_bytes = 2u * (NSUInteger)ncpsg * row_bytes_f16 +
                                  (NSUInteger)ncpsg * (NSUInteger)n_tokens * sizeof(uint16_t);
     const NSUInteger nrows = (NSUInteger)n_tokens * n_head;
-    const NSUInteger tmp_bytes = nrows * (NSUInteger)head_dim * (NSUInteger)nwg * sizeof(float) +
-                                 nrows * (2u * (NSUInteger)nwg) * sizeof(float);
+    NSUInteger tmp_bytes = direct_output ? 0u :
+        nrows * (NSUInteger)head_dim * (NSUInteger)nwg * sizeof(float) +
+        nrows * (2u * (NSUInteger)nwg) * sizeof(float);
 
     id<MTLBuffer> mask_buffer =
         ds4_gpu_new_transient_buffer(mask_bytes, "ds4_flash_attn_mask");
@@ -28879,10 +36395,11 @@ static int ds4_gpu_encode_flash_attention_prefill_raw_heads(
                                          &g_flash_attn_kv_bytes,
                                          kv_f16_bytes,
                                          "ds4_flash_attn_kv_f16") ||
-        !ds4_gpu_ensure_scratch_buffer(&g_flash_attn_tmp_buffer,
-                                         &g_flash_attn_tmp_bytes,
-                                         tmp_bytes,
-                                         "ds4_flash_attn_tmp")) {
+        (!direct_output &&
+         !ds4_gpu_ensure_scratch_buffer(&g_flash_attn_tmp_buffer,
+                                          &g_flash_attn_tmp_bytes,
+                                          tmp_bytes,
+                                          "ds4_flash_attn_tmp"))) {
         return 0;
     }
 
@@ -28919,16 +36436,39 @@ static int ds4_gpu_encode_flash_attention_prefill_raw_heads(
         if (!pad_pipeline) return 0;
     }
     id<MTLComputePipelineState> vec_pipeline =
-        ds4_gpu_get_flash_attn_vec_pipeline("kernel_flash_attn_ext_vec_f16_dk512_dv512",
-                                              true, true, false, false, true,
-                                              false,
-                                              (int32_t)head_dim,
-                                              (int32_t)head_dim,
-                                              (int32_t)nsg,
-                                              (int32_t)nwg);
-    id<MTLComputePipelineState> reduce_pipeline =
-        ds4_gpu_get_flash_attn_reduce_pipeline((int32_t)head_dim, (int32_t)nwg);
-    if (!vec_pipeline || !reduce_pipeline) return 0;
+        direct_output &&
+        (g_test_flags &
+         DS4_GPU_TEST_FLASH_ATTN_SMALL_PREFILL_NWG1_FAILURE) != 0u
+            ? nil
+            : ds4_gpu_get_flash_attn_vec_pipeline(
+                "kernel_flash_attn_ext_vec_f16_dk512_dv512",
+                true, true, false, false, true, false,
+                (int32_t)head_dim, (int32_t)head_dim,
+                (int32_t)nsg, (int32_t)nwg);
+    if (direct_output && !vec_pipeline) {
+        direct_output = false;
+        nwg = 32u;
+        nsg = ds4_gpu_flash_attn_vec_nsg(n_tokens, nwg, ncpsg);
+        tmp_bytes =
+            nrows * (NSUInteger)head_dim * (NSUInteger)nwg * sizeof(float) +
+            nrows * (2u * (NSUInteger)nwg) * sizeof(float);
+        if (!ds4_gpu_ensure_scratch_buffer(&g_flash_attn_tmp_buffer,
+                                             &g_flash_attn_tmp_bytes,
+                                             tmp_bytes,
+                                             "ds4_flash_attn_tmp")) {
+            return 0;
+        }
+        vec_pipeline = ds4_gpu_get_flash_attn_vec_pipeline(
+            "kernel_flash_attn_ext_vec_f16_dk512_dv512",
+            true, true, false, false, true, false,
+            (int32_t)head_dim, (int32_t)head_dim,
+            (int32_t)nsg, (int32_t)nwg);
+    }
+    id<MTLComputePipelineState> reduce_pipeline = direct_output ? nil :
+        ds4_gpu_get_flash_attn_reduce_pipeline(
+            (int32_t)head_dim, (int32_t)nwg);
+    if (!vec_pipeline || (!direct_output && !reduce_pipeline)) return 0;
+    g_test_last_flash_attn_prefill_nwg = nwg;
 
     if (!ds4_gpu_encode_cpy_f32_f16_1d(cb,
                                          rawbuf,
@@ -29021,25 +36561,29 @@ static int ds4_gpu_encode_flash_attention_prefill_raw_heads(
     [enc setBuffer:mask_buffer offset:0 atIndex:4];
     [enc setBuffer:sinks_buf offset:sinks_offset atIndex:5];
     [enc setBuffer:g_flash_attn_pad_buffer offset:0 atIndex:6];
-    [enc setBuffer:g_flash_attn_tmp_buffer offset:0 atIndex:7];
+    [enc setBuffer:direct_output ? headsbuf : g_flash_attn_tmp_buffer
+            offset:direct_output ? ds4_gpu_tensor_offset(heads) : 0u
+           atIndex:7];
     [enc setThreadgroupMemoryLength:shared_bytes atIndex:0];
     [enc dispatchThreadgroups:MTLSizeMake(n_tokens, n_head, nwg)
          threadsPerThreadgroup:MTLSizeMake(32, nsg, 1)];
     ds4_gpu_end_compute_encoder(cb, enc);
     DS4_METAL_PROFILE_FLASH_ATTN_STAGE("attention_vec");
 
-    ds4_gpu_flash_attn_reduce_args reduce_args = {
-        .nrows = (int32_t)nrows,
-    };
-    enc = ds4_gpu_compute_encoder(cb);
-    [enc setComputePipelineState:reduce_pipeline];
-    [enc setBytes:&reduce_args length:sizeof(reduce_args) atIndex:0];
-    [enc setBuffer:g_flash_attn_tmp_buffer offset:0 atIndex:1];
-    [enc setBuffer:headsbuf offset:ds4_gpu_tensor_offset(heads) atIndex:2];
-    [enc dispatchThreadgroups:MTLSizeMake(nrows, 1, 1)
-         threadsPerThreadgroup:MTLSizeMake(32u * nwg, 1, 1)];
-    ds4_gpu_end_compute_encoder(cb, enc);
-    DS4_METAL_PROFILE_FLASH_ATTN_STAGE("attention_reduce");
+    if (!direct_output) {
+        ds4_gpu_flash_attn_reduce_args reduce_args = {
+            .nrows = (int32_t)nrows,
+        };
+        enc = ds4_gpu_compute_encoder(cb);
+        [enc setComputePipelineState:reduce_pipeline];
+        [enc setBytes:&reduce_args length:sizeof(reduce_args) atIndex:0];
+        [enc setBuffer:g_flash_attn_tmp_buffer offset:0 atIndex:1];
+        [enc setBuffer:headsbuf offset:ds4_gpu_tensor_offset(heads) atIndex:2];
+        [enc dispatchThreadgroups:MTLSizeMake(nrows, 1, 1)
+             threadsPerThreadgroup:MTLSizeMake(32u * nwg, 1, 1)];
+        ds4_gpu_end_compute_encoder(cb, enc);
+        DS4_METAL_PROFILE_FLASH_ATTN_STAGE("attention_reduce");
+    }
 
 #undef DS4_METAL_PROFILE_FLASH_ATTN_STAGE
     return 1;
@@ -29125,7 +36669,9 @@ static int ds4_gpu_encode_flash_attention_gathered_heads(
         ds4_gpu_ported_m5_decode_feature_enabled(
             "DS4_METAL_DISABLE_PRE_M5_FLASH_ATTN_PACKED32_REDUCE",
             NULL) &&
-        !g_quality_mode && use_mask == 0u && comp_kv_f16 != 0u && n_comp != 0u &&
+        !g_quality_mode && use_mask == 0u && comp_kv_f16 != 0u &&
+        (n_comp != 0u ||
+         getenv("DS4_METAL_DISABLE_DECODE_RAW_PACKED32") == NULL) &&
         n_head == 64u && head_dim == 512u && nsg == 1u && nwg == 32u &&
         n_keys <= 1024u && g_decode_attn_rope_fuse != 0 &&
         g_decode_attn_rope_args.head_dim == 512 &&
@@ -30754,7 +38300,12 @@ int ds4_gpu_attention_decode_heads_tensor(
         id<MTLBuffer> sinks_buf = ds4_gpu_wrap_model_range(model_map, model_size, sinks_offset, sink_bytes, &sinks_inner);
         if (!sinks_buf) return 0;
 
-        if (n_comp == 0) {
+        /* Raw-only layers historically used a separate five-dispatch path.
+         * The gathered path handles n_comp == 0 with the same packed
+         * attention kernel and reduction topology, but fewer dispatches. */
+        if (n_comp == 0 &&
+            (use_mask != 0 ||
+             getenv("DS4_METAL_DISABLE_DECODE_RAW_GATHERED_ATTN") != NULL)) {
             /* The raw-only path stages through the same kernel when it
              * stages at all; a pending deferred kv task that nothing
              * consumes must run before the attention reads the cache. */
@@ -31428,6 +38979,8 @@ static int ds4_gpu_routed_mm_mpp_mask(void) {
 
 static id<MTLComputePipelineState> ds4_gpu_routed_mm_addr_pipeline(uint32_t type) {
     switch (type) {
+    case DS4_METAL_TENSOR_IQ2_XXS:
+        return ds4_gpu_get_mul_mm_id_pipeline("kernel_mul_mm_id_addr_iq2_xxs_f32", false);
     case DS4_METAL_TENSOR_Q2_K:
         return ds4_gpu_get_mul_mm_id_pipeline("kernel_mul_mm_id_addr_q2_K_f32", false);
     case DS4_METAL_TENSOR_Q4_K:
@@ -31439,6 +38992,20 @@ static int ds4_gpu_routed_mm_mpp_mask(void) {
     }
 }
 
+static id<MTLComputePipelineState>
+ds4_gpu_routed_mm_addr_tail_cull_pipeline(uint32_t type) {
+    switch (type) {
+    case DS4_METAL_TENSOR_IQ2_XXS:
+        return ds4_gpu_get_mul_mm_id_pipeline(
+            "kernel_mul_mm_id_addr_iq2_xxs_f32_tail_cull", false);
+    case DS4_METAL_TENSOR_Q2_K:
+        return ds4_gpu_get_mul_mm_id_pipeline(
+            "kernel_mul_mm_id_addr_q2_K_f32_tail_cull", false);
+    default:
+        return nil;
+    }
+}
+
 static id<MTLComputePipelineState> ds4_gpu_routed_mm_f16_rhs_pipeline(uint32_t type) {
     switch (type) {
     case DS4_METAL_TENSOR_Q8_0:
@@ -32325,7 +39892,8 @@ static int ds4_gpu_encode_mul_mv_addr_iq2_pair_swiglu(
     for (uint32_t i = 0; i < n_entries; i++) {
         if (!entries[i] || !entries[i]->gate_buffer || !entries[i]->up_buffer) return 0;
     }
-    if (!ds4_gpu_stream_expert_cache_mark_entries_inflight(entries,
+    if (n_entries != 0 &&
+        !ds4_gpu_stream_expert_cache_mark_entries_inflight(entries,
                                                            n_entries,
                                                            0)) {
         return 0;
@@ -32449,7 +40017,8 @@ static int ds4_gpu_encode_mul_mv_addr_q2_sum6(
     for (uint32_t i = 0; i < n_entries; i++) {
         if (!entries[i] || !entries[i]->down_buffer) return 0;
     }
-    if (!ds4_gpu_stream_expert_cache_mark_entries_inflight(entries,
+    if (n_entries != 0 &&
+        !ds4_gpu_stream_expert_cache_mark_entries_inflight(entries,
                                                            n_entries,
                                                            0)) {
         return 0;
@@ -32843,6 +40412,60 @@ static int ds4_gpu_encode_mul_mm_id(
                                              dst_off);
 }
 
+typedef struct ds4_gpu_mul_mm_id_map_layout {
+    NSUInteger tpe_bytes;
+    NSUInteger hids_bytes;
+    NSUInteger work_offset;
+    NSUInteger total_bytes;
+    uint64_t work_cap;
+} ds4_gpu_mul_mm_id_map_layout;
+
+/* The map stores uint2(count, packed-route-base) for every model expert, then
+ * exactly ne20 * ne21 token/slot route IDs.  Packed ranges are required for
+ * duplicate expert IDs: a fixed ne21 slice cannot represent the same expert
+ * appearing in more than one top-k slot of a token. */
+static int ds4_gpu_make_mul_mm_id_map_layout(
+        const ds4_gpu_mul_mm_id_args *args,
+        ds4_gpu_mul_mm_id_map_layout *layout) {
+    if (!args || !layout ||
+        args->ne02 <= 0 || args->ne20 <= 0 || args->ne21 <= 0) {
+        return 0;
+    }
+
+    const uint64_t ne02 = (uint32_t)args->ne02;
+    const uint64_t pair_rows =
+        (uint64_t)(uint32_t)args->ne20 * (uint32_t)args->ne21;
+    if (ne02 > (uint64_t)NSUIntegerMax / (2u * sizeof(uint32_t)) ||
+        pair_rows > (uint64_t)NSUIntegerMax / sizeof(int32_t) ||
+        ne02 > (UINT64_MAX - pair_rows - 31u) / 31u) {
+        return 0;
+    }
+
+    const NSUInteger tpe_bytes =
+        (NSUInteger)ne02 * 2u * sizeof(uint32_t);
+    const NSUInteger hids_bytes =
+        (NSUInteger)pair_rows * sizeof(int32_t);
+    if (tpe_bytes > NSUIntegerMax - hids_bytes - 7u) return 0;
+    const NSUInteger work_offset = (tpe_bytes + hids_bytes + 7u) & ~7u;
+    const uint64_t work_cap = (pair_rows + 31u * ne02 + 31u) / 32u;
+    const NSUInteger work_item_bytes = 2u * sizeof(uint32_t);
+    if (work_cap > (uint64_t)NSUIntegerMax ||
+        (NSUInteger)work_cap >
+            (NSUIntegerMax - work_offset - 8u) / work_item_bytes) {
+        return 0;
+    }
+
+    *layout = (ds4_gpu_mul_mm_id_map_layout) {
+        .tpe_bytes = tpe_bytes,
+        .hids_bytes = hids_bytes,
+        .work_offset = work_offset,
+        .total_bytes = work_offset + 8u +
+                       (NSUInteger)work_cap * work_item_bytes,
+        .work_cap = work_cap,
+    };
+    return 1;
+}
+
 static int ds4_gpu_encode_mul_mm_id_map(
         id<MTLCommandBuffer>        cb,
         id<MTLComputePipelineState> map_pipeline,
@@ -32855,26 +40478,11 @@ static int ds4_gpu_encode_mul_mm_id_map(
         return 0;
     }
 
-    const NSUInteger tpe_bytes = (NSUInteger)mm_args->ne02 * sizeof(int32_t);
-    const NSUInteger hids_bytes =
-        (NSUInteger)mm_args->ne02 * (NSUInteger)mm_args->ne21 * sizeof(int32_t);
-    if (tpe_bytes > NSUIntegerMax - hids_bytes) return 0;
-    const NSUInteger work_offset = (tpe_bytes + hids_bytes + 7u) & ~7u;
-    const uint64_t pair_rows =
-        (uint64_t)(uint32_t)mm_args->ne20 * (uint32_t)mm_args->ne21;
-    const uint64_t work_cap =
-        (pair_rows + 31u * (uint32_t)mm_args->ne02 + 31u) / 32u;
-    const NSUInteger work_item_bytes = 2u * sizeof(uint32_t);
-    if (work_cap > (NSUIntegerMax - 8u) / work_item_bytes ||
-        work_offset > NSUIntegerMax - 8u -
-                          (NSUInteger)work_cap * work_item_bytes) {
-        return 0;
-    }
-    const NSUInteger total_bytes =
-        work_offset + 8u + (NSUInteger)work_cap * work_item_bytes;
+    ds4_gpu_mul_mm_id_map_layout layout;
+    if (!ds4_gpu_make_mul_mm_id_map_layout(mm_args, &layout)) return 0;
     if (!ds4_gpu_ensure_scratch_buffer(&g_moe_id_map_buffer,
                                          &g_moe_id_map_bytes,
-                                         total_bytes,
+                                         layout.total_bytes,
                                          "ds4_moe_id_map")) {
         return 0;
     }
@@ -32884,9 +40492,16 @@ static int ds4_gpu_encode_mul_mm_id_map(
     [enc setBytes:map_args length:sizeof(*map_args) atIndex:0];
     [enc setBuffer:ids offset:ids_off atIndex:1];
     [enc setBuffer:g_moe_id_map_buffer offset:0 atIndex:2];
-    [enc setBuffer:g_moe_id_map_buffer offset:tpe_bytes atIndex:3];
-    [enc setBuffer:g_moe_id_map_buffer offset:work_offset atIndex:4];
-    [enc setThreadgroupMemoryLength:(NSUInteger)mm_args->ne02 * (NSUInteger)mm_args->ne20 * sizeof(uint16_t) atIndex:0];
+    [enc setBuffer:g_moe_id_map_buffer offset:layout.tpe_bytes atIndex:3];
+    [enc setBuffer:g_moe_id_map_buffer offset:layout.work_offset atIndex:4];
+    const NSUInteger staging_bytes =
+        (NSUInteger)mm_args->ne02 * (NSUInteger)mm_args->ne20 *
+        sizeof(uint16_t);
+    const NSUInteger scatter_bytes =
+        (NSUInteger)mm_args->ne02 * 2u * sizeof(uint32_t);
+    [enc setThreadgroupMemoryLength:
+        staging_bytes > scatter_bytes ? staging_bytes : scatter_bytes
+        atIndex:0];
     [enc dispatchThreadgroups:MTLSizeMake(1, 1, 1)
          threadsPerThreadgroup:MTLSizeMake((NSUInteger)mm_args->ne02, 1, 1)];
     ds4_gpu_end_compute_encoder(cb, enc);
@@ -32919,23 +40534,9 @@ static int ds4_gpu_encode_mul_mm_id_mapped_tile(
         getenv("DS4_METAL_MOE_MM_ID_USE_RESOURCES") != NULL &&
         getenv("DS4_METAL_DISABLE_MOE_MM_ID_USE_RESOURCES") == NULL;
 
-    const NSUInteger tpe_bytes = (NSUInteger)mm_args->ne02 * sizeof(int32_t);
-    const NSUInteger hids_bytes = (NSUInteger)mm_args->ne02 * (NSUInteger)mm_args->ne21 * sizeof(int32_t);
-    if (tpe_bytes > NSUIntegerMax - hids_bytes) {
-        return 0;
-    }
-    const NSUInteger work_offset = (tpe_bytes + hids_bytes + 7u) & ~7u;
-    const uint64_t pair_rows =
-        (uint64_t)(uint32_t)mm_args->ne20 * (uint32_t)mm_args->ne21;
-    const uint64_t work_cap =
-        (pair_rows + 31u * (uint32_t)mm_args->ne02 + 31u) / 32u;
-    const NSUInteger work_item_bytes = 2u * sizeof(uint32_t);
-    if (work_cap > NSUIntegerMax ||
-        work_offset > NSUIntegerMax - 8u ||
-        (NSUInteger)work_cap >
-            (NSUIntegerMax - work_offset - 8u) / work_item_bytes ||
-        g_moe_id_map_bytes <
-            work_offset + 8u + (NSUInteger)work_cap * work_item_bytes) {
+    ds4_gpu_mul_mm_id_map_layout layout;
+    if (!ds4_gpu_make_mul_mm_id_map_layout(mm_args, &layout) ||
+        g_moe_id_map_bytes < layout.total_bytes) {
         return 0;
     }
 
@@ -32945,14 +40546,14 @@ static int ds4_gpu_encode_mul_mm_id_mapped_tile(
     [enc setBuffer:src0 offset:src0_off atIndex:1];
     [enc setBuffer:src1 offset:src1_off atIndex:2];
     [enc setBuffer:g_moe_id_map_buffer offset:0 atIndex:3];
-    [enc setBuffer:g_moe_id_map_buffer offset:tpe_bytes atIndex:4];
+    [enc setBuffer:g_moe_id_map_buffer offset:layout.tpe_bytes atIndex:4];
     [enc setBuffer:dst offset:dst_off atIndex:5];
-    [enc setBuffer:g_moe_id_map_buffer offset:work_offset atIndex:6];
+    [enc setBuffer:g_moe_id_map_buffer offset:layout.work_offset atIndex:6];
     if (use_resource_hints) {
         [enc useResource:src0 usage:MTLResourceUsageRead];
     }
     [enc setThreadgroupMemoryLength:threadgroup_bytes atIndex:0];
-    [enc dispatchThreadgroups:MTLSizeMake((NSUInteger)work_cap,
+    [enc dispatchThreadgroups:MTLSizeMake((NSUInteger)layout.work_cap,
                                           ((NSUInteger)mm_args->ne0 + 63u) / 64u,
                                           1)
          threadsPerThreadgroup:MTLSizeMake(128, 1, 1)];
@@ -32981,12 +40582,9 @@ static int ds4_gpu_encode_mul_mm_id_addr_mapped_tile(
         return 0;
     }
 
-    const NSUInteger tile_n = 32u;
-    const NSUInteger tpe_bytes = (NSUInteger)mm_args->ne02 * sizeof(int32_t);
-    const NSUInteger hids_bytes =
-        (NSUInteger)mm_args->ne02 * (NSUInteger)mm_args->ne21 * sizeof(int32_t);
-    if (tpe_bytes > NSUIntegerMax - hids_bytes ||
-        g_moe_id_map_bytes < tpe_bytes + hids_bytes) {
+    ds4_gpu_mul_mm_id_map_layout layout;
+    if (!ds4_gpu_make_mul_mm_id_map_layout(mm_args, &layout) ||
+        g_moe_id_map_bytes < layout.total_bytes) {
         return 0;
     }
 
@@ -32996,8 +40594,9 @@ static int ds4_gpu_encode_mul_mm_id_addr_mapped_tile(
     [enc setBuffer:src0_addrs offset:0 atIndex:1];
     [enc setBuffer:src1 offset:src1_off atIndex:2];
     [enc setBuffer:g_moe_id_map_buffer offset:0 atIndex:3];
-    [enc setBuffer:g_moe_id_map_buffer offset:tpe_bytes atIndex:4];
+    [enc setBuffer:g_moe_id_map_buffer offset:layout.tpe_bytes atIndex:4];
     [enc setBuffer:dst offset:dst_off atIndex:5];
+    [enc setBuffer:g_moe_id_map_buffer offset:layout.work_offset atIndex:6];
     [enc useResource:src0_addrs usage:MTLResourceUsageRead];
     for (uint32_t i = 0; resources && i < resource_count; i++) {
         ds4_gpu_stream_expert_cache_entry *entry = resources[i];
@@ -33012,9 +40611,9 @@ static int ds4_gpu_encode_mul_mm_id_addr_mapped_tile(
         [enc useResource:overflow_resource usage:MTLResourceUsageRead];
     }
     [enc setThreadgroupMemoryLength:threadgroup_bytes atIndex:0];
-    [enc dispatchThreadgroups:MTLSizeMake(((NSUInteger)mm_args->ne21 + tile_n - 1u) / tile_n,
+    [enc dispatchThreadgroups:MTLSizeMake((NSUInteger)layout.work_cap,
                                           ((NSUInteger)mm_args->ne0 + 63u) / 64u,
-                                          (NSUInteger)mm_args->ne02)
+                                          1)
          threadsPerThreadgroup:MTLSizeMake(128, 1, 1)];
     ds4_gpu_end_compute_encoder(cb, enc);
     return 1;
@@ -33044,23 +40643,9 @@ static int ds4_gpu_encode_mul_mm_id_iq2_pair_swiglu_f16(
         return 0;
     }
 
-    const NSUInteger tpe_bytes = (NSUInteger)mm_args->ne02 * sizeof(int32_t);
-    const NSUInteger hids_bytes = (NSUInteger)mm_args->ne02 * (NSUInteger)mm_args->ne21 * sizeof(int32_t);
-    if (tpe_bytes > NSUIntegerMax - hids_bytes) {
-        return 0;
-    }
-    const NSUInteger work_offset = (tpe_bytes + hids_bytes + 7u) & ~7u;
-    const uint64_t pair_rows =
-        (uint64_t)(uint32_t)mm_args->ne20 * (uint32_t)mm_args->ne21;
-    const uint64_t work_cap =
-        (pair_rows + 31u * (uint32_t)mm_args->ne02 + 31u) / 32u;
-    const NSUInteger work_item_bytes = 2u * sizeof(uint32_t);
-    if (work_cap > NSUIntegerMax ||
-        work_offset > NSUIntegerMax - 8u ||
-        (NSUInteger)work_cap >
-            (NSUIntegerMax - work_offset - 8u) / work_item_bytes ||
-        g_moe_id_map_bytes <
-            work_offset + 8u + (NSUInteger)work_cap * work_item_bytes) {
+    ds4_gpu_mul_mm_id_map_layout layout;
+    if (!ds4_gpu_make_mul_mm_id_map_layout(mm_args, &layout) ||
+        g_moe_id_map_bytes < layout.total_bytes) {
         return 0;
     }
 
@@ -33072,13 +40657,13 @@ static int ds4_gpu_encode_mul_mm_id_iq2_pair_swiglu_f16(
     [enc setBuffer:up_src0 offset:up_src0_off atIndex:3];
     [enc setBuffer:src1 offset:src1_off atIndex:4];
     [enc setBuffer:g_moe_id_map_buffer offset:0 atIndex:5];
-    [enc setBuffer:g_moe_id_map_buffer offset:tpe_bytes atIndex:6];
+    [enc setBuffer:g_moe_id_map_buffer offset:layout.tpe_bytes atIndex:6];
     [enc setBuffer:mid offset:mid_off atIndex:7];
     [enc setBuffer:weights offset:weights_off atIndex:8];
-    [enc setBuffer:g_moe_id_map_buffer offset:work_offset atIndex:9];
+    [enc setBuffer:g_moe_id_map_buffer offset:layout.work_offset atIndex:9];
     const NSUInteger tile_m = compact_tile ? 32u : 64u;
     [enc setThreadgroupMemoryLength:compact_tile ? 8192u : 16384u atIndex:0];
-    [enc dispatchThreadgroups:MTLSizeMake((NSUInteger)work_cap,
+    [enc dispatchThreadgroups:MTLSizeMake((NSUInteger)layout.work_cap,
                                           ((NSUInteger)mm_args->ne0 + tile_m - 1u) / tile_m,
                                           1)
          threadsPerThreadgroup:MTLSizeMake(compact_tile ? 64u : 128u, 1, 1)];
@@ -33108,6 +40693,39 @@ static int ds4_gpu_encode_mul_mm_id_mapped(
                                                   8192u);
 }
 
+static int ds4_gpu_encode_attn_out_low_q4_direct(
+        id<MTLCommandBuffer>           cb,
+        id<MTLComputePipelineState>    pipeline,
+        const ds4_gpu_mul_mm_id_args *mm_args,
+        id<MTLBuffer>                  src0,
+        NSUInteger                     src0_off,
+        id<MTLBuffer>                  src1,
+        NSUInteger                     src1_off,
+        id<MTLBuffer>                  dst,
+        NSUInteger                     dst_off) {
+    if (!cb || !pipeline || !mm_args || !src0 || !src1 || !dst ||
+        mm_args->ne00 <= 0 || mm_args->ne0 <= 0 ||
+        mm_args->ne02 <= 0 || mm_args->ne1 <= 0 || mm_args->ne21 <= 0) {
+        return 0;
+    }
+
+    id<MTLComputeCommandEncoder> enc = ds4_gpu_compute_encoder(cb);
+    if (!enc) return 0;
+    [enc setComputePipelineState:pipeline];
+    [enc setBytes:mm_args length:sizeof(*mm_args) atIndex:0];
+    [enc setBuffer:src0 offset:src0_off atIndex:1];
+    [enc setBuffer:src1 offset:src1_off atIndex:2];
+    [enc setBuffer:dst offset:dst_off atIndex:3];
+    [enc setThreadgroupMemoryLength:8192u atIndex:0];
+    [enc dispatchThreadgroups:MTLSizeMake(
+            ((NSUInteger)mm_args->ne21 + 31u) / 32u,
+            ((NSUInteger)mm_args->ne0 + 63u) / 64u,
+            (NSUInteger)mm_args->ne02)
+         threadsPerThreadgroup:MTLSizeMake(128u, 1u, 1u)];
+    ds4_gpu_end_compute_encoder(cb, enc);
+    return 1;
+}
+
 static int ds4_gpu_encode_attn_out_low_mpp(
         id<MTLCommandBuffer>           cb,
         id<MTLComputePipelineState>    pipeline,
@@ -33621,7 +41239,6 @@ static int ds4_gpu_encode_router_select(
             (!pre_m5_device ||
              getenv("DS4_METAL_DISABLE_PRE_M5_ROUTER_SIMD_WEIGHTS_FUSION") == NULL);
         const bool use_pre_m5_transform_finalize_fusion_default =
-            !g_ssd_streaming_mode &&
             (pre_m5_device || ds4_gpu_device_name_contains("M5")) &&
             getenv("DS4_METAL_DISABLE_PRE_M5_ROUTER_TRANSFORM_FINALIZE_FUSION") == NULL;
         const bool use_transform_finalize_fusion =
@@ -35747,15 +43364,12 @@ static int ds4_gpu_glm_indexer_scores_batch_grouped_tensor(
         }
 
         const bool force_scalar = g_quality_mode;
-        const bool use_tiled_f32 = false;
         const bool use_tiled = !force_scalar && n_tokens >= 8u &&
                                n_head == 32u && head_dim == 128u;
         id<MTLComputePipelineState> pipeline =
             use_tiled
-                ? ds4_gpu_hot_pipeline(use_tiled_f32 ? g_glm_indexer_scores_tiled_f32_pipeline
-                                                     : g_glm_indexer_scores_tiled_pipeline,
-                                       use_tiled_f32 ? "kernel_glm_indexer_scores_tiled_f32"
-                                                     : "kernel_glm_indexer_scores_tiled")
+                ? ds4_gpu_hot_pipeline(g_glm_indexer_scores_tiled_pipeline,
+                                       "kernel_glm_indexer_scores_tiled")
                 : ds4_gpu_hot_pipeline(g_glm_indexer_scores_batch_pipeline,
                                        "kernel_glm_indexer_scores_batch");
         if (!pipeline) return 0;
@@ -35792,13 +43406,8 @@ static int ds4_gpu_glm_indexer_scores_batch_grouped_tensor(
             const NSUInteger q_shared = 8u * 128u;
             const NSUInteger k_shared = 32u * 128u;
             const NSUInteger dot_shared = 8u * 32u;
-            if (use_tiled_f32) {
-                [enc setThreadgroupMemoryLength:(q_shared + k_shared + dot_shared) *
-                                                sizeof(float) atIndex:0];
-            } else {
-                [enc setThreadgroupMemoryLength:(q_shared + k_shared) * sizeof(uint16_t) +
-                                                dot_shared * sizeof(float) atIndex:0];
-            }
+            [enc setThreadgroupMemoryLength:(q_shared + k_shared) * sizeof(uint16_t) +
+                                            dot_shared * sizeof(float) atIndex:0];
             [enc dispatchThreadgroups:MTLSizeMake(((NSUInteger)n_rows + 31u) / 32u,
                                                   ((NSUInteger)n_tokens + 7u) / 8u,
                                                   1)
@@ -38361,11 +45970,6 @@ static bool ds4_gpu_glm_routed_moe_batch_grouped_available(
            ds4_gpu_routed_mm_f16_rhs_pipeline(down_type) != nil;
 }
 
-static bool ds4_gpu_glm_grouped_moe_layer_enabled(uint32_t layer_index) {
-    (void)layer_index;
-    return true;
-}
-
 static int ds4_gpu_glm_routed_moe_batch_grouped_tensor(
         ds4_gpu_tensor       *out,
         ds4_gpu_tensor       *mid,
@@ -38391,7 +45995,6 @@ static int ds4_gpu_glm_routed_moe_batch_grouped_tensor(
         uint32_t                n_total_expert,
         uint32_t                n_expert,
         float                   swiglu_clamp,
-        uint32_t                layer_index,
         const ds4_gpu_tensor *x,
         uint32_t                n_tokens) {
     if (!ds4_gpu_glm_routed_moe_batch_grouped_available(gate_type,
@@ -38416,14 +46019,13 @@ static int ds4_gpu_glm_routed_moe_batch_grouped_tensor(
         return 0;
     }
 
-    const bool mid_f16 = true;
     const NSUInteger mm_id_threadgroup_bytes = 8192u;
     const uint64_t compact_mid_values = (uint64_t)pair_rows * expert_mid_dim;
     const uint64_t down_values = (uint64_t)pair_rows * out_dim;
     const uint64_t x_values = (uint64_t)n_tokens * expert_in_dim;
     const uint64_t out_values = (uint64_t)n_tokens * out_dim;
     if (compact_mid_values > UINT64_MAX / sizeof(float) ||
-        compact_mid_values > UINT64_MAX / (mid_f16 ? sizeof(uint16_t) : sizeof(float)) ||
+        compact_mid_values > UINT64_MAX / sizeof(uint16_t) ||
         down_values > UINT64_MAX / sizeof(float) ||
         x_values > UINT64_MAX / sizeof(float) ||
         out_values > UINT64_MAX / sizeof(float)) {
@@ -38431,7 +46033,7 @@ static int ds4_gpu_glm_routed_moe_batch_grouped_tensor(
     }
 
     const uint64_t gate_scratch_bytes = compact_mid_values * sizeof(float);
-    const uint64_t mid_bytes = compact_mid_values * (mid_f16 ? sizeof(uint16_t) : sizeof(float));
+    const uint64_t mid_bytes = compact_mid_values * sizeof(uint16_t);
     const uint64_t down_scratch_bytes = down_values * sizeof(float);
     const uint64_t x_bytes = x_values * sizeof(float);
     const uint64_t out_bytes = out_values * sizeof(float);
@@ -38522,7 +46124,7 @@ static int ds4_gpu_glm_routed_moe_batch_grouped_tensor(
             ds4_gpu_make_mul_mm_id_args_src1_size(expert_mid_dim, out_dim, n_total_expert,
                                                     down_row_bytes, down_expert_bytes,
                                                     n_expert, n_expert, n_tokens,
-                                                    mid_f16 ? sizeof(uint16_t) : sizeof(float));
+                                                    sizeof(uint16_t));
         gate_args.tp_rank = g_tp_split_rank;
         gate_args.tp_world = g_tp_split_world;
         gate_args.tp_expert_base = (int32_t)first_expert;
@@ -38536,57 +46138,13 @@ static int ds4_gpu_glm_routed_moe_batch_grouped_tensor(
         id<MTLCommandBuffer> cb = ds4_gpu_command_buffer(&owned);
         if (!cb) return 0;
 
-        const bool glm_moe_stage_profile = false;
-        const char *glm_moe_stage_filter = NULL;
-        double glm_moe_stage_t0 = 0.0;
-        if (glm_moe_stage_profile) {
-            if (ds4_gpu_end_commands() == 0 || ds4_gpu_begin_commands() == 0) {
-                return 0;
-            }
-            cb = ds4_gpu_command_buffer(&owned);
-            if (!cb) return 0;
-            glm_moe_stage_t0 = ds4_gpu_now_ms();
-        }
-
         int ok = 1;
-#define DS4_METAL_PROFILE_GLM_GROUPED_MOE_STAGE(name) do { \
-            if (ok && glm_moe_stage_profile) { \
-                if (ds4_gpu_end_commands() == 0) { \
-                    ok = 0; \
-                } else { \
-                    const char *stage_name = (name); \
-                    const double now_ms = ds4_gpu_now_ms(); \
-                    const int print_stage = \
-                        !glm_moe_stage_filter || !glm_moe_stage_filter[0] || \
-                        strstr(stage_name, glm_moe_stage_filter) != NULL; \
-                    if (print_stage) { \
-                        fprintf(stderr, \
-                                "ds4: Metal GLM grouped routed MoE stage layer=%u tokens=%u pairs=%u experts=%u " \
-                                "gate=%s down=%s mid=%s %s=%.3f ms\n", \
-                                layer_index, n_tokens, pair_rows, n_expert, \
-                                ds4_gpu_metal_tensor_type_name(gate_type), \
-                                ds4_gpu_metal_tensor_type_name(down_type), \
-                                mid_f16 ? "f16" : "f32", \
-                                stage_name, now_ms - glm_moe_stage_t0); \
-                    } \
-                    glm_moe_stage_t0 = now_ms; \
-                    if (ds4_gpu_begin_commands() == 0) { \
-                        ok = 0; \
-                    } else { \
-                        cb = ds4_gpu_command_buffer(&owned); \
-                        if (!cb) ok = 0; \
-                    } \
-                } \
-            } \
-        } while (0)
-
         ok = ds4_gpu_encode_mul_mm_id_map(cb,
                                            map_pipeline,
                                            &map_args,
                                            &gate_args,
                                            selectedbuf,
                                            ds4_gpu_tensor_offset(selected));
-        DS4_METAL_PROFILE_GLM_GROUPED_MOE_STAGE("map");
         if (ok) {
             ok = ds4_gpu_encode_mul_mm_id_mapped_tile(cb,
                                                        gate_pipeline,
@@ -38599,7 +46157,6 @@ static int ds4_gpu_glm_routed_moe_batch_grouped_tensor(
                                                        0,
                                                        mm_id_threadgroup_bytes);
         }
-        DS4_METAL_PROFILE_GLM_GROUPED_MOE_STAGE("gate");
         if (ok) {
             ok = ds4_gpu_encode_mul_mm_id_mapped_tile(cb,
                                                        up_pipeline,
@@ -38612,7 +46169,6 @@ static int ds4_gpu_glm_routed_moe_batch_grouped_tensor(
                                                        (NSUInteger)gate_scratch_bytes,
                                                        mm_id_threadgroup_bytes);
         }
-        DS4_METAL_PROFILE_GLM_GROUPED_MOE_STAGE("up");
         if (ok) {
             ok = ds4_gpu_encode_moe_swiglu_weight(cb,
                                                    g_moe_gate_scratch_buffer,
@@ -38626,10 +46182,8 @@ static int ds4_gpu_glm_routed_moe_batch_grouped_tensor(
                                                    expert_mid_dim,
                                                    pair_rows,
                                                    swiglu_clamp,
-                                                   mid_f16);
+                                                   true);
         }
-        DS4_METAL_PROFILE_GLM_GROUPED_MOE_STAGE("activation_weight");
-
         id<MTLBuffer> down_dst = n_expert == 1 ? outbuf : g_moe_down_scratch_buffer;
         NSUInteger down_dst_off = n_expert == 1 ? ds4_gpu_tensor_offset(out) : 0;
         if (ok) {
@@ -38644,7 +46198,6 @@ static int ds4_gpu_glm_routed_moe_batch_grouped_tensor(
                                                        down_dst_off,
                                                        mm_id_threadgroup_bytes);
         }
-        DS4_METAL_PROFILE_GLM_GROUPED_MOE_STAGE("down");
         if (ok && n_expert > 1) {
             ok = ds4_gpu_encode_moe_sum_experts(cb,
                                                  down_dst,
@@ -38655,13 +46208,11 @@ static int ds4_gpu_glm_routed_moe_batch_grouped_tensor(
                                                  n_expert,
                                                  n_tokens);
         }
-        DS4_METAL_PROFILE_GLM_GROUPED_MOE_STAGE("sum");
         if (!ok) return 0;
 
         if (!ds4_gpu_finish_command_buffer(cb, owned, "GLM grouped routed batch MoE")) {
             return 0;
         }
-#undef DS4_METAL_PROFILE_GLM_GROUPED_MOE_STAGE
     }
 
     return 1;
@@ -38693,7 +46244,6 @@ static int ds4_gpu_glm_routed_moe_batch_grouped_addr_tensor(
         uint32_t               n_total_expert,
         uint32_t               n_expert,
         float                  swiglu_clamp,
-        uint32_t               layer_index,
         const ds4_gpu_tensor *x,
         uint32_t               n_tokens,
         ds4_gpu_stream_expert_cache_entry * const *resources,
@@ -38716,7 +46266,6 @@ static int ds4_gpu_glm_routed_moe_batch_grouped_addr_tensor(
         return 0;
     }
 
-    const bool mid_f16 = true;
     const NSUInteger mm_id_threadgroup_bytes = 8192u;
     const uint64_t compact_mid_values = (uint64_t)pair_rows * expert_mid_dim;
     const uint64_t down_values = (uint64_t)pair_rows * out_dim;
@@ -38811,56 +46360,13 @@ static int ds4_gpu_glm_routed_moe_batch_grouped_addr_tensor(
         id<MTLCommandBuffer> cb = ds4_gpu_command_buffer(&owned);
         if (!cb) return 0;
 
-        const bool glm_moe_stage_profile = false;
-        const char *glm_moe_stage_filter = NULL;
-        double glm_moe_stage_t0 = 0.0;
-        if (glm_moe_stage_profile) {
-            if (ds4_gpu_end_commands() == 0 || ds4_gpu_begin_commands() == 0) {
-                return 0;
-            }
-            cb = ds4_gpu_command_buffer(&owned);
-            if (!cb) return 0;
-            glm_moe_stage_t0 = ds4_gpu_now_ms();
-        }
-
         int ok = 1;
-#define DS4_METAL_PROFILE_GLM_GROUPED_ADDR_MOE_STAGE(name) do { \
-            if (ok && glm_moe_stage_profile) { \
-                if (ds4_gpu_end_commands() == 0) { \
-                    ok = 0; \
-                } else { \
-                    const char *stage_name = (name); \
-                    const double now_ms = ds4_gpu_now_ms(); \
-                    const int print_stage = \
-                        !glm_moe_stage_filter || !glm_moe_stage_filter[0] || \
-                        strstr(stage_name, glm_moe_stage_filter) != NULL; \
-                    if (print_stage) { \
-                        fprintf(stderr, \
-                                "ds4: Metal GLM grouped-address routed MoE stage layer=%u tokens=%u pairs=%u experts=%u " \
-                                "gate=%s down=%s mid=f16 %s=%.3f ms\n", \
-                                layer_index, n_tokens, pair_rows, n_expert, \
-                                ds4_gpu_metal_tensor_type_name(gate_type), \
-                                ds4_gpu_metal_tensor_type_name(down_type), \
-                                stage_name, now_ms - glm_moe_stage_t0); \
-                    } \
-                    glm_moe_stage_t0 = now_ms; \
-                    if (ds4_gpu_begin_commands() == 0) { \
-                        ok = 0; \
-                    } else { \
-                        cb = ds4_gpu_command_buffer(&owned); \
-                        if (!cb) ok = 0; \
-                    } \
-                } \
-            } \
-        } while (0)
-
         ok = ds4_gpu_encode_mul_mm_id_map(cb,
                                            map_pipeline,
                                            &map_args,
                                            &gate_args,
                                            selectedbuf,
                                            ds4_gpu_tensor_offset(selected));
-        DS4_METAL_PROFILE_GLM_GROUPED_ADDR_MOE_STAGE("map");
         if (ok) {
             ok = ds4_gpu_encode_mul_mm_id_addr_mapped_tile(cb,
                                                            gate_pipeline,
@@ -38876,7 +46382,6 @@ static int ds4_gpu_glm_routed_moe_batch_grouped_addr_tensor(
                                                            0,
                                                            overflow_gate);
         }
-        DS4_METAL_PROFILE_GLM_GROUPED_ADDR_MOE_STAGE("gate");
         if (ok) {
             ok = ds4_gpu_encode_mul_mm_id_addr_mapped_tile(cb,
                                                            up_pipeline,
@@ -38892,7 +46397,6 @@ static int ds4_gpu_glm_routed_moe_batch_grouped_addr_tensor(
                                                            1,
                                                            overflow_up);
         }
-        DS4_METAL_PROFILE_GLM_GROUPED_ADDR_MOE_STAGE("up");
         if (ok) {
             ok = ds4_gpu_encode_moe_swiglu_weight(cb,
                                                    g_moe_gate_scratch_buffer,
@@ -38906,10 +46410,8 @@ static int ds4_gpu_glm_routed_moe_batch_grouped_addr_tensor(
                                                    expert_mid_dim,
                                                    pair_rows,
                                                    swiglu_clamp,
-                                                   mid_f16);
+                                                   true);
         }
-        DS4_METAL_PROFILE_GLM_GROUPED_ADDR_MOE_STAGE("activation_weight");
-
         id<MTLBuffer> down_dst = n_expert == 1 ? outbuf : g_moe_down_scratch_buffer;
         NSUInteger down_dst_off = n_expert == 1 ? ds4_gpu_tensor_offset(out) : 0;
         if (ok) {
@@ -38927,7 +46429,6 @@ static int ds4_gpu_glm_routed_moe_batch_grouped_addr_tensor(
                                                            2,
                                                            overflow_down);
         }
-        DS4_METAL_PROFILE_GLM_GROUPED_ADDR_MOE_STAGE("down");
         if (ok && n_expert > 1) {
             ok = ds4_gpu_encode_moe_sum_experts(cb,
                                                  down_dst,
@@ -38938,14 +46439,12 @@ static int ds4_gpu_glm_routed_moe_batch_grouped_addr_tensor(
                                                  n_expert,
                                                  n_tokens);
         }
-        DS4_METAL_PROFILE_GLM_GROUPED_ADDR_MOE_STAGE("sum");
         if (!ok) return 0;
 
         if (!ds4_gpu_finish_command_buffer(cb, owned,
                                            "GLM grouped-address routed batch MoE")) {
             return 0;
         }
-#undef DS4_METAL_PROFILE_GLM_GROUPED_ADDR_MOE_STAGE
     }
 
     return 1;
@@ -38980,8 +46479,7 @@ static int ds4_gpu_glm_routed_moe_batch_tensor_impl(
         const ds4_gpu_tensor *x,
         uint32_t                n_tokens,
         uint32_t                mid_token_stride,
-        bool                    allow_grouped,
-        bool                    force_scalar_q4_pair) {
+        bool                    allow_grouped) {
     if (!g_initialized && !ds4_gpu_init()) return 0;
     if (!out || !mid || !model_map || !selected || !weights || !x ||
         n_tokens == 0 ||
@@ -39034,7 +46532,6 @@ static int ds4_gpu_glm_routed_moe_batch_tensor_impl(
     if (allow_grouped &&
         (!g_ssd_streaming_mode ||
          ds4_gpu_glm_streaming_prefill_full_layer_active()) &&
-        ds4_gpu_glm_grouped_moe_layer_enabled(layer_index) &&
         ds4_gpu_glm_routed_moe_batch_grouped_available(gate_type,
                                                        up_type,
                                                        down_type,
@@ -39064,7 +46561,6 @@ static int ds4_gpu_glm_routed_moe_batch_tensor_impl(
                                                            n_total_expert,
                                                            n_expert,
                                                            swiglu_clamp,
-                                                           layer_index,
                                                            x,
                                                            n_tokens);
     }
@@ -39176,12 +46672,6 @@ static int ds4_gpu_glm_routed_moe_batch_tensor_impl(
                                                            down_type,
                                                            n_expert,
                                                            n_tokens);
-        const BOOL enable_q4_pair4 = true;
-        const BOOL q4_scalar_pair = false;
-        const BOOL q4_pair2 =
-            !use_stream_expert_addr_table &&
-            !gate_pair_q5 && !q4_scalar_pair &&
-            (force_scalar_q4_pair || !enable_q4_pair4);
         id<MTLComputePipelineState> pair_pipeline =
             use_stream_expert_addr_table ?
              (gate_pair_q2 ?
@@ -39195,14 +46685,8 @@ static int ds4_gpu_glm_routed_moe_batch_tensor_impl(
             gate_pair_q5 ?
              ds4_gpu_hot_pipeline(g_glm_q5_k_pair_swiglu_f32_pipeline,
                                   "kernel_glm_q5_K_pair_swiglu_f32") :
-            (q4_scalar_pair ?
-             ds4_gpu_hot_pipeline(g_glm_q4_k_pair_swiglu_f32_pipeline,
-                                  "kernel_glm_q4_K_pair_swiglu_f32") :
-             q4_pair2 ?
-              ds4_gpu_hot_pipeline(g_glm_q4_k_pair_swiglu2_f32_pipeline,
-                                   "kernel_glm_q4_K_pair_swiglu2_f32") :
-              ds4_gpu_hot_pipeline(g_glm_q4_k_pair_swiglu4_f32_pipeline,
-                                   "kernel_glm_q4_K_pair_swiglu4_f32"));
+             ds4_gpu_hot_pipeline(g_glm_q4_k_pair_swiglu4_f32_pipeline,
+                                  "kernel_glm_q4_K_pair_swiglu4_f32");
         id<MTLComputePipelineState> down_pipeline =
             use_stream_expert_addr_table ?
              (down_scalar_q2 ?
@@ -39249,7 +46733,8 @@ static int ds4_gpu_glm_routed_moe_batch_tensor_impl(
                         &stream_unique,
                         &stream_overflow_gate,
                         &stream_overflow_up,
-                        &stream_overflow_down)) {
+                        &stream_overflow_down,
+                        false)) {
                 return 0;
             }
             if (stream_unique == 0) {
@@ -39287,7 +46772,6 @@ static int ds4_gpu_glm_routed_moe_batch_tensor_impl(
                         n_total_expert,
                         n_expert,
                         swiglu_clamp,
-                        layer_index,
                         x,
                         n_tokens,
                         stream_resources,
@@ -39310,63 +46794,6 @@ static int ds4_gpu_glm_routed_moe_batch_tensor_impl(
         id<MTLCommandBuffer> cb = ds4_gpu_command_buffer(&owned);
         if (!cb) return 0;
 
-        const bool glm_moe_stage_profile = false;
-        const char *glm_moe_stage_filter = NULL;
-        const char *glm_pair_path = use_stream_expert_addr_table ?
-                                    (gate_pair_q2 ? "q2_stream_addr_swiglu" :
-                                                    "q4_stream_addr_swiglu") :
-                                    gate_pair_q2 ? "q2_scalar_swiglu" :
-                                    gate_pair_q5 ? "q5_pair_simd_swiglu" :
-                                    (q4_scalar_pair ? "q4_scalar_swiglu" :
-                                    (q4_pair2 ? "q4_pair2_simd_swiglu" :
-                                                "q4_pair4_simd_swiglu"));
-        const char *glm_down_path =
-            use_stream_expert_addr_table ?
-            (down_scalar_q2 ? "q2_stream_addr_down" : "q4_stream_addr_down_simd") :
-            down_scalar_q2 ? "q2_down_scalar" :
-            down_scalar_q4 ? "q4_down_simd" :
-            down_simd_q5 ? "q5_down_simd" : "q6_down_simd";
-        double glm_moe_stage_t0 = 0.0;
-        if (glm_moe_stage_profile) {
-            if (ds4_gpu_end_commands() == 0 || ds4_gpu_begin_commands() == 0) {
-                return 0;
-            }
-            cb = ds4_gpu_command_buffer(&owned);
-            if (!cb) return 0;
-            glm_moe_stage_t0 = ds4_gpu_now_ms();
-        }
-        int ok = 1;
-#define DS4_METAL_PROFILE_GLM_MOE_BATCH_STAGE(name) do { \
-            if (ok && glm_moe_stage_profile) { \
-                if (ds4_gpu_end_commands() == 0) { \
-                    ok = 0; \
-                } else { \
-                    const char *stage_name = (name); \
-                    const double now_ms = ds4_gpu_now_ms(); \
-                    const int print_stage = \
-                        !glm_moe_stage_filter || !glm_moe_stage_filter[0] || \
-                        strstr(stage_name, glm_moe_stage_filter) != NULL; \
-                    if (print_stage) { \
-                        fprintf(stderr, \
-                                "ds4: Metal GLM routed MoE batch stage layer=%u tokens=%u experts=%u " \
-                                "gate=%s down=%s pair=%s down_path=%s %s=%.3f ms\n", \
-                                layer_index, n_tokens, n_expert, \
-                                ds4_gpu_metal_tensor_type_name(gate_type), \
-                                ds4_gpu_metal_tensor_type_name(down_type), \
-                                glm_pair_path, glm_down_path, \
-                                stage_name, now_ms - glm_moe_stage_t0); \
-                    } \
-                    glm_moe_stage_t0 = now_ms; \
-                    if (ds4_gpu_begin_commands() == 0) { \
-                        ok = 0; \
-                    } else { \
-                        cb = ds4_gpu_command_buffer(&owned); \
-                        if (!cb) ok = 0; \
-                    } \
-                } \
-            } \
-        } while (0)
-
         ds4_gpu_glm_routed_moe_args args = {
             .tp_rank = g_tp_split_rank,
             .tp_world = g_tp_split_world,
@@ -39393,13 +46820,7 @@ static int ds4_gpu_glm_routed_moe_batch_tensor_impl(
                             (NSUInteger)((expert_mid_dim + 7u) / 8u)) :
             gate_pair_q5 ? (NSUInteger)((expert_mid_dim + 7u) / 8u) :
             use_stream_expert_addr_table ? (NSUInteger)((expert_mid_dim + 3u) / 4u) :
-            q4_scalar_pair ? (NSUInteger)expert_mid_dim :
-            q4_pair2 ? (NSUInteger)((expert_mid_dim + 1u) / 2u) :
             (NSUInteger)((expert_mid_dim + 7u) / 8u);
-        const NSUInteger pair_threadgroup_bytes =
-            q4_scalar_pair ? 512u * sizeof(float) : 0u;
-        const NSUInteger pair_threads =
-            q4_scalar_pair ? 256u : 64u;
         const NSUInteger down_x_groups =
             down_scalar_q2 ? (NSUInteger)((out_dim + 7u) / 8u) :
             down_simd_q4 ? (NSUInteger)((out_dim + 3u) / 4u) :
@@ -39438,16 +46859,11 @@ static int ds4_gpu_glm_routed_moe_batch_tensor_impl(
             if (stream_overflow_gate) [enc useResource:stream_overflow_gate usage:MTLResourceUsageRead];
             if (stream_overflow_up) [enc useResource:stream_overflow_up usage:MTLResourceUsageRead];
         }
-        if (pair_threadgroup_bytes != 0u) {
-            [enc setThreadgroupMemoryLength:pair_threadgroup_bytes atIndex:0];
-        }
         [enc dispatchThreadgroups:MTLSizeMake(pair_x_groups,
                                               (NSUInteger)n_expert,
                                               (NSUInteger)n_tokens)
-             threadsPerThreadgroup:MTLSizeMake(pair_threads, 1, 1)];
+             threadsPerThreadgroup:MTLSizeMake(64u, 1, 1)];
         ds4_gpu_end_compute_encoder(cb, enc);
-        DS4_METAL_PROFILE_GLM_MOE_BATCH_STAGE("pair");
-
         enc = ds4_gpu_compute_encoder(cb);
         [enc setComputePipelineState:down_pipeline];
         [enc setBytes:&args length:sizeof(args) atIndex:0];
@@ -39471,11 +46887,7 @@ static int ds4_gpu_glm_routed_moe_batch_tensor_impl(
                                               1)
              threadsPerThreadgroup:MTLSizeMake(down_threads, 1, 1)];
         ds4_gpu_end_compute_encoder(cb, enc);
-        DS4_METAL_PROFILE_GLM_MOE_BATCH_STAGE("down");
-
-        if (!ok) return 0;
         if (!ds4_gpu_finish_command_buffer(cb, owned, "GLM routed batch MoE")) return 0;
-#undef DS4_METAL_PROFILE_GLM_MOE_BATCH_STAGE
     }
 
     return 1;
@@ -39540,8 +46952,7 @@ int ds4_gpu_glm_routed_moe_batch_tensor(
                                                     x,
                                                     n_tokens,
                                                     mid_token_stride,
-                                                    true,
-                                                    false);
+                                                    true);
 }
 
 int ds4_gpu_glm_routed_moe_batch_direct_scalar_q4_tensor(
@@ -39601,7 +47012,6 @@ int ds4_gpu_glm_routed_moe_batch_direct_scalar_q4_tensor(
                                                     x,
                                                     n_tokens,
                                                     mid_token_stride,
-                                                    false,
                                                     false);
 }
 
@@ -39912,6 +47322,12 @@ static bool ds4_gpu_mxfp4_moe_decode_nsg1_enabled(uint32_t n_tokens) {
            getenv("DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_DECODE_NSG1") == NULL;
 }
 
+static bool ds4_gpu_m1_iq2_mid_only_split_supported(
+        const ds4_gpu_stream_expert_split_args *split) {
+    return split && split->accumulate == 0u && split->active_mask != 0u &&
+           (split->active_mask & ~0x3fu) == 0u;
+}
+
 int ds4_gpu_routed_moe_one_tensor(
         ds4_gpu_tensor       *out,
         ds4_gpu_tensor       *gate,
@@ -39965,7 +47381,9 @@ int ds4_gpu_routed_moe_one_tensor(
         return 0;
     }
     if ((expert_in_dim % 256u) != 0 || (expert_mid_dim % 256u) != 0) { if (getenv("DS4_GLM_TP_DEBUG")) fprintf(stderr, "ds4: routed_moe_one silent return at line %d\n", 32047); return 0; }
-    ds4_gpu_stream_expert_cache_note_token(layer_index);
+    if (!force_resident) {
+        ds4_gpu_stream_expert_cache_note_token(layer_index);
+    }
 
     @autoreleasepool {
         id<MTLBuffer> xbuf = ds4_gpu_tensor_buffer(x);
@@ -40016,6 +47434,9 @@ int ds4_gpu_routed_moe_one_tensor(
         __unsafe_unretained id<MTLBuffer> up_slot_bufs[DS4_METAL_MAX_ROUTED_EXPERT_USED] = { nil };
         __unsafe_unretained id<MTLBuffer> down_slot_bufs[DS4_METAL_MAX_ROUTED_EXPERT_USED] = { nil };
         ds4_gpu_stream_expert_cache_entry *stream_slot_entries[DS4_METAL_MAX_ROUTED_EXPERT_USED] = { NULL };
+        ds4_gpu_stream_expert_cache_entry * const *stream_addr_resources =
+            stream_slot_entries;
+        uint32_t stream_addr_resource_count = n_expert;
         NSUInteger gate_slot_offsets[DS4_METAL_MAX_ROUTED_EXPERT_USED] = { 0 };
         NSUInteger up_slot_offsets[DS4_METAL_MAX_ROUTED_EXPERT_USED] = { 0 };
         NSUInteger down_slot_offsets[DS4_METAL_MAX_ROUTED_EXPERT_USED] = { 0 };
@@ -40025,6 +47446,9 @@ int ds4_gpu_routed_moe_one_tensor(
         id<MTLBuffer> stream_gate_addr_buf = nil;
         id<MTLBuffer> stream_up_addr_buf = nil;
         id<MTLBuffer> stream_down_addr_buf = nil;
+        id<MTLBuffer> stream_overflow_gate = nil;
+        id<MTLBuffer> stream_overflow_up = nil;
+        id<MTLBuffer> stream_overflow_down = nil;
         bool use_stream_expert_addr_table = false;
         bool use_stream_expert_masked_addr_table = false;
         bool use_stream_compact_addr_table = false;
@@ -40270,6 +47694,29 @@ int ds4_gpu_routed_moe_one_tensor(
             (n_expert == 6 || (n_expert == 8 && g_tp_split_world == 2)) &&
             n_tokens == 1 &&
             down_sum6_pipeline != nil;
+        /* Default M1 SSD-streaming specialization. The eventual dispatch is
+         * additionally required to be an address-table path supported by the
+         * unmasked or masked mid-only pipeline, where the following Q2 sum
+         * consumes only `mid`.  Presence of the disable switch always wins,
+         * and every failed predicate retains the canonical address-table
+         * kernel. */
+        const bool m1_iq2_mid_only_disabled =
+            ds4_gpu_env_bool("DS4_METAL_DISABLE_M1_IQ2_MID_ONLY") == 1;
+        const bool m1_iq2_mid_only_required =
+            ds4_gpu_env_bool("DS4_METAL_REQUIRE_M1_IQ2_MID_ONLY") == 1;
+        const bool m1_iq2_addr_mid_only_candidate =
+            ds4_gpu_m1_iq2_mid_only_enabled() &&
+            g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_mid_only_4096x2048_pipeline != nil &&
+            !force_resident && g_ssd_streaming_mode &&
+            gate_type == DS4_METAL_TENSOR_IQ2_XXS &&
+            down_type == DS4_METAL_TENSOR_Q2_K &&
+            n_tokens == 1 && n_expert == 6 && n_total_expert == 256 &&
+            expert_in_dim == 4096 && expert_mid_dim == 2048 &&
+            gate_row_bytes == 1056 && gate_expert_bytes == 2162688 &&
+            gate_args.ne00 == 4096 && gate_args.ne01 == 2048 &&
+            gate_args.ne0 == 2048 && gate_args.nr0 == 4 &&
+            g_tp_split_rank == 0 && g_tp_split_world == 1 && add_in == NULL &&
+            fuse_pair_swiglu && direct_down_sum;
 
         if (g_parallel_q8_pending) {
             /* A concurrent encoder invalidates every implicit dependency in
@@ -40603,6 +48050,95 @@ int ds4_gpu_routed_moe_one_tensor(
         const bool use_selected_slots =
             use_q4_selected_slots || use_iq2_selected_slots ||
             use_mxfp4_selected_slots || use_iq2_stream_addr_table;
+        ds4_gpu_stream_expert_exact_rows_scope *exact_rows_scope =
+            &g_stream_expert_exact_rows_scope;
+        bool use_exact_rows_scope = false;
+        if (exact_rows_scope->active) {
+            uint64_t expected_selected_offset = exact_rows_scope->selected_offset;
+            const uint64_t row_selected_bytes =
+                (uint64_t)exact_rows_scope->n_selected * sizeof(int32_t);
+            if ((uint64_t)exact_rows_scope->row >
+                    (UINT64_MAX - expected_selected_offset) / row_selected_bytes) {
+                fprintf(stderr, "ds4: Metal exact-row selected offset overflow\n");
+                return 0;
+            }
+            expected_selected_offset +=
+                (uint64_t)exact_rows_scope->row * row_selected_bytes;
+            bool exact_rows_supported =
+                exact_rows_scope->row_armed &&
+                exact_rows_scope->row < exact_rows_scope->n_rows &&
+                exact_rows_scope->n_selected == n_expert &&
+                exact_rows_scope->table.model_map == model_map &&
+                exact_rows_scope->table.model_size == model_size &&
+                exact_rows_scope->table.layer == layer_index &&
+                exact_rows_scope->table.n_total_expert == n_total_expert &&
+                exact_rows_scope->table.gate_offset == gate_offset &&
+                exact_rows_scope->table.up_offset == up_offset &&
+                exact_rows_scope->table.down_offset == down_offset &&
+                exact_rows_scope->table.gate_expert_bytes == gate_expert_bytes &&
+                exact_rows_scope->table.down_expert_bytes == down_expert_bytes &&
+                exact_rows_scope->selected_buffer == selectedbuf &&
+                expected_selected_offset == ds4_gpu_tensor_offset(selected) &&
+                g_tp_split_world == 1 &&
+                gate_type == DS4_METAL_TENSOR_IQ2_XXS &&
+                down_type == DS4_METAL_TENSOR_Q2_K &&
+                use_iq2_selected_slots &&
+                g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_pipeline != nil &&
+                g_moe_mul_mv_addr_q2_k_sum6_pipeline != nil;
+            if (exact_rows_supported) {
+                exact_rows_supported =
+                    exact_rows_scope->unique_count != 0 &&
+                    exact_rows_scope->n_resources <=
+                        exact_rows_scope->unique_count &&
+                    exact_rows_scope->gate_addrs &&
+                    exact_rows_scope->up_addrs &&
+                    exact_rows_scope->down_addrs &&
+                    exact_rows_scope->gate_addrs !=
+                        g_stream_expert_cache_gate_addr_buffers[layer_index] &&
+                    exact_rows_scope->up_addrs !=
+                        g_stream_expert_cache_up_addr_buffers[layer_index] &&
+                    exact_rows_scope->down_addrs !=
+                        g_stream_expert_cache_down_addr_buffers[layer_index] &&
+                    exact_rows_scope->gate_addrs !=
+                        g_stream_compact_gate_addr_buffers[layer_index] &&
+                    exact_rows_scope->up_addrs !=
+                        g_stream_compact_up_addr_buffers[layer_index] &&
+                    exact_rows_scope->down_addrs !=
+                        g_stream_compact_down_addr_buffers[layer_index] &&
+                    (exact_rows_scope->n_resources ==
+                         exact_rows_scope->unique_count ||
+                     (exact_rows_scope->overflow_gate &&
+                      exact_rows_scope->overflow_up &&
+                      exact_rows_scope->overflow_down));
+            }
+            for (uint32_t i = 0;
+                 exact_rows_supported && i < exact_rows_scope->n_resources;
+                 i++) {
+                ds4_gpu_stream_expert_cache_entry *entry =
+                    exact_rows_scope->resources[i];
+                exact_rows_supported =
+                    entry && entry->valid &&
+                    entry->gate_buffer == exact_rows_scope->resource_gate[i] &&
+                    entry->up_buffer == exact_rows_scope->resource_up[i] &&
+                    entry->down_buffer == exact_rows_scope->resource_down[i];
+            }
+            if (!exact_rows_supported) {
+                fprintf(stderr,
+                        "ds4: Metal exact-row expert union does not match routed MoE layer=%u row=%u\n",
+                        layer_index,
+                        exact_rows_scope->row);
+                return 0;
+            }
+            use_exact_rows_scope = true;
+            stream_addr_resources = exact_rows_scope->resources;
+            stream_addr_resource_count = exact_rows_scope->n_resources;
+            stream_gate_addr_buf = exact_rows_scope->gate_addrs;
+            stream_up_addr_buf = exact_rows_scope->up_addrs;
+            stream_down_addr_buf = exact_rows_scope->down_addrs;
+            stream_overflow_gate = exact_rows_scope->overflow_gate;
+            stream_overflow_up = exact_rows_scope->overflow_up;
+            stream_overflow_down = exact_rows_scope->overflow_down;
+        }
         id<MTLComputePipelineState> slots_pair_swiglu_pipeline =
             use_iq2_selected_slots ? g_moe_mul_mv_slots6_iq2_xxs_pair_swiglu_pipeline :
             (use_mxfp4_selected_slots ? g_moe_mul_mv_slots6_mxfp4_pair_swiglu_pipeline :
@@ -40938,21 +48474,23 @@ int ds4_gpu_routed_moe_one_tensor(
             bool selected_ids_available = true;
             bool selected_exec_ids_from_host = false;
             const int stream_expert_cache_size_known =
+                use_exact_rows_scope ? 1 :
                 ds4_gpu_stream_expert_cache_note_expert_size(gate_expert_bytes,
                                                              down_expert_bytes);
             const bool use_iq2_full_expert_addr_table =
-                use_iq2_selected_slots &&
+                !use_exact_rows_scope && use_iq2_selected_slots &&
                 ds4_gpu_stream_full_expert_addr_table_requested() &&
                 g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_pipeline != nil &&
                 g_moe_mul_mv_addr_q2_k_sum6_pipeline != nil;
             use_stream_expert_cache =
-                !use_iq2_full_expert_addr_table &&
-                (use_iq2_selected_slots || use_iq2_stream_addr_table ||
-                 use_q4_selected_slots || use_mxfp4_selected_slots) &&
-                stream_expert_cache_size_known &&
-                ds4_gpu_stream_expert_cache_effective_cap(layer_index,
-                                                          n_total_expert,
-                                                          n_expert) != 0;
+                use_exact_rows_scope ||
+                (!use_iq2_full_expert_addr_table &&
+                 (use_iq2_selected_slots || use_iq2_stream_addr_table ||
+                  use_q4_selected_slots || use_mxfp4_selected_slots) &&
+                 stream_expert_cache_size_known &&
+                 ds4_gpu_stream_expert_cache_effective_cap(layer_index,
+                                                           n_total_expert,
+                                                           n_expert) != 0);
             if (use_iq2_stream_addr_table && !use_stream_expert_cache) {
                 fprintf(stderr,
                         "ds4: Metal IQ2/IQ2 streaming decode requires a non-empty expert cache\n");
@@ -40960,10 +48498,10 @@ int ds4_gpu_routed_moe_one_tensor(
                 return 0;
             }
             const bool stream_split_ready =
-                use_stream_expert_cache &&
+                !use_exact_rows_scope && use_stream_expert_cache &&
                 ds4_gpu_stream_expert_split_ready();
             const bool use_stream_compact_addr =
-                use_stream_expert_cache &&
+                !use_exact_rows_scope && use_stream_expert_cache &&
                 use_iq2_selected_slots &&
                 ds4_gpu_stream_compact_addr_requested() &&
                 !stream_split_ready &&
@@ -40971,14 +48509,14 @@ int ds4_gpu_routed_moe_one_tensor(
                 g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_pipeline != nil &&
                 g_moe_mul_mv_addr_q2_k_sum6_pipeline != nil;
             use_stream_expert_split_candidate =
-                use_stream_expert_cache &&
+                !use_exact_rows_scope && use_stream_expert_cache &&
                 use_iq2_selected_slots &&
                 !use_stream_compact_addr &&
                 stream_split_ready &&
                 g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_masked_pipeline != nil &&
                 g_moe_mul_mv_addr_q2_k_sum6_masked_pipeline != nil;
             const bool use_stream_hit_validator =
-                use_stream_expert_cache &&
+                !use_exact_rows_scope && use_stream_expert_cache &&
                 use_iq2_selected_slots &&
                 ds4_gpu_stream_expert_hit_validator_requested() &&
                 g_moe_stream_expert_cache_validate_pipeline != nil &&
@@ -40988,7 +48526,19 @@ int ds4_gpu_routed_moe_one_tensor(
                                                          &stream_gate_addr_buf,
                                                          &stream_up_addr_buf,
                                                          &stream_down_addr_buf);
-            if (use_iq2_full_expert_addr_table) {
+            if (use_exact_rows_scope) {
+                selected_id_source = "exact-union";
+                memcpy(selected_ids,
+                       exact_rows_scope->selected_ids +
+                           (uint64_t)exact_rows_scope->row * n_expert,
+                       (size_t)n_expert * sizeof(selected_ids[0]));
+                g_routed_moe_selected_override_n = 0;
+                use_stream_expert_cache = true;
+                use_stream_expert_addr_table = true;
+                use_stream_expert_masked_addr_table = false;
+                use_stream_expert_split_candidate = false;
+                use_stream_expert_split_deferred = false;
+            } else if (use_iq2_full_expert_addr_table) {
                 selected_id_source = "gpu-full-addr";
                 selected_ids_available = false;
                 g_routed_moe_selected_override_n = 0;
@@ -41111,7 +48661,7 @@ int ds4_gpu_routed_moe_one_tensor(
                 selected_t0 = ds4_gpu_now_ms();
             }
 
-            if (selected_ids_available) {
+            if (selected_ids_available && !use_exact_rows_scope) {
                 for (uint32_t i = 0; i < n_expert; i++) {
                     if (selected_ids[i] < 0 || (uint32_t)selected_ids[i] >= n_total_expert) {
                         fprintf(stderr,
@@ -41244,10 +48794,44 @@ int ds4_gpu_routed_moe_one_tensor(
                     }
                 }
             }
-            if (use_stream_expert_cache) {
+            if (use_exact_rows_scope) {
+                /* The union scope owns a full, private address table for the
+                 * whole speculative microbatch.  Do not run any ordinary
+                 * per-token cache setup here: it mutates the shared per-layer
+                 * tables and may prune entries already referenced by an
+                 * earlier exact row in this command buffer. */
+                stream_addr_resources = exact_rows_scope->resources;
+                stream_addr_resource_count = exact_rows_scope->n_resources;
+                stream_gate_addr_buf = exact_rows_scope->gate_addrs;
+                stream_up_addr_buf = exact_rows_scope->up_addrs;
+                stream_down_addr_buf = exact_rows_scope->down_addrs;
+                stream_overflow_gate = exact_rows_scope->overflow_gate;
+                stream_overflow_up = exact_rows_scope->overflow_up;
+                stream_overflow_down = exact_rows_scope->overflow_down;
+                selected_exec_buf = selectedbuf;
+                selected_exec_off = ds4_gpu_tensor_offset(selected);
+                use_stream_expert_cache = true;
+                use_stream_expert_addr_table = true;
+                use_stream_expert_masked_addr_table = false;
+                use_stream_compact_addr_table = false;
+                use_stream_expert_split_candidate = false;
+                use_stream_expert_split_deferred = false;
+                if (!stream_gate_addr_buf || !stream_up_addr_buf ||
+                    !stream_down_addr_buf ||
+                    (stream_addr_resource_count == 0 &&
+                     (!stream_overflow_gate || !stream_overflow_up ||
+                      !stream_overflow_down))) {
+                    fprintf(stderr,
+                            "ds4: Metal exact-row expert union lost private resources at layer=%u row=%u\n",
+                            layer_index,
+                            exact_rows_scope->row);
+                    return 0;
+                }
+            } else if (use_stream_expert_cache) {
                 use_stream_expert_addr_table =
                     ((use_iq2_selected_slots &&
-                      ds4_gpu_stream_expert_addr_table_kernel_requested() &&
+                      (ds4_gpu_stream_expert_addr_table_kernel_requested() ||
+                       m1_iq2_addr_mid_only_candidate) &&
                       g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_pipeline != nil &&
                       g_moe_mul_mv_addr_q2_k_sum6_pipeline != nil) ||
                      use_iq2_stream_addr_table) &&
@@ -41466,6 +49050,33 @@ int ds4_gpu_routed_moe_one_tensor(
             [cb useResidencySet:q4_table_layer_residency];
         }
 
+        const bool use_m1_iq2_addr_mid_only =
+            m1_iq2_addr_mid_only_candidate &&
+            use_iq2_selected_slots && use_stream_expert_addr_table &&
+            (!use_stream_expert_masked_addr_table ||
+             g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_mid_only_4096x2048_masked_pipeline != nil);
+        if (m1_iq2_mid_only_required && !m1_iq2_mid_only_disabled &&
+            !use_m1_iq2_addr_mid_only) {
+            fprintf(stderr,
+                    "ds4: required Metal M1 IQ2 address-table mid-only producer was not selected "
+                    "candidate=%d selected_slots=%d addr=%d masked=%d split=%d "
+                    "gate=%u down=%u experts=%u/%u shape=%ux%u ssd=%d resident=%d\n",
+                    m1_iq2_addr_mid_only_candidate ? 1 : 0,
+                    use_iq2_selected_slots ? 1 : 0,
+                    use_stream_expert_addr_table ? 1 : 0,
+                    use_stream_expert_masked_addr_table ? 1 : 0,
+                    use_stream_expert_split_deferred ? 1 : 0,
+                    gate_type,
+                    down_type,
+                    n_expert,
+                    n_total_expert,
+                    expert_in_dim,
+                    expert_mid_dim,
+                    g_ssd_streaming_mode ? 1 : 0,
+                    force_resident ? 1 : 0);
+            return 0;
+        }
+
         const bool moe_one_stage_profile =
             g_batch_cb != nil &&
             ds4_gpu_stage_profile_enabled_for_layer("DS4_METAL_MOE_ONE_STAGE_PROFILE",
@@ -41481,6 +49092,10 @@ int ds4_gpu_routed_moe_one_tensor(
             use_q4_expert_address_table ? "q4_addr_pair_swiglu" :
             use_q4_expert_table ? "q4_table_pair_swiglu" :
             use_q4_gather_slots ? "q4_gather_slots6_pair_swiglu" :
+            use_m1_iq2_addr_mid_only ?
+                (use_stream_expert_masked_addr_table ?
+                    "iq2_stream_addr_mask_mid_only_4096x2048" :
+                    "iq2_stream_addr_mid_only_4096x2048") :
             use_stream_expert_split_deferred ? "iq2_stream_split_pair_swiglu" :
             use_stream_expert_masked_addr_table ? "iq2_stream_addr_mask_pair_swiglu" :
             use_stream_expert_addr_table ? "iq2_stream_addr_pair_swiglu" :
@@ -41850,7 +49465,10 @@ int ds4_gpu_routed_moe_one_tensor(
                             .accumulate = 0u,
                         };
                         ok = ds4_gpu_encode_mul_mv_addr_iq2_pair_swiglu_masked(cb,
-                                                                               g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_masked_pipeline,
+                                                                               use_m1_iq2_addr_mid_only &&
+                                                                               ds4_gpu_m1_iq2_mid_only_split_supported(&resident_pair_args) ?
+                                                                                   g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_mid_only_4096x2048_masked_pipeline :
+                                                                                   g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_masked_pipeline,
                                                                                &gate_args,
                                                                                &act_args,
                                                                                &resident_pair_args,
@@ -41872,13 +49490,6 @@ int ds4_gpu_routed_moe_one_tensor(
                                                                                gate_smem,
                                                                                2,
                                                                                false);
-                        if (ok) {
-                            ok = ds4_gpu_flush_commands();
-                            if (ok) {
-                                cb = ds4_gpu_command_buffer(&owned);
-                                if (!cb) ok = 0;
-                            }
-                        }
                         const double stream_split_resident_ms =
                             stream_split_timing ? ds4_gpu_now_ms() - stream_split_t0 : 0.0;
                         if (stream_split_timing) stream_split_t0 = ds4_gpu_now_ms();
@@ -41960,19 +49571,13 @@ int ds4_gpu_routed_moe_one_tensor(
                             }
                         }
                         if (ok) {
-                            /*
-                             * The resident stage was submitted before the
-                             * CPU read of missing experts so I/O can overlap
-                             * with GPU work. The missing stage reuses the same
-                             * gate/up/mid scratch buffers, so it must not
-                             * execute until the resident command buffer has
-                             * finished. The down/sum pass is issued once after
-                             * all six mid slots exist; this keeps the final
-                             * accumulation order stable regardless of the
-                             * resident/missing split.
-                             */
+                            /* Resident and missing kernels share scratch but
+                             * are encoded serially in the current command
+                             * buffer. Drain older pending buffers to advance
+                             * cache epochs without committing this layer's
+                             * resident stage separately. */
                             ok = ds4_gpu_wait_pending_command_buffers(
-                                    "streaming expert split resident");
+                                    "streaming expert split prior pending");
                             if (stream_split_timing) {
                                 const double now_ms = ds4_gpu_now_ms();
                                 stream_split_missing_wait_ms =
@@ -41994,7 +49599,10 @@ int ds4_gpu_routed_moe_one_tensor(
                         };
                         if (ok) {
                             ok = ds4_gpu_encode_mul_mv_addr_iq2_pair_swiglu_masked(cb,
-                                                                                   g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_masked_pipeline,
+                                                                                   use_m1_iq2_addr_mid_only &&
+                                                                                   ds4_gpu_m1_iq2_mid_only_split_supported(&missing_pair_args) ?
+                                                                                       g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_mid_only_4096x2048_masked_pipeline :
+                                                                                       g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_masked_pipeline,
                                                                                    &gate_args,
                                                                                    &act_args,
                                                                                    &missing_pair_args,
@@ -42064,7 +49672,10 @@ int ds4_gpu_routed_moe_one_tensor(
                             .accumulate = 0u,
                         };
                         ok = ds4_gpu_encode_mul_mv_addr_iq2_pair_swiglu_masked(cb,
-                                                                               g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_masked_pipeline,
+                                                                               use_m1_iq2_addr_mid_only &&
+                                                                               ds4_gpu_m1_iq2_mid_only_split_supported(&split_args) ?
+                                                                                   g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_mid_only_4096x2048_masked_pipeline :
+                                                                                   g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_masked_pipeline,
                                                                                &gate_args,
                                                                                &act_args,
                                                                                &split_args,
@@ -42089,11 +49700,13 @@ int ds4_gpu_routed_moe_one_tensor(
                     }
                 } else {
                     ok = ds4_gpu_encode_mul_mv_addr_iq2_pair_swiglu(cb,
-                                                                    g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_pipeline,
+                                                                    use_m1_iq2_addr_mid_only ?
+                                                                        g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_mid_only_4096x2048_pipeline :
+                                                                        g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_pipeline,
                                                                     &gate_args,
                                                                     &act_args,
-                                                                    stream_slot_entries,
-                                                                    n_expert,
+                                                                    stream_addr_resources,
+                                                                    stream_addr_resource_count,
                                                                     stream_gate_addr_buf,
                                                                     stream_up_addr_buf,
                                                                     xbuf,
@@ -42111,8 +49724,8 @@ int ds4_gpu_routed_moe_one_tensor(
                                                                     gate_smem,
                                                                     2,
                                                                     false,
-                                                                    nil,
-                                                                    nil);
+                                                                    stream_overflow_gate,
+                                                                    stream_overflow_up);
                 }
             } else {
                 ok = (!use_stream_expert_cache ||
@@ -42441,8 +50054,8 @@ int ds4_gpu_routed_moe_one_tensor(
                     ok = ds4_gpu_encode_mul_mv_addr_q2_sum6(cb,
                                                              g_moe_mul_mv_addr_q2_k_sum6_pipeline,
                                                              &down_args,
-                                                             stream_slot_entries,
-                                                             n_expert,
+                                                             stream_addr_resources,
+                                                             stream_addr_resource_count,
                                                              stream_down_addr_buf,
                                                              midbuf,
                                                              ds4_gpu_tensor_offset(mid),
@@ -42452,7 +50065,7 @@ int ds4_gpu_routed_moe_one_tensor(
                                                              selected_exec_off,
                                                              down_smem,
                                                              2,
-                                                             nil);
+                                                             stream_overflow_down);
                 }
             } else {
                 ok = (!use_stream_expert_cache ||
@@ -42528,6 +50141,10 @@ int ds4_gpu_routed_moe_one_tensor(
                 return 0;
             }
         }
+        if (use_exact_rows_scope) {
+            exact_rows_scope->row_armed = 0;
+            exact_rows_scope->next_row++;
+        }
 #undef DS4_METAL_PROFILE_MOE_ONE_STAGE
     }
 
@@ -42565,7 +50182,6 @@ int ds4_gpu_routed_moe_batch_tensor(
         uint32_t                n_tokens,
         bool                   *mid_is_f16,
         bool                    force_resident) {
-    (void)force_resident;
     if (!g_initialized && !ds4_gpu_init()) return 0;
     /* TP sharding (see ds4_gpu_routed_moe_one_tensor): bind from the owned
      * expert range and rebase ids in the kernels. */
@@ -42665,7 +50281,7 @@ int ds4_gpu_routed_moe_batch_tensor(
         n_tokens == 1 &&
         n_expert == 6 &&
         n_total_expert >= 128 &&
-        (g_ssd_streaming_mode ||
+        ((g_ssd_streaming_mode && !force_resident) ||
          (gate_tensor_bytes >= q4_selected_min_tensor_bytes &&
           down_tensor_bytes >= q4_selected_min_tensor_bytes)) &&
         !g_quality_mode &&
@@ -42690,7 +50306,7 @@ int ds4_gpu_routed_moe_batch_tensor(
         getenv("DS4_METAL_DISABLE_ROUTED_PAIR_SWIGLU_FUSION") == NULL &&
         g_moe_mul_mv_id_mxfp4_pair_swiglu_pipeline != nil &&
         g_moe_mul_mv_id_mxfp4_sum6_pipeline != nil &&
-        (!g_ssd_streaming_mode ||
+        (force_resident || !g_ssd_streaming_mode ||
          (g_moe_mul_mv_slots6_mxfp4_pair_swiglu_pipeline != nil &&
           g_moe_mul_mv_slots6_mxfp4_sum6_pipeline != nil));
     if (use_single_token_q4_one_tensor || use_single_token_mxfp4_one_tensor) {
@@ -42722,7 +50338,7 @@ int ds4_gpu_routed_moe_batch_tensor(
                                              x,
                                              NULL,
                                              layer_index,
-                                             false);
+                                             force_resident);
     }
 
     @autoreleasepool {
@@ -42788,6 +50404,7 @@ int ds4_gpu_routed_moe_batch_tensor(
             return 0;
         }
         const bool use_iq2_batch_selected_addr =
+            !force_resident &&
             ds4_gpu_stream_prefill_batch_selected_addr_enabled(n_tokens,
                                                                n_total_expert,
                                                                n_expert,
@@ -42801,6 +50418,102 @@ int ds4_gpu_routed_moe_batch_tensor(
             g_moe_mul_mv_addr_iq2_xxs_pair_swiglu_pipeline != nil &&
             g_moe_mul_mv_addr_q2_k_sum6_pipeline != nil;
 
+        /* IQ2 grouped-MM controls are value-aware. The sparse address-MM
+         * path is automatic for eligible IQ2_XXS/Q2_K SSD prefill. Once the
+         * full selected-address domain is available, the automatic choice is
+         * fail-closed; smaller byte/automatic caches retain the sparse-MV
+         * fallback. Explicit REQUIRE is intentionally stronger. */
+        const int enable_iq2_batch_addr_mm =
+            ds4_gpu_env_bool("DS4_METAL_ENABLE_IQ2_XXS_SSD_PREFILL_MM");
+        const int require_iq2_batch_addr_mm_env =
+            ds4_gpu_env_bool("DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM");
+        const int disable_iq2_batch_addr_mm =
+            ds4_gpu_env_bool("DS4_METAL_DISABLE_IQ2_XXS_SSD_PREFILL_MM");
+        int request_iq2_batch_addr_mm = 0;
+        int require_iq2_batch_addr_mm = 0;
+        if (!ds4_gpu_iq2_stream_addr_mm_resolve_policy(
+                enable_iq2_batch_addr_mm,
+                require_iq2_batch_addr_mm_env,
+                disable_iq2_batch_addr_mm,
+                use_iq2_batch_selected_addr,
+                &request_iq2_batch_addr_mm,
+                &require_iq2_batch_addr_mm)) {
+            fprintf(stderr, "ds4: invalid Metal IQ2_XXS SSD grouped-MM policy\n");
+            return 0;
+        }
+        const bool iq2_batch_addr_mm_candidate =
+            !force_resident && g_ssd_streaming_mode &&
+            gate_type == DS4_METAL_TENSOR_IQ2_XXS &&
+            down_type == DS4_METAL_TENSOR_Q2_K &&
+            n_tokens >= 32u && n_expert == 6 &&
+            n_total_expert != 0 &&
+            n_total_expert <= DS4_METAL_STREAM_EXPERT_CACHE_MAX_EXPERT &&
+            g_tp_split_world == 1 &&
+            !g_quality_mode &&
+            getenv("DS4_METAL_MOE_WRITE_CLAMPED_ACT") == NULL &&
+            getenv("DS4_METAL_DISABLE_ROUTED_PAIR_SWIGLU_FUSION") == NULL &&
+            !ds4_gpu_stage_profile_enabled_for_layer(
+                "DS4_METAL_MOE_STAGE_PROFILE",
+                "DS4_METAL_MOE_STAGE_PROFILE_LAYER",
+                layer_index) &&
+            getenv("DS4_METAL_GRAPH_DUMP_PREFIX") == NULL;
+        if (iq2_batch_addr_mm_candidate) {
+            ds4_gpu_iq2_stream_addr_mm_stats_note_candidate();
+        }
+        const bool iq2_batch_addr_mm_policy =
+            request_iq2_batch_addr_mm &&
+            iq2_batch_addr_mm_candidate &&
+            use_iq2_batch_selected_addr;
+        /* Balanced full-model A/B on M1 Max is stable above the promotion
+         * gate at 512 tokens, while the shorter batches remain I/O-bound.
+         * An explicit ENABLE forces the specialization below the automatic
+         * threshold; ENABLE=0 and DISABLE=1 are rollback controls. */
+        const int enable_iq2_batch_addr_mm_tail_cull =
+            ds4_gpu_env_bool(
+                "DS4_METAL_ENABLE_IQ2_XXS_SSD_PREFILL_MM_ADDR_TAIL_CULL");
+        const int disable_iq2_batch_addr_mm_tail_cull =
+            ds4_gpu_env_bool(
+                "DS4_METAL_DISABLE_IQ2_XXS_SSD_PREFILL_MM_ADDR_TAIL_CULL");
+        const bool iq2_batch_addr_mm_tail_cull =
+            disable_iq2_batch_addr_mm_tail_cull != 1 &&
+            (enable_iq2_batch_addr_mm_tail_cull == 1 ||
+             (enable_iq2_batch_addr_mm_tail_cull == -1 &&
+              ds4_gpu_device_is_pre_m5_apple_silicon() &&
+              n_tokens >= 512u));
+        id<MTLComputePipelineState> iq2_gate_addr_mm_pipeline =
+            iq2_batch_addr_mm_policy &&
+            (g_test_flags &
+             DS4_GPU_TEST_IQ2_SSD_GROUPED_PIPELINE_FAILURE) == 0u ?
+                (iq2_batch_addr_mm_tail_cull ?
+                    ds4_gpu_routed_mm_addr_tail_cull_pipeline(gate_type) :
+                    ds4_gpu_routed_mm_addr_pipeline(gate_type)) : nil;
+        id<MTLComputePipelineState> iq2_down_addr_mm_pipeline =
+            iq2_batch_addr_mm_policy &&
+            (g_test_flags &
+             DS4_GPU_TEST_IQ2_SSD_GROUPED_PIPELINE_FAILURE) == 0u ?
+                (iq2_batch_addr_mm_tail_cull ?
+                    ds4_gpu_routed_mm_addr_tail_cull_pipeline(down_type) :
+                    ds4_gpu_routed_mm_addr_pipeline(down_type)) : nil;
+        const bool use_iq2_batch_addr_mm =
+            iq2_batch_addr_mm_policy &&
+            iq2_gate_addr_mm_pipeline != nil &&
+            iq2_down_addr_mm_pipeline != nil;
+
+        /* Both explicit and materially-ready implicit REQUIRE only assert
+         * calls satisfying the specialization contract. Short final chunks
+         * keep the established path. */
+        if (require_iq2_batch_addr_mm &&
+            !disable_iq2_batch_addr_mm &&
+            iq2_batch_addr_mm_candidate &&
+            !use_iq2_batch_addr_mm) {
+            ds4_gpu_iq2_stream_addr_mm_stats_note_require_failure();
+            fprintf(stderr,
+                    "ds4: required Metal IQ2_XXS SSD grouped address-MM path "
+                    "was not selected tokens=%u gate=%u down=%u\n",
+                    n_tokens, gate_type, down_type);
+            return 0;
+        }
+
         ds4_gpu_mul_mv_id_args gate_args =
             ds4_gpu_make_mul_mv_id_args(expert_in_dim, expert_mid_dim, n_total_expert,
                                           gate_row_bytes, gate_expert_bytes,
@@ -42911,13 +50624,39 @@ int ds4_gpu_routed_moe_batch_tensor(
          * order, and the same epilogue math as separate GEMMs + SwiGLU, so the
          * mid tensor is bit-identical.
          */
-        const bool use_mm_id_pair_swiglu =
+        /* Repeated resident A/B on M1 Max covers this exact GLM prefill
+         * shape at 4096 tokens.  Keep the production default inside that
+         * measured domain: the SSD path has different page-fault locality,
+         * and other batch sizes/devices need their own promotion data.  The
+         * REQUIRE test flag may opt the same exact resident shape in on other
+         * Metal devices so the portable harness can collect that data. */
+        const bool measured_top8_iq2_pair_swiglu_shape =
+            n_expert == 8 &&
+            n_total_expert == 288u &&
+            gate_type == DS4_METAL_TENSOR_IQ2_XXS &&
+            down_type == DS4_METAL_TENSOR_Q2_K &&
+            expert_in_dim == 4096u &&
+            expert_mid_dim == 2048u &&
+            out_dim == 4096u &&
+            gate_row_bytes == 1056u &&
+            gate_expert_bytes == 2162688u &&
+            down_row_bytes == 672u &&
+            down_expert_bytes == 2752512u &&
+            clamp == 0.0f &&
+            n_tokens == 4096u &&
+            !g_ssd_streaming_mode;
+        const bool use_measured_top8_iq2_pair_swiglu =
+            measured_top8_iq2_pair_swiglu_shape &&
+            (ds4_gpu_device_name_contains("M1 Max") ||
+             (g_test_flags &
+              DS4_GPU_TEST_REQUIRE_IQ2_TOP8_PAIR_SWIGLU) != 0u);
+        bool use_mm_id_pair_swiglu =
             use_mm_id &&
             !(gate_type == DS4_METAL_TENSOR_IQ2_XXS &&
               (ds4_gpu_routed_mm_mpp_mask() & 3) == 3) &&
-            g_tp_split_world != 2 &&    /* pair-swiglu mm kernel lacks expert ownership */
+            g_tp_split_world == 1 &&    /* pair-swiglu mm kernel lacks expert ownership */
             request_mid_f16 &&
-            n_expert == 6 &&
+            (n_expert == 6 || use_measured_top8_iq2_pair_swiglu) &&
             ((gate_type == DS4_METAL_TENSOR_IQ2_XXS &&
               down_type == DS4_METAL_TENSOR_Q2_K) ||
              (gate_type == DS4_METAL_TENSOR_Q4_K &&
@@ -42927,6 +50666,52 @@ int ds4_gpu_routed_moe_batch_tensor(
             getenv("DS4_METAL_DISABLE_MOE_MM_ID_PAIR_SWIGLU") == NULL &&
             getenv("DS4_METAL_MOE_WRITE_CLAMPED_ACT") == NULL &&
             getenv("DS4_METAL_GRAPH_DUMP_PREFIX") == NULL;
+        if ((g_test_flags & DS4_GPU_TEST_REQUIRE_IQ2_TOP8_PAIR_SWIGLU) != 0u &&
+            !(measured_top8_iq2_pair_swiglu_shape &&
+              use_mm_id_pair_swiglu)) {
+            fprintf(stderr,
+                    "ds4: required Metal IQ2 top-8 MM ID pair-SwiGLU path "
+                    "was not selected tokens=%u experts=%u\n",
+                    n_tokens, n_expert);
+            return 0;
+        }
+        /* The specialization is arithmetically valid for every grouped-MM
+         * IQ2_XXS/Q2_K top-6 shape below.  Keep the automatic default narrower:
+         * resident kernel A/B and full-model SSD A/B measured the exact DS4
+         * production geometry at 4096 tokens.  An explicit ENABLE opts other
+         * eligible shapes in; ENABLE=0 and DISABLE=1 are rollback controls. */
+        const bool iq2_q2_mm_id_pair_tail_cull_eligible =
+            use_mm_id &&
+            gate_type == DS4_METAL_TENSOR_IQ2_XXS &&
+            down_type == DS4_METAL_TENSOR_Q2_K &&
+            request_mid_f16 &&
+            n_expert == 6 &&
+            g_tp_split_world == 1 &&
+            n_tokens >= 32u;
+        const bool iq2_q2_mm_id_pair_tail_cull_production_geometry =
+            n_total_expert == 256u &&
+            expert_in_dim == 4096u &&
+            expert_mid_dim == 2048u &&
+            out_dim == 4096u &&
+            gate_row_bytes == 1056u &&
+            gate_expert_bytes == 2162688u &&
+            down_row_bytes == 672u &&
+            down_expert_bytes == 2752512u;
+        const int enable_iq2_xxs_mm_id_pair_tail_simdgroup_cull =
+            ds4_gpu_env_bool(
+                "DS4_METAL_ENABLE_IQ2_XXS_MOE_MM_ID_PAIR_TAIL_SIMDGROUP_CULL");
+        const int disable_iq2_xxs_mm_id_pair_tail_simdgroup_cull =
+            ds4_gpu_env_bool(
+                "DS4_METAL_DISABLE_IQ2_XXS_MOE_MM_ID_PAIR_TAIL_SIMDGROUP_CULL");
+        const bool use_iq2_xxs_mm_id_pair_tail_simdgroup_cull =
+            iq2_q2_mm_id_pair_tail_cull_eligible &&
+            use_mm_id_pair_swiglu &&
+            disable_iq2_xxs_mm_id_pair_tail_simdgroup_cull != 1 &&
+            (enable_iq2_xxs_mm_id_pair_tail_simdgroup_cull == 1 ||
+             (enable_iq2_xxs_mm_id_pair_tail_simdgroup_cull == -1 &&
+              ds4_gpu_device_is_pre_m5_apple_silicon() &&
+              iq2_q2_mm_id_pair_tail_cull_production_geometry &&
+              n_tokens >= 4096u));
         /*
          * The MXFP4 32x32 specialization uses two SIMDgroups and 8 KiB of
          * threadgroup memory, and exactly culls SIMDgroup 1 on at-most-16-row
@@ -43014,7 +50799,7 @@ int ds4_gpu_routed_moe_batch_tensor(
             g_tp_split_world == 1 &&
             (use_pre_m5_mxfp4_mm_id_down_half_lut_default ||
              (g_test_flags & DS4_GPU_TEST_MXFP4_DOWN_HALF_LUT) != 0u);
-        if (use_mm_id) {
+        if (use_mm_id || use_iq2_batch_addr_mm) {
             gate_map_args =
                 ds4_gpu_make_mul_mm_id_map_args(expert_in_dim, n_total_expert, 1, n_expert, n_tokens);
             gate_mm_args =
@@ -43037,9 +50822,13 @@ int ds4_gpu_routed_moe_batch_tensor(
                 use_mxfp4_mm_id_map_scatter ?
                     "kernel_mul_mm_id_map_scatter_work_ne20_6" :
                     ds4_gpu_mul_mm_id_map0_name(n_expert));
-            gate_mm_pipeline = ds4_gpu_routed_mm_pipeline(gate_type);
-            up_mm_pipeline = ds4_gpu_routed_mm_pipeline(gate_type);
-            down_mm_pipeline = use_mxfp4_mm_id_down_half_lut ?
+            gate_mm_pipeline = use_iq2_batch_addr_mm ?
+                iq2_gate_addr_mm_pipeline :
+                ds4_gpu_routed_mm_pipeline(gate_type);
+            up_mm_pipeline = gate_mm_pipeline;
+            down_mm_pipeline = use_iq2_batch_addr_mm ?
+                iq2_down_addr_mm_pipeline :
+                use_mxfp4_mm_id_down_half_lut ?
                 ds4_gpu_get_mul_mm_id_pipeline(
                     use_mxfp4_mm_id_down_tail_simdgroup_cull ?
                         "kernel_mul_mm_id_mxfp4_f16_half_lut_tail_cull" :
@@ -43052,7 +50841,8 @@ int ds4_gpu_routed_moe_batch_tensor(
                     ds4_gpu_routed_mm_f16_rhs_pipeline(down_type) :
                     ds4_gpu_routed_mm_pipeline(down_type);
             const int mpp_mask = ds4_gpu_routed_mm_mpp_mask();
-            if (mpp_mask && gate_type == DS4_METAL_TENSOR_IQ2_XXS) {
+            if (!use_iq2_batch_addr_mm &&
+                mpp_mask && gate_type == DS4_METAL_TENSOR_IQ2_XXS) {
                 id<MTLComputePipelineState> mpp =
                     ds4_gpu_get_mul_mm_id_pipeline("kernel_mul_mm_id_iq2_xxs_f32_mpp", false);
                 if (mpp) {
@@ -43081,7 +50871,8 @@ int ds4_gpu_routed_moe_batch_tensor(
                     ds4_gpu_get_mul_mm_id_pipeline("kernel_mul_mm_id_mxfp4_f16_mpp", false);
                 if (mpp) down_mm_pipeline = mpp;
             }
-            if ((mpp_mask & 4) && request_mid_f16 &&
+            if (!use_iq2_batch_addr_mm &&
+                (mpp_mask & 4) && request_mid_f16 &&
                 (down_type == DS4_METAL_TENSOR_Q2_K || down_type == DS4_METAL_TENSOR_IQ2_XXS)) {
                 id<MTLComputePipelineState> mpp = ds4_gpu_get_mul_mm_id_pipeline(
                     down_type == DS4_METAL_TENSOR_Q2_K ?
@@ -43120,7 +50911,31 @@ int ds4_gpu_routed_moe_batch_tensor(
                                 (use_mxfp4_mm_id_pair_half_scale ?
                                     "kernel_mul_mm_id_mxfp4_pair_swiglu_f16_half_scale" :
                                     "kernel_mul_mm_id_mxfp4_pair_swiglu_f16")) :
-                            "kernel_mul_mm_id_iq2_xxs_pair_swiglu_f16");
+                            (use_iq2_xxs_mm_id_pair_tail_simdgroup_cull ?
+                                "kernel_mul_mm_id_iq2_xxs_pair_swiglu_f16_tail_cull" :
+                                "kernel_mul_mm_id_iq2_xxs_pair_swiglu_f16"));
+                /* Custom DS4_METAL_MOE_SOURCE files may predate the promoted
+                 * symbol.  Preserve automatic-default compatibility by
+                 * falling back to the established pair kernel.  An explicit
+                 * ENABLE remains strict so benchmark/bring-up cannot silently
+                 * measure the baseline under the candidate label. */
+                if (!pair_swiglu_mm_pipeline &&
+                    gate_type == DS4_METAL_TENSOR_IQ2_XXS &&
+                    use_iq2_xxs_mm_id_pair_tail_simdgroup_cull &&
+                    enable_iq2_xxs_mm_id_pair_tail_simdgroup_cull != 1) {
+                    pair_swiglu_mm_pipeline = ds4_gpu_get_pipeline(
+                        "kernel_mul_mm_id_iq2_xxs_pair_swiglu_f16");
+                }
+                /* A custom Metal source may also predate the base pair
+                 * symbol.  The measured top-8 production default must remain
+                 * fail-soft; only REQUIRE is allowed to turn that into a hard
+                 * error so benchmark labels cannot hide a fallback. */
+                if (!pair_swiglu_mm_pipeline &&
+                    use_measured_top8_iq2_pair_swiglu &&
+                    (g_test_flags &
+                     DS4_GPU_TEST_REQUIRE_IQ2_TOP8_PAIR_SWIGLU) == 0u) {
+                    use_mm_id_pair_swiglu = false;
+                }
             }
             if (!map_pipeline || !gate_mm_pipeline || !up_mm_pipeline || !down_mm_pipeline ||
                 (use_mm_id_pair_swiglu && !pair_swiglu_mm_pipeline)) {
@@ -43155,7 +50970,8 @@ int ds4_gpu_routed_moe_batch_tensor(
                         &stream_unique,
                         &stream_overflow_gate,
                         &stream_overflow_up,
-                        &stream_overflow_down)) {
+                        &stream_overflow_down,
+                        false)) {
                 g_stream_prefill_batch_selected_addr_building--;
                 return 0;
             }
@@ -43297,6 +51113,14 @@ int ds4_gpu_routed_moe_batch_tensor(
         int owned = 0;
         id<MTLCommandBuffer> cb = ds4_gpu_command_buffer(&owned);
         if (!cb) return 0;
+        if (use_iq2_batch_addr_mm && stream_resource_count != 0 &&
+            !ds4_gpu_stream_expert_cache_mark_entries_inflight(
+                stream_resources, stream_resource_count, 0)) {
+            fprintf(stderr,
+                    "ds4: Metal IQ2_XXS SSD grouped address-MM could not pin "
+                    "expert resources for the command buffer\n");
+            return 0;
+        }
         if (use_q4_batch_expert_table && !ds4_gpu_use_model_residency_set(cb)) {
             return 0;
         }
@@ -43313,6 +51137,7 @@ int ds4_gpu_routed_moe_batch_tensor(
         const char *moe_stage_filter = getenv("DS4_METAL_MOE_STAGE_PROFILE_FILTER");
         const char *moe_path =
             use_q4_batch_expert_table ? "q4_table_pair_swiglu" :
+            use_iq2_batch_addr_mm ? "iq2_batch_stream_addr_mm" :
             use_iq2_batch_selected_addr ? "iq2_batch_stream_addr" :
             use_mm_id_pair_swiglu ? "mm_id_pair_swiglu" :
             use_mm_id ? "mm_id" :
@@ -43334,19 +51159,23 @@ int ds4_gpu_routed_moe_batch_tensor(
                 } else { \
                     const char *stage_name = (name); \
                     const double now_ms = ds4_gpu_now_ms(); \
+                    const double gpu_ms = \
+                        g_last_completed_gpu_time_valid ? \
+                            g_last_completed_gpu_seconds * 1000.0 : -1.0; \
                     const int print_stage = \
                         !moe_stage_filter || !moe_stage_filter[0] || \
                         strstr(stage_name, moe_stage_filter) != NULL; \
                     if (print_stage) { \
                         fprintf(stderr, \
                                 "ds4: Metal routed MoE stage layer=%u tokens=%u pairs=%u experts=%u " \
-                                "gate=%s down=%s path=%s mid=%s %s=%.3f ms\n", \
+                                "gate=%s down=%s path=%s mid=%s " \
+                                "%s=%.3f ms gpu=%.3f ms\n", \
                                 layer_index, n_tokens, pair_rows, n_expert, \
                                 ds4_gpu_metal_tensor_type_name(gate_type), \
                                 ds4_gpu_metal_tensor_type_name(down_type), \
                                 moe_path, \
                                 request_mid_f16 ? "f16" : "f32", \
-                                stage_name, now_ms - moe_stage_t0); \
+                                stage_name, now_ms - moe_stage_t0, gpu_ms); \
                     } \
                     moe_stage_t0 = now_ms; \
                     if (ds4_gpu_begin_commands() == 0) { \
@@ -43381,7 +51210,31 @@ int ds4_gpu_routed_moe_batch_tensor(
             n_tokens <= 4u &&
             down_sum6_pipeline != nil;
         int ok = 0;
-        if (use_iq2_batch_selected_addr) {
+        if (use_iq2_batch_addr_mm) {
+            ok = ds4_gpu_encode_mul_mm_id_map(cb,
+                                               map_pipeline,
+                                               &gate_map_args,
+                                               &gate_mm_args,
+                                               selectedbuf,
+                                               ds4_gpu_tensor_offset(selected));
+            DS4_METAL_PROFILE_MOE_STAGE("map");
+            if (ok) {
+                ok = ds4_gpu_encode_mul_mm_id_addr_mapped_tile(
+                        cb, gate_mm_pipeline, &gate_mm_args,
+                        stream_gate_addr_buf,
+                        xbuf, ds4_gpu_tensor_offset(x),
+                        gatebuf, ds4_gpu_tensor_offset(gate),
+                        8192u, stream_resources, stream_resource_count,
+                        0u, stream_overflow_gate) &&
+                     ds4_gpu_encode_mul_mm_id_addr_mapped_tile(
+                        cb, up_mm_pipeline, &gate_mm_args,
+                        stream_up_addr_buf,
+                        xbuf, ds4_gpu_tensor_offset(x),
+                        upbuf, ds4_gpu_tensor_offset(up),
+                        8192u, stream_resources, stream_resource_count,
+                        1u, stream_overflow_up);
+            }
+        } else if (use_iq2_batch_selected_addr) {
             ds4_gpu_dsv4_moe_swiglu_weight_args act_args = {
                 .width = expert_mid_dim,
                 .rows = pair_rows,
@@ -43460,6 +51313,10 @@ int ds4_gpu_routed_moe_batch_tensor(
                                                             false,
                                                             q4_batch_table_queue_residency);
         } else if (use_mm_id) {
+            /* kernel_mul_mm_id_mpp double-buffers tiles (12 KiB). */
+            const NSUInteger gate_up_tg =
+                (ds4_gpu_routed_mm_mpp_mask() != 0 &&
+                 gate_type == DS4_METAL_TENSOR_IQ2_XXS) ? 12288u : 8192u;
             /*
              * The routed pair ids are the same for gate, up, and down. Build
              * the expert-major work map once, then reuse it for all three
@@ -43509,7 +51366,7 @@ int ds4_gpu_routed_moe_batch_tensor(
                                                            ds4_gpu_tensor_offset(x),
                                                            gatebuf,
                                                            ds4_gpu_tensor_offset(gate),
-                                                           8192u);
+                                                           gate_up_tg);
                 DS4_METAL_PROFILE_MOE_STAGE("gate");
             }
             if (ok && !use_mm_id_pair_swiglu) {
@@ -43522,7 +51379,7 @@ int ds4_gpu_routed_moe_batch_tensor(
                                                    ds4_gpu_tensor_offset(x),
                                                    upbuf,
                                                    ds4_gpu_tensor_offset(up),
-                                                   8192u);
+                                                   gate_up_tg);
                 DS4_METAL_PROFILE_MOE_STAGE("up");
             }
         } else if (use_tiny_pair_swiglu) {
@@ -43619,7 +51476,7 @@ int ds4_gpu_routed_moe_batch_tensor(
             use_fused_activation &&
             request_mid_f16;
         if (mid_is_f16) *mid_is_f16 = use_mid_f16;
-        if (ok && use_iq2_batch_selected_addr) {
+        if (ok && use_iq2_batch_selected_addr && !use_iq2_batch_addr_mm) {
             /* The address-table pair kernel already wrote weighted SwiGLU rows into mid. */
         } else if (ok && use_q4_batch_expert_table) {
             /* The table pair kernel already wrote weighted SwiGLU rows into mid. */
@@ -43721,7 +51578,22 @@ int ds4_gpu_routed_moe_batch_tensor(
         NSUInteger down_dst_off = n_expert == 1 ? ds4_gpu_tensor_offset(out) :
             (expertsbuf ? ds4_gpu_tensor_offset(experts) : 0);
         if (ok) {
-            if (use_iq2_batch_selected_addr) {
+            if (use_iq2_batch_addr_mm) {
+                ok = ds4_gpu_encode_mul_mm_id_addr_mapped_tile(
+                        cb,
+                        down_mm_pipeline,
+                        &down_mm_args,
+                        stream_down_addr_buf,
+                        midbuf,
+                        ds4_gpu_tensor_offset(mid),
+                        down_dst,
+                        down_dst_off,
+                        8192u,
+                        stream_resources,
+                        stream_resource_count,
+                        2u,
+                        stream_overflow_down);
+            } else if (use_iq2_batch_selected_addr) {
                 ok = ds4_gpu_encode_mul_mv_addr_q2_sum6(
                         cb,
                         g_moe_mul_mv_addr_q2_k_sum6_pipeline,
@@ -43777,6 +51649,11 @@ int ds4_gpu_routed_moe_batch_tensor(
                                                      down_smem,
                                                      2);
             } else if (use_mm_id) {
+                const NSUInteger down_tg =
+                    ((ds4_gpu_routed_mm_mpp_mask() & 4) != 0 &&
+                     request_mid_f16 &&
+                     (down_type == DS4_METAL_TENSOR_Q2_K ||
+                      down_type == DS4_METAL_TENSOR_IQ2_XXS)) ? 12288u : 8192u;
                 ok = ds4_gpu_encode_mul_mm_id_mapped_tile(cb,
                                                        down_mm_pipeline,
                                                        &down_mm_args,
@@ -43786,7 +51663,7 @@ int ds4_gpu_routed_moe_batch_tensor(
                                                        ds4_gpu_tensor_offset(mid),
                                                        down_dst,
                                                        down_dst_off,
-                                                       8192u);
+                                                       down_tg);
             } else {
                 ok = ds4_gpu_encode_mul_mv_id(cb,
                                                      down_mv_pipeline,
@@ -43809,7 +51686,7 @@ int ds4_gpu_routed_moe_batch_tensor(
             n_expert > 1 &&
             !direct_down_sum &&
             !use_q4_batch_expert_table &&
-            !use_iq2_batch_selected_addr) {
+            (!use_iq2_batch_selected_addr || use_iq2_batch_addr_mm)) {
             ok = ds4_gpu_encode_moe_sum_experts(cb,
                                                        down_dst,
                                                        down_dst_off,
@@ -43853,6 +51730,9 @@ int ds4_gpu_routed_moe_batch_tensor(
                 }
             }
         }
+        if (use_iq2_batch_addr_mm) {
+            ds4_gpu_iq2_stream_addr_mm_stats_note_selected(n_tokens, n_expert);
+        }
         if (q4_batch_table_boundary) {
             if (ds4_gpu_end_commands() == 0 || ds4_gpu_begin_commands() == 0) {
                 return 0;
@@ -43946,6 +51826,7 @@ int ds4_gpu_hc_split_sinkhorn_tensor(
 
 static int ds4_gpu_hc_weighted_sum_strided(
         ds4_gpu_tensor       *out,
+        ds4_gpu_tensor       *last_out,
         const ds4_gpu_tensor *residual_hc,
         const ds4_gpu_tensor *weights,
         uint64_t                weight_offset,
@@ -43963,6 +51844,8 @@ static int ds4_gpu_hc_weighted_sum_strided(
         id<MTLBuffer> xbuf = ds4_gpu_tensor_buffer(residual_hc);
         id<MTLBuffer> wbuf = ds4_gpu_tensor_buffer(weights);
         id<MTLBuffer> outbuf = ds4_gpu_tensor_buffer(out);
+        id<MTLBuffer> lastbuf = last_out ?
+            ds4_gpu_tensor_buffer(last_out) : nil;
         const uint64_t out_row_bytes = (uint64_t)n_embd * sizeof(float);
         const uint64_t out_tensor_bytes = ds4_gpu_tensor_bytes(out);
         if (out_row_bytes == 0 || out_tensor_bytes < out_row_bytes || out_tensor_bytes % out_row_bytes != 0) {
@@ -43989,9 +51872,10 @@ static int ds4_gpu_hc_weighted_sum_strided(
         const uint64_t w_last = weight_offset +
                                 (n_tokens64 - 1u) * weight_row_stride +
                                 (uint64_t)n_hc * sizeof(float);
-        if (!xbuf || !wbuf || !outbuf ||
+        if (!xbuf || !wbuf || !outbuf || (last_out && !lastbuf) ||
             ds4_gpu_tensor_bytes(residual_hc) < x_bytes ||
-            ds4_gpu_tensor_bytes(weights) < w_last) {
+            ds4_gpu_tensor_bytes(weights) < w_last ||
+            (last_out && ds4_gpu_tensor_bytes(last_out) < out_row_bytes)) {
             fprintf(stderr, "ds4: Metal HC weighted sum received undersized activation buffers\n");
             return 0;
         }
@@ -44016,11 +51900,18 @@ static int ds4_gpu_hc_weighted_sum_strided(
         if (!cb) return 0;
 
         id<MTLComputeCommandEncoder> enc = ds4_gpu_compute_encoder(cb);
-        [enc setComputePipelineState:g_hc_weighted_sum_pipeline];
+        [enc setComputePipelineState:last_out ?
+             g_hc_weighted_sum_capture_last_pipeline :
+             g_hc_weighted_sum_pipeline];
         [enc setBytes:&args length:sizeof(args) atIndex:0];
         [enc setBuffer:xbuf offset:ds4_gpu_tensor_offset(residual_hc) atIndex:1];
         [enc setBuffer:wbuf offset:ds4_gpu_tensor_offset(weights) + (NSUInteger)weight_offset atIndex:2];
         [enc setBuffer:outbuf offset:ds4_gpu_tensor_offset(out) atIndex:3];
+        if (last_out) {
+            [enc setBuffer:lastbuf
+                     offset:ds4_gpu_tensor_offset(last_out)
+                    atIndex:4];
+        }
         [enc dispatchThreadgroups:MTLSizeMake(n_tg, 1, 1)
              threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];
         ds4_gpu_end_compute_encoder(cb, enc);
@@ -44038,6 +51929,7 @@ int ds4_gpu_hc_weighted_sum_tensor(
         uint32_t                n_embd,
         uint32_t                n_hc) {
     return ds4_gpu_hc_weighted_sum_strided(out,
+                                             NULL,
                                              residual_hc,
                                              weights,
                                              0,
@@ -44047,6 +51939,25 @@ int ds4_gpu_hc_weighted_sum_tensor(
                                              "HC weighted sum");
 }
 
+int ds4_gpu_hc_weighted_sum_capture_last_tensor(
+        ds4_gpu_tensor       *out,
+        ds4_gpu_tensor       *last_out,
+        const ds4_gpu_tensor *residual_hc,
+        const ds4_gpu_tensor *weights,
+        uint32_t                n_embd,
+        uint32_t                n_hc) {
+    if (!last_out) return 0;
+    return ds4_gpu_hc_weighted_sum_strided(out,
+                                             last_out,
+                                             residual_hc,
+                                             weights,
+                                             0,
+                                             (uint64_t)n_hc * sizeof(float),
+                                             n_embd,
+                                             n_hc,
+                                             "HC weighted sum/capture last");
+}
+
 int ds4_gpu_hc_weighted_sum_split_tensor(
         ds4_gpu_tensor       *out,
         const ds4_gpu_tensor *residual_hc,
@@ -44055,6 +51966,7 @@ int ds4_gpu_hc_weighted_sum_split_tensor(
         uint32_t                n_hc) {
     const uint64_t mix_hc = 2ull * n_hc + (uint64_t)n_hc * n_hc;
     return ds4_gpu_hc_weighted_sum_strided(out,
+                                             NULL,
                                              residual_hc,
                                              split,
                                              0,
@@ -44425,7 +52337,6 @@ int ds4_gpu_hc_rms_norm_mix_f16_tensor(
     return 1;
 }
 
-
 int ds4_gpu_hc_rms_norm_mix_split_norm_f16_tensor(
         ds4_gpu_tensor       *mix,
         ds4_gpu_tensor       *out,
@@ -45731,7 +53642,137 @@ int ds4_gpu_matmul_q8_0_hc_expand_tensor(
     return 1;
 }
 
+int ds4_gpu_matmul_q4_K_hc_expand_available(void) {
+    if (!g_initialized && !ds4_gpu_init()) return 0;
+    if (getenv("DS4_METAL_DISABLE_Q4_ATTN_OUT_HC_FUSE") != NULL) return 0;
+    return ds4_gpu_get_mul_mv_ext_pipeline(
+            "kernel_dsv4_q4_K_hc_expand4", 2, 8) != nil;
+}
 
+int ds4_gpu_matmul_q4_K_hc_expand_tensor(
+        ds4_gpu_tensor       *out_hc,
+        ds4_gpu_tensor       *block_out,
+        const void           *model_map,
+        uint64_t              model_size,
+        uint64_t              weight_offset,
+        uint64_t              in_dim,
+        uint64_t              out_dim,
+        const ds4_gpu_tensor *x,
+        const ds4_gpu_tensor *residual_hc,
+        const ds4_gpu_tensor *split,
+        uint32_t              n_embd,
+        uint32_t              n_hc) {
+    if (!g_initialized && !ds4_gpu_init()) return 0;
+    if (getenv("DS4_METAL_DISABLE_Q4_ATTN_OUT_HC_FUSE") != NULL) return 0;
+    if (!out_hc || !block_out || !model_map || !x || !residual_hc || !split ||
+        n_embd == 0u || n_hc != 4u || out_dim != n_embd ||
+        in_dim == 0u || (in_dim % 256u) != 0u ||
+        (out_dim & 1u) != 0u ||
+        in_dim > UINT32_MAX || out_dim > UINT32_MAX) {
+        return 0;
+    }
+
+    @autoreleasepool {
+        const uint64_t row_bytes = (in_dim / 256u) * 144u;
+        if (out_dim > UINT64_MAX / row_bytes) return 0;
+        const uint64_t weight_bytes = out_dim * row_bytes;
+        const uint64_t x_bytes = in_dim * sizeof(float);
+        const uint64_t embd_bytes = out_dim * sizeof(float);
+        const uint64_t hc_bytes = (uint64_t)n_hc * n_embd * sizeof(float);
+        const uint64_t mix_hc = 2ull * n_hc + (uint64_t)n_hc * n_hc;
+        const uint64_t split_bytes = mix_hc * sizeof(float);
+        if (weight_offset > model_size ||
+            weight_bytes > model_size - weight_offset) {
+            return 0;
+        }
+
+        id<MTLBuffer> xbuf = ds4_gpu_tensor_buffer(x);
+        id<MTLBuffer> blockbuf = ds4_gpu_tensor_buffer(block_out);
+        id<MTLBuffer> resbuf = ds4_gpu_tensor_buffer(residual_hc);
+        id<MTLBuffer> splitbuf = ds4_gpu_tensor_buffer(split);
+        id<MTLBuffer> outbuf = ds4_gpu_tensor_buffer(out_hc);
+        if (!xbuf || !blockbuf || !resbuf || !splitbuf || !outbuf ||
+            ds4_gpu_tensor_bytes(x) < x_bytes ||
+            ds4_gpu_tensor_bytes(block_out) < embd_bytes ||
+            ds4_gpu_tensor_bytes(residual_hc) < hc_bytes ||
+            ds4_gpu_tensor_bytes(split) < split_bytes ||
+            ds4_gpu_tensor_bytes(out_hc) < hc_bytes) {
+            return 0;
+        }
+
+        uint64_t inner_offset = 0;
+        id<MTLBuffer> wbuf = ds4_gpu_wrap_model_range(
+            model_map, model_size, weight_offset, weight_bytes,
+            &inner_offset);
+        if (!wbuf) return 0;
+
+        ds4_gpu_q8_0_matvec_args mv_args = {
+            .ne00 = (int32_t)in_dim, .ne01 = (int32_t)out_dim, .ne02 = 1,
+            .nb00 = 1, .nb01 = row_bytes,
+            .nb02 = row_bytes * out_dim, .nb03 = row_bytes * out_dim,
+            .ne10 = (int32_t)in_dim, .ne11 = 1, .ne12 = 1,
+            .nb10 = sizeof(float), .nb11 = in_dim * sizeof(float),
+            .nb12 = in_dim * sizeof(float), .nb13 = in_dim * sizeof(float),
+            .ne0 = (int32_t)out_dim, .ne1 = 1, .nr0 = 2,
+            .r2 = 1, .r3 = 1,
+        };
+        ds4_gpu_hc_expand_args hc_args = {
+            .n_embd = n_embd,
+            .n_hc = n_hc,
+            .n_tokens = 1,
+            .nb_block0 = sizeof(float),
+            .nb_block1 = (uint64_t)n_embd * sizeof(float),
+            .nb_add0 = sizeof(float),
+            .nb_add1 = (uint64_t)n_embd * sizeof(float),
+            .nb_res0 = sizeof(float),
+            .nb_res1 = (uint64_t)n_embd * sizeof(float),
+            .nb_res2 = (uint64_t)n_hc * n_embd * sizeof(float),
+            .nb_post0 = sizeof(float),
+            .nb_post1 = mix_hc * sizeof(float),
+            .nb_comb0 = sizeof(float),
+            .nb_comb1 = (uint64_t)n_hc * sizeof(float),
+            .nb_comb2 = mix_hc * sizeof(float),
+            .nb0 = sizeof(float),
+            .nb1 = (uint64_t)n_embd * sizeof(float),
+            .nb2 = (uint64_t)n_hc * n_embd * sizeof(float),
+            .has_add = 0,
+        };
+
+        const int16_t nsg = 2;
+        id<MTLComputePipelineState> pipeline = ds4_gpu_get_mul_mv_ext_pipeline(
+            "kernel_dsv4_q4_K_hc_expand4", nsg, 8);
+        if (!pipeline) return 0;
+
+        int owned = 0;
+        id<MTLCommandBuffer> cb = ds4_gpu_command_buffer(&owned);
+        if (!cb) return 0;
+        id<MTLComputeCommandEncoder> enc = ds4_gpu_compute_encoder(cb);
+        [enc setComputePipelineState:pipeline];
+        [enc setBytes:&mv_args length:sizeof(mv_args) atIndex:0];
+        [enc setBytes:&hc_args length:sizeof(hc_args) atIndex:1];
+        [enc setBuffer:wbuf offset:(NSUInteger)inner_offset atIndex:2];
+        [enc setBuffer:xbuf offset:ds4_gpu_tensor_offset(x) atIndex:3];
+        [enc setBuffer:blockbuf offset:ds4_gpu_tensor_offset(block_out) atIndex:4];
+        [enc setBuffer:resbuf offset:ds4_gpu_tensor_offset(residual_hc) atIndex:5];
+        [enc setBuffer:splitbuf
+                 offset:ds4_gpu_tensor_offset(split) +
+                        (NSUInteger)n_hc * sizeof(float)
+                atIndex:6];
+        [enc setBuffer:splitbuf
+                 offset:ds4_gpu_tensor_offset(split) +
+                        (NSUInteger)(2u * n_hc) * sizeof(float)
+                atIndex:7];
+        [enc setBuffer:outbuf offset:ds4_gpu_tensor_offset(out_hc) atIndex:8];
+        [enc setThreadgroupMemoryLength:32u atIndex:0];
+        [enc dispatchThreadgroups:MTLSizeMake(
+                ((NSUInteger)out_dim + 3u) / 4u, 1, 1)
+             threadsPerThreadgroup:MTLSizeMake(32, (NSUInteger)nsg, 1)];
+        ds4_gpu_end_compute_encoder(cb, enc);
+
+        return ds4_gpu_finish_command_buffer(
+            cb, owned, "Q4_K HC expand fused");
+    }
+}
 
 /* Kimi Delta Attention primitive shared with the GLM-5.3 graph. */
 static int glm53_gpu_mul_u64(uint64_t a, uint64_t b, uint64_t *out) {
@@ -47109,3 +55150,81 @@ int ds4_gpu_glm53_kda_prefill(
 void ds4_gpu_set_glm_mtp_verify_mode(bool enabled) {
     (void)enabled;
 }
+
+static bool ds4_gpu_tensor_prefixes_overlap(
+        const ds4_gpu_tensor *a,
+        uint64_t              a_bytes,
+        const ds4_gpu_tensor *b,
+        uint64_t              b_bytes) {
+    if (!a || !b || a_bytes == 0 || b_bytes == 0 ||
+        ds4_gpu_tensor_buffer(a) != ds4_gpu_tensor_buffer(b)) {
+        return false;
+    }
+    const uint64_t a_off = (uint64_t)ds4_gpu_tensor_offset(a);
+    const uint64_t b_off = (uint64_t)ds4_gpu_tensor_offset(b);
+    return a_off <= b_off ? b_off - a_off < a_bytes
+                          : a_off - b_off < b_bytes;
+}
+
+/* Encode a dense or quantized prefill matmul against an RHS that has already
+ * been rounded to F16.  Materializing that RHS once avoids repeating the
+ * conversion for every 64-row output tile without changing the MMA or F32
+ * accumulation schedule.  This helper deliberately does not submit the
+ * command buffer so callers can keep conversion and consumers ordered. */
+static int ds4_gpu_encode_f16_rhs_mm(
+        id<MTLCommandBuffer>       cb,
+        id<MTLComputePipelineState> pipeline,
+        id<MTLBuffer>               weights,
+        NSUInteger                  weights_offset,
+        const ds4_gpu_tensor       *rhs_f16,
+        ds4_gpu_tensor             *out,
+        uint64_t                    in_dim,
+        uint64_t                    out_dim,
+        uint64_t                    n_tok,
+        uint64_t                    row_bytes,
+        bool                        bc_out) {
+    if (!cb || !pipeline || !weights || !rhs_f16 || !out ||
+        in_dim == 0 || out_dim == 0 || n_tok == 0 ||
+        (in_dim % 256u) != 0 ||
+        in_dim > INT32_MAX || out_dim > INT32_MAX || n_tok > INT32_MAX ||
+        (row_bytes != 0 && out_dim > UINT64_MAX / row_bytes) ||
+        in_dim > UINT64_MAX / n_tok ||
+        in_dim * n_tok > UINT64_MAX / sizeof(uint16_t) ||
+        out_dim > UINT64_MAX / n_tok ||
+        out_dim * n_tok > UINT64_MAX / sizeof(float)) {
+        return 0;
+    }
+
+    const uint64_t rhs_bytes = in_dim * n_tok * sizeof(uint16_t);
+    const uint64_t out_bytes = out_dim * n_tok * sizeof(float);
+    id<MTLBuffer> rhs_buf = ds4_gpu_tensor_buffer(rhs_f16);
+    id<MTLBuffer> out_buf = ds4_gpu_tensor_buffer(out);
+    if (!rhs_buf || !out_buf ||
+        ds4_gpu_tensor_bytes(rhs_f16) < rhs_bytes ||
+        ds4_gpu_tensor_bytes(out) < out_bytes) {
+        return 0;
+    }
+
+    ds4_gpu_mul_mm_args args =
+        ds4_gpu_make_mm_args(in_dim, out_dim, n_tok, row_bytes);
+    args.nb10 = sizeof(uint16_t);
+    args.nb11 = in_dim * sizeof(uint16_t);
+    args.nb12 = in_dim * n_tok * sizeof(uint16_t);
+    args.nb13 = args.nb12;
+
+    id<MTLComputeCommandEncoder> enc = ds4_gpu_compute_encoder(cb);
+    if (!enc) return 0;
+    [enc setComputePipelineState:pipeline];
+    [enc setBytes:&args length:sizeof(args) atIndex:0];
+    [enc setBuffer:weights offset:weights_offset atIndex:1];
+    [enc setBuffer:rhs_buf offset:ds4_gpu_tensor_offset(rhs_f16) atIndex:2];
+    [enc setBuffer:out_buf offset:ds4_gpu_tensor_offset(out) atIndex:3];
+    [enc setThreadgroupMemoryLength:(bc_out ? 8192u : 6144u) atIndex:0];
+    [enc dispatchThreadgroups:
+            MTLSizeMake(((NSUInteger)n_tok + 31u) / 32u,
+                        ((NSUInteger)out_dim + 63u) / 64u,
+                        1)
+         threadsPerThreadgroup:MTLSizeMake(128, 1, 1)];
+    ds4_gpu_end_compute_encoder(cb, enc);
+    return 1;
+}
diff --git a/ds4_rocm.cu b/ds4_rocm.cu
index 566248edf0..425a3a2a09 100644
--- a/ds4_rocm.cu
+++ b/ds4_rocm.cu
@@ -1,6 +1,5 @@
 #ifdef __HIP_PLATFORM_AMD__
 #include "ds4_rocm.h"
-#include <hipblaslt/hipblaslt.h>
 #include <rocblas/rocblas.h>
 
 #define FULL_WARP_MASK 0xFFFFFFFFFFFFFFFFULL
@@ -66,7 +65,6 @@ extern "C" int ds4_mmq_q2_K_moe_down_sum6_vec(
 #endif
 
 #define CUDA_QK_K 256
-#define DS4_ROCM_UNUSED __attribute__((unused))
 
 enum {
     /* attention_decode_mixed_kernel stores raw-window scores plus visible
@@ -115,13 +113,6 @@ typedef struct {
 
 static_assert(sizeof(cuda_block_mxfp4) == 17, "cuda_block_mxfp4 must match the GGUF MXFP4 block layout");
 
-/* Twice the MXFP4 values so each 32-value sub-block can use signed-int8
- * dp4a; the factor of 1/2 is folded into the sub-block scale. */
-__device__ __constant__ static const int8_t cuda_mxfp4_values_x2[16] = {
-     0,  1,  2,  3,  4,  6,  8,  12,
-     0, -1, -2, -3, -4, -6, -8, -12,
-};
-
 #include "ds4_iq2_tables_cuda.inc"
 
 #include "rocm/ds4_rocm_runtime.cuh"
@@ -132,6 +123,8 @@ extern "C" int ds4_gpu_dspark_gfx1151_fast_path(void) {
     return ds4_rocm_is_gfx1151();
 }
 
+#include "rocm/ds4_rocm_q4_qb_sidecar.cuh"
+
 #include "rocm/ds4_rocm_q8.cuh"
 
 #include "rocm/ds4_rocm_norm_rope.cuh"
@@ -163,6 +156,8 @@ extern "C" int ds4_gpu_dspark_gfx1151_fast_path(void) {
 
 #include "rocm/ds4_rocm_moe.cuh"
 
+#include "rocm/ds4_rocm_q4.cuh"
+
 #include "rocm/ds4_rocm_moe_launch.cuh"
 
 #include "rocm/ds4_rocm_glm.cuh"
@@ -183,6 +178,42 @@ extern "C" int ds4_gpu_tp_gate_encode(uint32_t layer, uint32_t gate) {
     return 0;
 }
 
+/* The TP flag-fold and deferred kv-norm paths are Metal-only optimizations;
+ * non-Apple backends always take the plain fallback (add without the checked
+ * flag, kv norm always standalone). */
+extern "C" int ds4_gpu_add_tensor_tp_flag(
+        ds4_gpu_tensor       *out,
+        const ds4_gpu_tensor *a,
+        const ds4_gpu_tensor *b,
+        uint32_t              n,
+        uint32_t              layer,
+        uint32_t              gate) {
+    (void)layer; (void)gate;
+    return ds4_gpu_add_tensor(out, a, b, n);
+}
+
+extern "C" void ds4_gpu_tp_flag_fold_request(uint32_t layer, uint32_t gate) {
+    (void)layer; (void)gate;
+}
+
+extern "C" void ds4_gpu_dsv4_qkv_norm_defer_kv_next(void) {
+}
+
+extern "C" int ds4_gpu_kv_norm_task_pending(void) {
+    return 0;
+}
+
+extern "C" int ds4_gpu_kv_norm_task_flush(void) {
+    return 0;
+}
+
+extern "C" int ds4_gpu_kv_norm_task_begin_concurrent(void) {
+    return 0;
+}
+
+extern "C" void ds4_gpu_kv_norm_task_end_concurrent(void) {
+}
+
 extern "C" void ds4_gpu_tp_set_batch_exchange(ds4_gpu_tp_batch_exchange_fn fn) {
     (void)fn;
 }
diff --git a/ds4_rocm_compat.cu b/ds4_rocm_compat.cu
index 554745379d..7227bb8b2b 100644
--- a/ds4_rocm_compat.cu
+++ b/ds4_rocm_compat.cu
@@ -8,6 +8,11 @@
 #include "ds4_gpu.h"
 #include "ds4_gpu_args.h"
 
+extern "C" int ds4_rocm_matmul_q4_K_tensor(
+        ds4_gpu_tensor *out, const void *model_map, uint64_t model_size,
+        uint64_t weight_offset, uint64_t in_dim, uint64_t out_dim,
+        const ds4_gpu_tensor *x, uint64_t n_tok);
+
 ds4_gpu_ctx g_gpu[DS4_MAX_GPUS] = {};
 int g_n_gpus = 1;
 int g_gpu_peer_ok[DS4_MAX_GPUS][DS4_MAX_GPUS] = {{1}};
@@ -299,16 +304,16 @@ extern "C" int ds4_gpu_matmul_quant_tensor(
                                           weight_offset, in_dim, out_dim, x,
                                           n_tok);
     }
+    if (weight_type == 12u) {
+        return ds4_rocm_matmul_q4_K_tensor(out, model_map, model_size,
+                                           weight_offset, in_dim, out_dim, x,
+                                           n_tok);
+    }
     if (weight_type == 1u) {
         return ds4_gpu_matmul_f16_tensor(out, model_map, model_size,
                                          weight_offset, in_dim, out_dim, x,
                                          n_tok);
     }
-    if (weight_type == 12u) {
-        return ds4_gpu_matmul_q4_K_tensor(out, model_map, model_size,
-                                          weight_offset, in_dim, out_dim, x,
-                                          n_tok);
-    }
     return 0;
 }
 
diff --git a/ds4_rocm_unavailable.cu b/ds4_rocm_unavailable.cu
index 311c8004da..2282971fce 100644
--- a/ds4_rocm_unavailable.cu
+++ b/ds4_rocm_unavailable.cu
@@ -9,9 +9,7 @@
 
 ROCM_UNAVAILABLE_INT(ds4_gpu_add_xdev_tensor)
 ROCM_UNAVAILABLE_INT(ds4_gpu_attention_decode_rows_rope_tensor)
-ROCM_UNAVAILABLE_INT(ds4_gpu_attention_output_low_q4_K_slice_tensor)
 ROCM_UNAVAILABLE_INT(ds4_gpu_attention_output_low_q8_rows_exact_tensor)
-ROCM_UNAVAILABLE_INT(ds4_gpu_attention_output_q4_K_batch_tensor)
 ROCM_UNAVAILABLE_INT(ds4_gpu_attention_prefill_raw_heads_range_tensor)
 ROCM_UNAVAILABLE_INT(ds4_gpu_attention_prefill_static_mixed_heads_range_tensor)
 ROCM_UNAVAILABLE_INT(ds4_gpu_device_cache_support_tensors)
@@ -22,6 +20,7 @@ ROCM_UNAVAILABLE_INT(ds4_gpu_matmul_q8_0_kslice_hc_expand_add_tensor)
 ROCM_UNAVAILABLE_INT(ds4_gpu_matmul_q8_0_kslice_rows_tensor)
 ROCM_UNAVAILABLE_INT(ds4_gpu_matmul_q8_0_top1_tensor)
 ROCM_UNAVAILABLE_INT(ds4_gpu_matmul_quant_kslice_tensor)
+ROCM_UNAVAILABLE_INT(ds4_gpu_matmul_quant_rows_scalar_tensor)
 ROCM_UNAVAILABLE_INT(ds4_gpu_moe_handoff_pack_tensor)
 ROCM_UNAVAILABLE_INT(ds4_gpu_register_model_map_no_copy)
 ROCM_UNAVAILABLE_INT(ds4_gpu_register_support_map)
diff --git a/ds4_server.c b/ds4_server.c
index 50efbe6bc1..95b0a514fc 100644
--- a/ds4_server.c
+++ b/ds4_server.c
@@ -3286,13 +3286,6 @@ static char *render_live_tool_tail_for_syntax(server_model_syntax syntax,
     return render_deepseek_live_tool_tail(msgs, start, think_mode);
 }
 
-static DS4_SERVER_MAYBE_UNUSED char *render_live_tool_tail(
-        const chat_msgs *msgs, int start,
-        ds4_think_mode think_mode) {
-    return render_live_tool_tail_for_syntax(SERVER_MODEL_SYNTAX_DEEPSEEK,
-                                            msgs, start, NULL, think_mode);
-}
-
 static void chat_msg_collect_tool_call_ids(const chat_msg *m, stop_list *ids) {
     if (!m || !ids) return;
     id_list_push_unique(ids, m->tool_call_id);
diff --git a/ds4_tp.c b/ds4_tp.c
index 0c8f573a16..d09a962b49 100644
--- a/ds4_tp.c
+++ b/ds4_tp.c
@@ -388,18 +388,6 @@ bool ds4_tp_enabled(const ds4_tp_options *opt) {
     return opt && opt->role != DS4_TP_NONE;
 }
 
-void ds4_tp_usage(FILE *fp) {
-    fprintf(fp,
-        "Tensor parallelism (two identical machines):\n"
-        "  --tensor-parallel           Use --role/--listen/--coordinator for a 50/50 TP pair.\n"
-        "  --transport <auto|rdma|tcp> Gate transport (default auto).\n"
-        "  --rdma-device <name>        Select a verbs device such as rdma_en1.\n"
-        "  --rdma-gid-index <n>        Select the local verbs GID index.\n"
-        "  --tensor-parallel-token-prefill\n"
-        "                              GLM diagnostic: prefill one token at a time.\n"
-        "  --debug-hash <n>            Cross-check hidden state every n tokens.\n");
-}
-
 int ds4_tp_parse_cli_arg(
         const char *arg,
         int *index,
@@ -1889,7 +1877,6 @@ void ds4_tp_free(ds4_tp *tp) {
 
 int ds4_tp_rank(const ds4_tp *tp) { return tp->rank; }
 bool ds4_tp_is_rdma(const ds4_tp *tp) { return tp->rdma_active; }
-uint32_t ds4_tp_peer_ctx(const ds4_tp *tp) { return tp->peer_ctx; }
 bool ds4_tp_failed(const ds4_tp *tp) {
     return tp && atomic_load_explicit(&tp->failed, memory_order_acquire);
 }
@@ -2654,29 +2641,6 @@ int ds4_tp_recv_verify_commit(ds4_tp *tp, int32_t *mode, int32_t *token_count) {
     return 1;
 }
 
-int ds4_tp_hash_check(ds4_tp *tp, uint64_t seq, uint64_t hash, char *err, size_t errlen) {
-    struct { uint64_t seq; uint64_t hash; } mine = { seq, hash }, theirs;
-    if (!tp_send_frame(tp->control_fd, DS4_TP_FRAME_HASH, &mine, sizeof(mine))) {
-        tp_set_err(err, errlen, "tp: hash send failed");
-        return 0;
-    }
-    uint32_t type = 0, bytes = 0;
-    if (!tp_read_frame_header(tp->control_fd, &type, &bytes) ||
-        type != DS4_TP_FRAME_HASH || bytes != sizeof(theirs) ||
-        !tp_read_full(tp->control_fd, &theirs, sizeof(theirs))) {
-        tp_set_err(err, errlen, "tp: hash recv failed");
-        return 0;
-    }
-    if (theirs.seq != seq || theirs.hash != hash) {
-        tp_set_err(err, errlen,
-                   "tp: LOCKSTEP DIVERGENCE at seq %llu: local %016llx peer %016llx",
-                   (unsigned long long)seq,
-                   (unsigned long long)hash, (unsigned long long)theirs.hash);
-        return -1;
-    }
-    return 1;
-}
-
 /* ------------------------------------------------------------------------
  * Worker main loop.
  * --------------------------------------------------------------------- */
diff --git a/ds4_tp.h b/ds4_tp.h
index f810f578e1..4b5a60debf 100644
--- a/ds4_tp.h
+++ b/ds4_tp.h
@@ -4,7 +4,6 @@
 #include <stdbool.h>
 #include <stddef.h>
 #include <stdint.h>
-#include <stdio.h>
 
 #include "ds4.h"
 
@@ -81,8 +80,6 @@ int ds4_tp_adopt_distributed_options(
         ds4_distributed_options *dist,
         char *err,
         size_t errlen);
-void ds4_tp_usage(FILE *fp);
-
 /* Validates option combinations that TP cannot run with (SSD streaming,
  * distributed mode, MTP drafting, CPU backend). */
 int ds4_tp_validate_engine_options(
@@ -103,7 +100,6 @@ void ds4_tp_free(ds4_tp *tp);
 
 int ds4_tp_rank(const ds4_tp *tp);
 bool ds4_tp_is_rdma(const ds4_tp *tp);
-uint32_t ds4_tp_peer_ctx(const ds4_tp *tp);
 bool ds4_tp_failed(const ds4_tp *tp);
 void ds4_tp_mark_failed(ds4_tp *tp);
 
@@ -187,9 +183,7 @@ typedef enum {
     DS4_TP_FRAME_REWIND = 3,
     DS4_TP_FRAME_INVALIDATE = 4,
     DS4_TP_FRAME_STOP = 5,
-    DS4_TP_FRAME_HASH = 6,
     DS4_TP_FRAME_RDMA_INFO = 7,
-    DS4_TP_FRAME_SYNC_ACK = 8,
     DS4_TP_FRAME_RDMA_READY = 9,
     DS4_TP_FRAME_LOGITS = 10,
     DS4_TP_FRAME_VERIFY = 11,
@@ -224,10 +218,6 @@ int ds4_tp_recv_command(
         size_t errlen);
 void ds4_tp_command_free(ds4_tp_command *command);
 
-/* Debug lockstep check: both sides send their hidden-state hash for a token
- * and compare.  Returns 0 on transport failure, -1 on hash mismatch. */
-int ds4_tp_hash_check(ds4_tp *tp, uint64_t seq, uint64_t hash, char *err, size_t errlen);
-
 /* Vocab-split output head: the worker ships its logits half to the leader
  * after every eval (and after a sync) on the control socket. */
 int ds4_tp_send_logits_half(ds4_tp *tp, const float *half, uint32_t count);
diff --git a/gguf-tools/Makefile b/gguf-tools/Makefile
index 37d5d2cdab..53357faf90 100644
--- a/gguf-tools/Makefile
+++ b/gguf-tools/Makefile
@@ -32,10 +32,10 @@ endif
 NVCCFLAGS ?= -O3 --use_fast_math $(NVCC_ARCH_FLAGS) -Xcompiler $(NATIVE_CPU_FLAG) -Xcompiler -pthread
 QUALITY_LDLIBS ?= -lm -Xcompiler -pthread -L$(CUDA_HOME)/targets/sbsa-linux/lib -L$(CUDA_HOME)/lib64 -lcudart -lcublas
 QUALITY_TARGETS := ds4.o ds4_cuda.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o ds4_image.o rax.o ds4_gpu_args.o \
-	cuda/mmq/ds4_ggml_stubs.o cuda/mmq/ds4_mmq.o cuda/mmq/ds4_mmq_d2r.o \
+	cuda/mmq/ds4_ggml_stubs.o cuda/mmq/ds4_mmq.o cuda/mmq/ds4_mmq_d2r.o cuda/mmq/ds4_mmq_q4_16warp.o \
 	cuda/mmq/quantize.o cuda/mmq/mmid.o cuda/mmq/mmvq.o cuda/mmq/ds4_repack.o
 QUALITY_OBJS := ../ds4.o ../ds4_cuda.o ../ds4_distributed.o ../ds4_tp.o ../ds4_ssd.o ../ds4_layer_pack.o ../ds4_image.o ../rax.o ../ds4_gpu_args.o \
-	../cuda/mmq/ds4_ggml_stubs.o ../cuda/mmq/ds4_mmq.o ../cuda/mmq/ds4_mmq_d2r.o \
+	../cuda/mmq/ds4_ggml_stubs.o ../cuda/mmq/ds4_mmq.o ../cuda/mmq/ds4_mmq_d2r.o ../cuda/mmq/ds4_mmq_q4_16warp.o \
 	../cuda/mmq/quantize.o ../cuda/mmq/mmid.o ../cuda/mmq/mmvq.o ../cuda/mmq/ds4_repack.o
 QUALITY_LINK := $(NVCC) $(NVCCFLAGS) -I.. -o
 LLAMA_CPP_LDLIBS ?= -L$(LLAMA_CPP_BIN) -Wl,-rpath,$(abspath $(LLAMA_CPP_BIN)) \
diff --git a/gguf-tools/README.md b/gguf-tools/README.md
index f8d68410a6..e2d52f01a4 100644
--- a/gguf-tools/README.md
+++ b/gguf-tools/README.md
@@ -66,6 +66,47 @@ family, plus enough free disk for the temporary output.  Use `--dry-run` and
 `--compare-tensor` before starting a full write, and use `--overwrite` only when
 you really mean to replace an existing GGUF.
 
+### Requantize a GGUF Directly
+
+Dense attention projections can be requantized directly from an existing GGUF
+without the original Hugging Face safetensors. Direct requantization supports
+`Q8_0 -> Q4_K` and `F16 -> Q4_K`; tensors not selected by the policy are copied
+byte for byte. The output path must differ from the source path. For a full run,
+write to a temporary output name and rename it only after validation; an
+interrupted run leaves a partial output file.
+
+Validate the plan and all required imatrix entries first:
+
+```sh
+gguf-tools/deepseek4-quantize \
+  --source-gguf /path/to/DeepSeek-V4-Flash-AProjQ8.gguf \
+  --attention-proj q4_k \
+  --imatrix /path/to/DeepSeek-V4-Flash-chat-v2-routed-and-dense-ds4-220k.dat \
+  --imatrix-strict \
+  --dry-run
+```
+
+Then write the Q4 GGUF:
+
+```sh
+gguf-tools/deepseek4-quantize \
+  --source-gguf /path/to/DeepSeek-V4-Flash-AProjQ8.gguf \
+  --out /path/to/DeepSeek-V4-Flash-AProjQ4.gguf \
+  --attention-proj q4_k \
+  --imatrix /path/to/DeepSeek-V4-Flash-chat-v2-routed-and-dense-ds4-220k.dat \
+  --imatrix-strict
+```
+
+Quantize only the sparse-attention indexer query projections while preserving
+the F16 indexer compressors and weight projection:
+
+```sh
+gguf-tools/deepseek4-quantize \
+  --source-gguf /path/to/DeepSeek-V4-Flash-AProjQ4.gguf \
+  --out /path/to/DeepSeek-V4-Flash-AProjQ4-IndexerQ4.gguf \
+  --indexer-q q4_k
+```
+
 Q2 routed experts with imatrix:
 
 ```sh
@@ -103,6 +144,7 @@ You can override tensor families:
 --experts iq2_xxs
 --routed-w2 q2_k
 --attention-proj q8_0
+--indexer-q q4_k
 --shared q8_0
 --output q8_0
 ```
diff --git a/gguf-tools/deepseek4-quantize.c b/gguf-tools/deepseek4-quantize.c
index 59fe4df9f3..e4ff06687a 100644
--- a/gguf-tools/deepseek4-quantize.c
+++ b/gguf-tools/deepseek4-quantize.c
@@ -35,6 +35,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <sys/stat.h>
 #include <pthread.h>
 
 #if defined(_WIN32)
@@ -1249,17 +1250,22 @@ typedef struct {
 
 typedef struct {
     ds4q_type routed_w1, routed_w2, routed_w3;
-    ds4q_type attention_proj, attention, shared, embedding, output, dense;
+    ds4q_type attention_proj, indexer_q, attention, shared, embedding, output, dense;
     type_override *overrides;
     int n_overrides;
 } quant_policy;
 
 static bool is_attention_projection(const char *name) {
+    if (strstr(name, ".indexer.")) return false;
     return strstr(name, ".attn_kv.weight") || strstr(name, ".attn_q_a.weight") ||
            strstr(name, ".attn_q_b.weight") || strstr(name, ".attn_output_a.weight") ||
            strstr(name, ".attn_output_b.weight");
 }
 
+static bool is_indexer_q_projection(const char *name) {
+    return strstr(name, ".indexer.attn_q_b.weight") != NULL;
+}
+
 static bool is_attention_tensor(const char *name) {
     return strstr(name, ".attn") || strstr(name, "attn_") || strstr(name, ".indexer") || strstr(name, "indexer_");
 }
@@ -1309,6 +1315,7 @@ static ds4q_type policy_type(const quant_policy *p, const char *name, const tens
     if (strcmp(name, "token_embd.weight") == 0 && p->embedding != DS4Q_TYPE_COUNT) return p->embedding;
     if (is_output_tensor(name) && p->output != DS4Q_TYPE_COUNT) return p->output;
     if (is_shared_expert(name) && p->shared != DS4Q_TYPE_COUNT) return p->shared;
+    if (is_indexer_q_projection(name) && p->indexer_q != DS4Q_TYPE_COUNT) return p->indexer_q;
     if (is_attention_projection(name) && p->attention_proj != DS4Q_TYPE_COUNT) return p->attention_proj;
     if (is_attention_tensor(name) && p->attention != DS4Q_TYPE_COUNT) return p->attention;
     if (p->dense != DS4Q_TYPE_COUNT) return p->dense;
@@ -1794,7 +1801,7 @@ static gguf_file load_gguf_metadata_with_override(const char *path,
     if (!fp) die_errno("open GGUF", path);
     char magic[4];
     if (fread(magic, 1, sizeof(magic), fp) != sizeof(magic) || memcmp(magic, "GGUF", 4) != 0) {
-        die("bad GGUF template");
+        die("bad GGUF");
     }
     g.version = read_u32_le_fp(fp, "GGUF version");
     g.n_tensors = read_u64_le_fp(fp, "GGUF tensor count");
@@ -1991,7 +1998,7 @@ static output_context build_output_context(const gguf_file *tmpl, const quant_po
                                            const hf_model_metadata *metadata) {
     output_context out = {0};
     out.n_tensors = tmpl->n_tensors;
-    out.n_kv_extra = extra_hf_metadata_kv_count(metadata) +
+    out.n_kv_extra = (metadata ? extra_hf_metadata_kv_count(metadata) : 0u) +
                      extra_imatrix_kv_count(im);
     out.alignment = tmpl->alignment;
     out.tensors = xcalloc((size_t)out.n_tensors, sizeof(out.tensors[0]));
@@ -2021,7 +2028,8 @@ static output_context build_output_context(const gguf_file *tmpl, const quant_po
     }
     out.tensor_bytes = off;
     out.meta_size = 4 + 4 + 8 + 8 + tmpl->kv_raw_len +
-                    extra_hf_metadata_kv_size(metadata) + extra_imatrix_kv_size(im) + tensor_info;
+                    (metadata ? extra_hf_metadata_kv_size(metadata) : 0u) +
+                    extra_imatrix_kv_size(im) + tensor_info;
     out.data_offset = ds4q_pad(out.meta_size, tmpl->alignment);
     return out;
 }
@@ -2080,6 +2088,167 @@ static void write_full_gguf(st_db *db, const gguf_file *tmpl, const output_conte
     fclose(fp);
 }
 
+static void copy_bytes(FILE *dst, FILE *src, uint64_t n, const char *path) {
+    uint8_t buf[1u << 20];
+    while (n) {
+        size_t chunk = n < sizeof(buf) ? (size_t)n : sizeof(buf);
+        if (fread(buf, 1, chunk, src) != chunk) die_errno("read source GGUF", path);
+        if (fwrite(buf, 1, chunk, dst) != chunk) die("write output tensor failed");
+        n -= chunk;
+    }
+}
+
+static void dequantize_q8_0_rows(const uint8_t *src, float *dst,
+                                 int64_t nrows, int64_t ncols) {
+    const int64_t blocks = ncols / 32;
+    for (int64_t r = 0; r < nrows; r++) {
+        const uint8_t *row = src + (size_t)r * (size_t)blocks * 34u;
+        float *out = dst + (size_t)r * (size_t)ncols;
+        for (int64_t b = 0; b < blocks; b++) {
+            uint16_t hd;
+            memcpy(&hd, row + (size_t)b * 34u, sizeof(hd));
+            const float d = ds4q_f16_to_f32(hd);
+            const int8_t *qs = (const int8_t *)(row + (size_t)b * 34u + 2u);
+            for (int j = 0; j < 32; j++) out[b * 32 + j] = d * (float)qs[j];
+        }
+    }
+}
+
+static void dequantize_f16_rows(const uint8_t *src, float *dst,
+                                int64_t nrows, int64_t ncols) {
+    const size_t row_size = (size_t)ncols * sizeof(uint16_t);
+    for (int64_t r = 0; r < nrows; r++) {
+        const uint8_t *row = src + (size_t)r * row_size;
+        float *out = dst + (size_t)r * (size_t)ncols;
+        for (int64_t c = 0; c < ncols; c++) {
+            out[c] = ds4q_f16_to_f32(load_u16_le(row + (size_t)c * sizeof(uint16_t)));
+        }
+    }
+}
+
+static bool direct_requant_supported(const tensor_meta *src,
+                                     const tensor_meta *dst) {
+    const int64_t block_size = ds4q_block_size(DS4Q_TYPE_Q4_K);
+    return dst->type == DS4Q_TYPE_Q4_K &&
+           (src->type == DS4Q_TYPE_Q8_0 || src->type == DS4Q_TYPE_F16) &&
+           block_size > 0 && src->ne[0] > 0 && src->ne[0] % block_size == 0;
+}
+
+static void validate_requant_plan(const gguf_file *src_g,
+                                  const output_context *out_ctx,
+                                  const imatrix_store *imatrix) {
+    size_t changed = 0;
+    size_t changed_q8 = 0;
+    size_t changed_f16 = 0;
+    for (uint64_t i = 0; i < out_ctx->n_tensors; i++) {
+        const tensor_meta *src = &src_g->tensors[i];
+        const tensor_meta *dst = &out_ctx->tensors[i];
+        if (src->type == dst->type) continue;
+        changed++;
+        if (!direct_requant_supported(src, dst)) {
+            fprintf(stderr,
+                    "error: direct requantization unsupported for %s (%s -> %s)\n",
+                    src->name, ds4q_type_name(src->type),
+                    ds4q_type_name(dst->type));
+            exit(1);
+        }
+        if (src->type == DS4Q_TYPE_Q8_0) changed_q8++;
+        else changed_f16++;
+        const char *names[1] = { src->name };
+        (void)imatrix_find(imatrix, names, 1, src->ne[0], -1, 0);
+    }
+    if (changed == 0) {
+        die("direct requantization plan does not change any tensors");
+    }
+    fprintf(stderr,
+            "validated direct GGUF requantization plan: %zu tensors -> Q4_K "
+            "(%zu Q8_0, %zu F16)\n",
+            changed, changed_q8, changed_f16);
+}
+
+static void write_requant_gguf(const gguf_file *src_g, const output_context *out_ctx,
+                               const char *out_path, const imatrix_store *imatrix) {
+    FILE *src_fp = fopen(src_g->path, "rb");
+    if (!src_fp) die_errno("open source GGUF", src_g->path);
+    FILE *fp = fopen(out_path, "wb");
+    if (!fp) die_errno("open output", out_path);
+    if (fwrite("GGUF", 1, 4, fp) != 4) die("write GGUF magic failed");
+    write_u32(fp, src_g->version);
+    write_u64(fp, src_g->n_tensors);
+    write_u64(fp, src_g->n_kv + out_ctx->n_kv_extra);
+    if (fwrite(src_g->kv_raw, 1, src_g->kv_raw_len, fp) != src_g->kv_raw_len) {
+        die("write GGUF KV failed");
+    }
+    write_imatrix_kvs(fp, imatrix);
+    for (uint64_t i = 0; i < out_ctx->n_tensors; i++) {
+        const tensor_meta *t = &out_ctx->tensors[i];
+        write_gguf_string(fp, t->name);
+        write_u32(fp, (uint32_t)t->n_dims);
+        for (int j = 0; j < t->n_dims; j++) write_u64(fp, (uint64_t)t->ne[j]);
+        write_u32(fp, (uint32_t)t->type);
+        write_u64(fp, t->new_offset);
+    }
+    off_t pos = ftello(fp);
+    if (pos < 0 || (size_t)pos > out_ctx->data_offset) die("bad output metadata size");
+    write_padding(fp, out_ctx->data_offset - (size_t)pos);
+
+    for (uint64_t i = 0; i < out_ctx->n_tensors; i++) {
+        const tensor_meta *src = &src_g->tensors[i];
+        const tensor_meta *dst = &out_ctx->tensors[i];
+        fprintf(stderr, "[%4" PRIu64 "/%4" PRIu64 "] %s: %s -> %s\n",
+                i + 1, out_ctx->n_tensors, src->name,
+                ds4q_type_name(src->type), ds4q_type_name(dst->type));
+        if (fseeko(src_fp, (off_t)(src_g->data_offset + src->old_offset), SEEK_SET) != 0) {
+            die_errno("seek source GGUF", src_g->path);
+        }
+        if (src->type == dst->type) {
+            copy_bytes(fp, src_fp, src->size, src_g->path);
+        } else {
+            if (!direct_requant_supported(src, dst)) {
+                fprintf(stderr, "error: direct requantization unsupported for %s (%s -> %s)\n",
+                        src->name, ds4q_type_name(src->type), ds4q_type_name(dst->type));
+                exit(1);
+            }
+            int64_t nrows = 1;
+            for (int d = 1; d < src->n_dims; d++) nrows *= src->ne[d];
+            const int64_t ncols = src->ne[0];
+            const size_t src_row = ds4q_row_size(src->type, ncols);
+            const size_t q4_row = ds4q_row_size(DS4Q_TYPE_Q4_K, ncols);
+            const int64_t batch_cap = 16;
+            uint8_t *encoded = xmalloc((size_t)batch_cap * src_row);
+            float *f32 = xmalloc((size_t)batch_cap * (size_t)ncols * sizeof(float));
+            uint8_t *q4 = xmalloc((size_t)batch_cap * q4_row);
+            const char *names[1] = { src->name };
+            const float *imat = imatrix_find(imatrix, names, 1, ncols, -1, 0);
+            ds4q_quantize_init(DS4Q_TYPE_Q4_K);
+            for (int64_t row0 = 0; row0 < nrows; row0 += batch_cap) {
+                const int64_t nr = nrows - row0 < batch_cap ? nrows - row0 : batch_cap;
+                if (fread(encoded, src_row, (size_t)nr, src_fp) != (size_t)nr) {
+                    die_errno("read source tensor", src_g->path);
+                }
+                if (src->type == DS4Q_TYPE_Q8_0) {
+                    dequantize_q8_0_rows(encoded, f32, nr, ncols);
+                } else {
+                    dequantize_f16_rows(encoded, f32, nr, ncols);
+                }
+                const size_t wrote = ds4q_quantize_chunk(
+                    DS4Q_TYPE_Q4_K, f32, q4, 0, nr, ncols, imat);
+                if (wrote != (size_t)nr * q4_row ||
+                    fwrite(q4, 1, wrote, fp) != wrote) {
+                    die("Q4_K requantization write failed");
+                }
+            }
+            free(q4);
+            free(f32);
+            free(encoded);
+        }
+        const size_t padded = ds4q_pad(dst->size, out_ctx->alignment);
+        write_padding(fp, padded - dst->size);
+    }
+    if (fclose(fp) != 0) die_errno("close output", out_path);
+    fclose(src_fp);
+}
+
 static void print_plan(const gguf_file *tmpl, const output_context *out_ctx) {
     size_t tensor_bytes = 0;
     size_t changed = 0;
@@ -2129,6 +2298,7 @@ static void dspark_support_defaults(dspark_support_options *o) {
 
 typedef struct {
     char *hf_dir;
+    char *source_gguf;
     char *template_gguf;
     char *out_gguf;
     char *compare_gguf;
@@ -2863,11 +3033,12 @@ static void free_dspark_support_plan(dspark_support_plan *plan) {
 }
 
 static void usage(const char *argv0) {
-    printf("usage: %s --hf DIR --template MODEL.gguf --out OUT.gguf [options]\n", argv0);
+    printf("usage: %s (--hf DIR --template MODEL.gguf | --source-gguf MODEL.gguf) --out OUT.gguf [options]\n", argv0);
     printf("\nDeepSeek V4 Flash/Pro safetensors -> GGUF quantizer in plain C.\n\n");
     printf("options:\n");
     printf("  --hf DIR               Hugging Face model directory with model.safetensors.index.json\n");
-    printf("  --template FILE        existing DS4 GGUF used for metadata, tensor order, shapes\n");
+    printf("  --source-gguf FILE     requantize Q8_0/F16 tensors directly to Q4_K\n");
+    printf("  --template FILE        GGUF metadata/layout template required with --hf\n");
     printf("  --out FILE             output GGUF path\n");
     printf("  --compare-gguf FILE    reference GGUF for --compare-tensor; normal mode defaults to template\n");
     printf("  --compare-tensor NAME  regenerate one tensor, checksum, optionally byte-compare, and exit\n");
@@ -2887,6 +3058,7 @@ static void usage(const char *argv0) {
     printf("  --routed-w2 TYPE       routed down expert tensor type\n");
     printf("  --routed-w3 TYPE       routed up expert tensor type\n");
     printf("  --attention-proj TYPE  attn_q/kv/output projection type\n");
+    printf("  --indexer-q TYPE       indexer query projection type\n");
     printf("  --attention TYPE       other 2D attention/indexer/compressor type\n");
     printf("  --shared TYPE          shared expert tensor type\n");
     printf("  --embedding TYPE       token embedding type\n");
@@ -2950,10 +3122,45 @@ static bool file_exists(const char *path) {
     return true;
 }
 
+static bool same_existing_file(const char *a, const char *b) {
+    struct stat sa;
+    struct stat sb;
+    if (!a || !b || stat(a, &sa) != 0 || stat(b, &sb) != 0) return false;
+    return sa.st_dev == sb.st_dev && sa.st_ino == sb.st_ino;
+}
+
+static void require_complete_gguf(const gguf_file *g) {
+    uint64_t required = (uint64_t)g->data_offset;
+    for (uint64_t i = 0; i < g->n_tensors; i++) {
+        const tensor_meta *t = &g->tensors[i];
+        if (t->old_offset > UINT64_MAX - (uint64_t)t->size ||
+            (uint64_t)g->data_offset >
+                UINT64_MAX - (t->old_offset + (uint64_t)t->size)) {
+            die("GGUF tensor extent overflows file size");
+        }
+        const uint64_t end =
+            (uint64_t)g->data_offset + t->old_offset + (uint64_t)t->size;
+        if (end > required) required = end;
+    }
+
+    struct stat st;
+    if (stat(g->path, &st) != 0) die_errno("stat source GGUF", g->path);
+    if (st.st_size < 0 || (uint64_t)st.st_size < required) {
+        fprintf(stderr,
+                "error: source GGUF is incomplete: %s has %" PRIu64
+                " bytes, needs at least %" PRIu64 "\n",
+                g->path,
+                st.st_size < 0 ? 0 : (uint64_t)st.st_size,
+                required);
+        exit(1);
+    }
+}
+
 static params parse_args(int argc, char **argv) {
     params p = {0};
     p.policy.routed_w1 = p.policy.routed_w2 = p.policy.routed_w3 = DS4Q_TYPE_COUNT;
-    p.policy.attention_proj = p.policy.attention = p.policy.shared = DS4Q_TYPE_COUNT;
+    p.policy.attention_proj = p.policy.indexer_q = DS4Q_TYPE_COUNT;
+    p.policy.attention = p.policy.shared = DS4Q_TYPE_COUNT;
     p.policy.embedding = p.policy.output = p.policy.dense = DS4Q_TYPE_COUNT;
     p.n_experts = 0;
     p.n_threads = 8;
@@ -2966,6 +3173,8 @@ static params parse_args(int argc, char **argv) {
             exit(0);
         } else if (strcmp(arg, "--hf") == 0) {
             p.hf_dir = need_value(argc, argv, &i, arg);
+        } else if (strcmp(arg, "--source-gguf") == 0) {
+            p.source_gguf = need_value(argc, argv, &i, arg);
         } else if (strcmp(arg, "--template") == 0) {
             p.template_gguf = need_value(argc, argv, &i, arg);
         } else if (strcmp(arg, "--out") == 0) {
@@ -3007,6 +3216,8 @@ static params parse_args(int argc, char **argv) {
             p.policy.routed_w3 = parse_type(need_value(argc, argv, &i, arg));
         } else if (strcmp(arg, "--attention-proj") == 0 || strcmp(arg, "--attn-proj") == 0) {
             p.policy.attention_proj = parse_type(need_value(argc, argv, &i, arg));
+        } else if (strcmp(arg, "--indexer-q") == 0) {
+            p.policy.indexer_q = parse_type(need_value(argc, argv, &i, arg));
         } else if (strcmp(arg, "--attention") == 0) {
             p.policy.attention = parse_type(need_value(argc, argv, &i, arg));
         } else if (strcmp(arg, "--shared") == 0) {
@@ -3033,7 +3244,21 @@ static params parse_args(int argc, char **argv) {
             exit(1);
         }
     }
-    if (!p.hf_dir) die("--hf is required");
+    if (!!p.hf_dir == !!p.source_gguf) {
+        die("exactly one of --hf or --source-gguf is required");
+    }
+    if (p.source_gguf && (p.dspark_manifest || p.dspark_support)) {
+        die("--source-gguf is not supported for DSpark modes");
+    }
+    if (p.source_gguf && p.template_gguf) {
+        die("--template is not used with --source-gguf");
+    }
+    if (p.source_gguf && p.compare_tensor) {
+        die("--compare-tensor is not supported with --source-gguf");
+    }
+    if (p.imatrix_strict && !p.imatrix_file) {
+        die("--imatrix-strict requires --imatrix");
+    }
     if (p.dspark_manifest && p.dspark_support) die("--dspark-manifest and --dspark-support are mutually exclusive");
     if (p.dspark_manifest) return p;
     if (p.dspark_support) {
@@ -3045,9 +3270,13 @@ static params parse_args(int argc, char **argv) {
         }
         return p;
     }
-    if (!p.template_gguf) die("--template is required");
+    if (p.hf_dir && !p.template_gguf) die("--template is required with --hf");
     if (!p.dry_run && !p.compare_tensor && !p.out_gguf) die("--out is required unless --dry-run or --compare-tensor is used");
     if (p.compare_tensor && !p.compare_gguf) p.compare_gguf = p.template_gguf;
+    if (p.source_gguf && p.out_gguf &&
+        same_existing_file(p.source_gguf, p.out_gguf)) {
+        die("--out must differ from --source-gguf");
+    }
     if (p.out_gguf && file_exists(p.out_gguf) && !p.overwrite) die("output exists; use --overwrite");
     return p;
 }
@@ -3197,13 +3426,19 @@ int main(int argc, char **argv) {
         return 0;
     }
 
-    hf_model_metadata metadata = load_hf_model_metadata(p.hf_dir,
-                                                        p.source_revision);
-    gguf_file tmpl = load_gguf_metadata_with_override(p.template_gguf, &metadata);
+    hf_model_metadata metadata = {0};
+    gguf_file tmpl;
+    if (p.source_gguf) {
+        tmpl = load_gguf_metadata(p.source_gguf);
+        require_complete_gguf(&tmpl);
+    } else {
+        metadata = load_hf_model_metadata(p.hf_dir, p.source_revision);
+        tmpl = load_gguf_metadata_with_override(p.template_gguf, &metadata);
+    }
     if (p.n_experts <= 0) {
         if (tmpl.n_experts > 0) {
             p.n_experts = tmpl.n_experts;
-            fprintf(stderr, "using %d routed experts from template metadata\n", p.n_experts);
+            fprintf(stderr, "using %d routed experts from GGUF metadata\n", p.n_experts);
         } else {
             p.n_experts = 256;
             fprintf(stderr, "warning: template has no deepseek4.expert_count; using Flash default %d routed experts\n", p.n_experts);
@@ -3211,9 +3446,16 @@ int main(int argc, char **argv) {
     } else {
         fprintf(stderr, "using %d routed experts from --n-experts\n", p.n_experts);
     }
-    output_context out_ctx = build_output_context(&tmpl, &p.policy, &imatrix, &metadata);
+    output_context out_ctx = build_output_context(
+        &tmpl, &p.policy, &imatrix, p.source_gguf ? NULL : &metadata);
+    if (p.source_gguf) {
+        validate_requant_plan(&tmpl, &out_ctx, &imatrix);
+    }
     print_plan(&tmpl, &out_ctx);
-    printf("compress_ratios: source=config.json count=%" PRIu64 "\n", metadata.n_compress_ratios);
+    if (!p.source_gguf) {
+        printf("compress_ratios: source=config.json count=%" PRIu64 "\n",
+               metadata.n_compress_ratios);
+    }
     if (p.dry_run) {
         free_hf_model_metadata(&metadata);
         imatrix_free(&imatrix);
@@ -3224,6 +3466,18 @@ int main(int argc, char **argv) {
         return 0;
     }
 
+    if (p.source_gguf) {
+        write_requant_gguf(&tmpl, &out_ctx, p.out_gguf, &imatrix);
+        fprintf(stderr, "wrote %s\n", p.out_gguf);
+        free_hf_model_metadata(&metadata);
+        imatrix_free(&imatrix);
+        free_gguf_file(&tmpl);
+        free(out_ctx.tensors);
+        for (int i = 0; i < p.policy.n_overrides; i++) free(p.policy.overrides[i].prefix);
+        free(p.policy.overrides);
+        return 0;
+    }
+
     st_db db;
     db_open(&db, p.hf_dir);
     if (p.compare_tensor) {
diff --git a/gguf-tools/quality-testing/results/score-q4-requant-imatrix-0731.tsv b/gguf-tools/quality-testing/results/score-q4-requant-imatrix-0731.tsv
new file mode 100644
index 0000000000..f6ecf067fd
--- /dev/null
+++ b/gguf-tools/quality-testing/results/score-q4-requant-imatrix-0731.tsv
@@ -0,0 +1,101 @@
+id	prompt_tokens	target_tokens	nll	avg_nll	first_match	greedy_lcp	api_ref_tokens	api_target_tokens	api_target_mae	api_target_mean_delta	api_top_items	api_top_mapped	api_top_coverage	api_top1_count	api_top1_match	api_top1_rate	api_topn_ref	api_topn_hit	api_topn_recall	api_top_logprob_count	api_top_mae	api_top_mean_delta	api_pair_total	api_pair_agree	api_pair_rate
+case_000	17	24	17.312644945	0.721360206	0	0	24	24	0.721360206	-0.721360206	480	480	1.000000000	24	20	0.833333333	480	359	0.747916667	480	9487.421797487	9487.349661466	456	448	0.982456140
+case_001	18	24	3.519118909	0.146629955	1	16	24	24	0.146629955	-0.146629955	480	478	0.995833333	24	22	0.916666667	478	361	0.755230126	478	9482.871267111	9482.856542764	454	452	0.995594714
+case_002	19	24	9.016919033	0.375704960	1	5	24	24	0.375704960	-0.375704960	480	478	0.995833333	24	21	0.875000000	478	384	0.803347280	478	9484.455142063	9484.417414368	454	450	0.991189427
+case_003	19	24	15.901699850	0.662570827	0	0	24	24	0.662570827	-0.662570827	480	479	0.997916667	24	20	0.833333333	479	341	0.711899791	479	9485.298247316	9485.231851910	455	446	0.980219780
+case_004	15	24	8.241609899	0.343400412	1	9	24	24	0.343400412	-0.343400412	480	479	0.997916667	24	21	0.875000000	479	361	0.753653445	479	9473.645505916	9473.611094183	473	468	0.989429175
+case_005	14	24	11.783083390	0.490961808	1	14	24	24	0.490961808	-0.490961808	480	477	0.993750000	24	20	0.833333333	477	387	0.811320755	477	9482.151350529	9482.101945567	453	448	0.988962472
+case_006	18	24	5.836509358	0.243187890	1	14	24	24	0.243187890	-0.243187890	480	478	0.995833333	24	23	0.958333333	478	372	0.778242678	478	9484.115720328	9484.091299787	454	452	0.995594714
+case_007	14	24	17.234519687	0.718104987	1	7	24	24	0.718104987	-0.718104987	480	480	1.000000000	24	17	0.708333333	480	351	0.731250000	480	9470.292701638	9470.220891139	474	464	0.978902954
+case_008	20	24	6.375413965	0.265642249	1	8	24	24	0.265642249	-0.265642249	480	480	1.000000000	24	21	0.875000000	480	363	0.756250000	480	9486.949410766	9486.922846541	456	453	0.993421053
+case_009	17	24	3.422861613	0.142619234	1	9	24	24	0.142619234	-0.142619234	480	475	0.989583333	24	22	0.916666667	475	370	0.778947368	475	9480.378199020	9480.363786971	451	449	0.995565410
+case_010	26	24	8.309394612	0.346224776	1	1	24	24	0.346224776	-0.346224776	480	477	0.993750000	24	21	0.875000000	477	370	0.775681342	477	9479.529638554	9479.494798325	453	447	0.986754967
+case_011	21	24	5.323985858	0.221832744	0	0	24	24	0.221832744	-0.221832744	480	479	0.997916667	24	22	0.916666667	479	374	0.780793319	479	9485.093759309	9485.071529723	455	453	0.995604396
+case_012	22	24	15.004470772	0.625186282	0	0	24	24	0.625186282	-0.625186282	480	480	1.000000000	24	18	0.750000000	480	363	0.756250000	480	9485.013779102	9484.951260474	456	446	0.978070175
+case_013	22	24	1.986351234	0.082764635	1	24	24	24	0.082764635	-0.082764635	480	480	1.000000000	24	24	1.000000000	480	371	0.772916667	480	9485.200640849	9485.192364385	456	456	1.000000000
+case_014	21	24	7.670987196	0.319624466	0	0	24	24	0.319624466	-0.319624466	480	479	0.997916667	24	22	0.916666667	479	368	0.768267223	479	9484.922548967	9484.890519793	455	452	0.993406593
+case_015	22	24	16.408808564	0.683700357	0	0	24	24	0.683700357	-0.683700357	480	480	1.000000000	24	20	0.833333333	480	381	0.793750000	480	9486.967876887	9486.899506851	456	443	0.971491228
+case_016	19	24	26.730866900	1.113786121	0	0	24	24	1.113786121	-1.113786121	480	479	0.997916667	24	14	0.583333333	479	350	0.730688935	479	9477.064931615	9476.953320479	473	450	0.951374207
+case_017	26	24	12.455984403	0.518999350	0	0	24	24	0.518999350	-0.518999350	480	480	1.000000000	24	17	0.708333333	480	348	0.725000000	480	9484.117106852	9484.065206917	456	449	0.984649123
+case_018	25	24	5.721043043	0.238376793	1	1	24	24	0.238376793	-0.238376793	480	479	0.997916667	24	22	0.916666667	479	369	0.770354906	479	9484.483962822	9484.460075377	455	452	0.993406593
+case_019	19	24	15.545420350	0.647725848	1	13	24	24	0.647725848	-0.647725848	480	478	0.995833333	24	18	0.750000000	478	347	0.725941423	478	9478.883681977	9478.818638377	472	459	0.972457627
+case_020	18	24	8.268487782	0.344520324	1	9	24	24	0.344520324	-0.344520324	480	480	1.000000000	24	23	0.958333333	480	370	0.770833333	480	9472.865039636	9472.830587603	474	472	0.995780591
+case_021	18	24	0.338914253	0.014121427	1	24	24	24	0.014121427	-0.014121427	480	475	0.989583333	24	24	1.000000000	475	386	0.812631579	475	9477.136670357	9477.135243349	451	451	1.000000000
+case_022	16	24	6.040604720	0.251691863	0	0	24	24	0.251691863	-0.251691863	480	479	0.997916667	24	22	0.916666667	479	360	0.751565762	479	9483.089156574	9483.063934842	455	450	0.989010989
+case_023	14	24	14.442730262	0.601780428	0	0	24	24	0.601780428	-0.601780428	480	478	0.995833333	24	19	0.791666667	478	346	0.723849372	478	9482.216005966	9482.155576133	454	444	0.977973568
+case_024	15	24	8.854954396	0.368956433	1	1	24	24	0.368956433	-0.368956433	480	478	0.995833333	24	22	0.916666667	478	384	0.803347280	478	9485.557892918	9485.520842899	454	452	0.995594714
+case_025	20	24	2.162639309	0.090109971	1	24	24	24	0.090109971	-0.090109971	480	479	0.997916667	24	24	1.000000000	479	361	0.753653445	479	9482.883694833	9482.874665024	455	455	1.000000000
+case_026	17	24	10.303078678	0.429294945	1	16	24	24	0.429294945	-0.429294945	480	480	1.000000000	24	21	0.875000000	480	365	0.760416667	480	9470.195046088	9470.152116593	492	484	0.983739837
+case_027	14	24	4.583833605	0.190993067	1	4	24	24	0.190993067	-0.190993067	480	479	0.997916667	24	22	0.916666667	479	354	0.739039666	479	9484.276314923	9484.257175743	455	453	0.995604396
+case_028	16	24	9.848800531	0.410366689	0	0	24	24	0.410366689	-0.410366689	480	479	0.997916667	24	19	0.791666667	479	339	0.707724426	479	9483.994738382	9483.953616042	455	449	0.986813187
+case_029	18	24	17.256905416	0.719037726	0	0	24	24	0.719037726	-0.719037726	480	480	1.000000000	24	17	0.708333333	480	381	0.793750000	480	9489.060572890	9488.988669117	456	442	0.969298246
+case_030	23	24	6.501602475	0.270900103	1	14	24	24	0.270900103	-0.270900103	480	480	1.000000000	24	22	0.916666667	480	382	0.795833333	480	9485.646879825	9485.619789814	456	454	0.995614035
+case_031	22	24	4.198487506	0.174936979	1	3	24	24	0.174936979	-0.174936979	480	477	0.993750000	24	22	0.916666667	477	376	0.788259958	477	9480.597036038	9480.579432317	453	451	0.995584989
+case_032	18	24	6.839275155	0.284969798	0	0	24	24	0.284969798	-0.284969798	480	480	1.000000000	24	21	0.875000000	480	365	0.760416667	480	9483.364223786	9483.335726806	456	452	0.991228070
+case_033	15	24	9.137401905	0.380725079	0	0	24	24	0.380725079	-0.380725079	480	478	0.995833333	24	21	0.875000000	478	370	0.774058577	478	9482.877146716	9482.838914909	454	447	0.984581498
+case_034	22	24	5.741615119	0.239233963	1	6	24	24	0.239233963	-0.239233963	480	479	0.997916667	24	22	0.916666667	479	380	0.793319415	479	9480.537679329	9480.513705988	455	452	0.993406593
+case_035	23	24	10.828044113	0.451168505	0	0	24	24	0.451168505	-0.451168505	480	479	0.997916667	24	21	0.875000000	479	369	0.770354906	479	9483.513360421	9483.468149381	455	448	0.984615385
+case_036	21	24	3.612839144	0.150534964	0	0	24	24	0.150534964	-0.150534964	480	479	0.997916667	24	23	0.958333333	479	364	0.759916493	479	9483.886827858	9483.871742934	455	454	0.997802198
+case_037	18	24	3.712592811	0.154691367	0	0	24	24	0.154691367	-0.154691367	480	479	0.997916667	24	22	0.916666667	479	359	0.749478079	479	9479.677214647	9479.661713215	455	451	0.991208791
+case_038	26	24	6.750856550	0.281285690	1	20	24	24	0.281285690	-0.281285690	480	479	0.997916667	24	23	0.958333333	479	343	0.716075157	479	9481.224720656	9481.196533364	455	453	0.995604396
+case_039	23	24	20.108833071	0.837868045	1	12	24	24	0.837868045	-0.837868045	480	480	1.000000000	24	19	0.791666667	480	367	0.764583333	480	9475.577381004	9475.493594199	492	475	0.965447154
+case_040	21	24	15.393823258	0.641409302	1	15	24	24	0.641409302	-0.641409302	480	479	0.997916667	24	19	0.791666667	479	372	0.776617954	479	9484.803952761	9484.739677924	455	444	0.975824176
+case_041	15	24	13.209647992	0.550402000	0	0	24	24	0.550402000	-0.550402000	480	480	1.000000000	24	19	0.791666667	480	358	0.745833333	480	9488.504924398	9488.449884198	456	449	0.984649123
+case_042	14	24	11.930940702	0.497122529	1	1	24	24	0.497122529	-0.497122529	480	480	1.000000000	24	19	0.791666667	480	390	0.812500000	480	9487.484085239	9487.434372986	456	450	0.986842105
+case_043	18	24	0.968710698	0.040362946	1	24	24	24	0.040362946	-0.040362946	480	472	0.983333333	24	24	1.000000000	472	343	0.726694915	472	9474.593028965	9474.588924259	448	448	1.000000000
+case_044	20	24	8.053645895	0.335568579	0	0	24	24	0.335568579	-0.335568579	480	479	0.997916667	24	21	0.875000000	479	379	0.791231733	479	9484.742704744	9484.709077830	455	449	0.986813187
+case_045	18	24	10.508414715	0.437850613	1	5	24	24	0.437850613	-0.437850613	480	480	1.000000000	24	21	0.875000000	480	348	0.725000000	480	9487.114973161	9487.071188099	456	451	0.989035088
+case_046	19	24	2.811793445	0.117158060	1	2	24	24	0.117158060	-0.117158060	480	476	0.991666667	24	23	0.958333333	476	349	0.733193277	476	9480.652843728	9480.641029470	452	451	0.997787611
+case_047	16	24	7.877445790	0.328226908	1	5	24	24	0.328226908	-0.328226908	480	479	0.997916667	24	20	0.833333333	479	341	0.711899791	479	9484.296954290	9484.264063076	455	450	0.989010989
+case_048	18	24	6.200179281	0.258340803	1	23	24	24	0.258340803	-0.258340803	480	479	0.997916667	24	23	0.958333333	479	374	0.780793319	479	9485.266618269	9485.240730255	455	450	0.989010989
+case_049	18	24	20.722330966	0.863430457	1	14	24	24	0.863430457	-0.863430457	480	480	1.000000000	24	19	0.791666667	480	369	0.768750000	480	9466.904310408	9466.817967362	474	458	0.966244726
+case_050	27	24	18.225562645	0.759398444	1	11	24	24	0.759398444	-0.759398444	480	480	1.000000000	24	19	0.791666667	480	350	0.729166667	480	9484.084624933	9484.008685089	456	447	0.980263158
+case_051	21	24	12.529212802	0.522050533	0	0	24	24	0.522050533	-0.522050533	480	480	1.000000000	24	20	0.833333333	480	364	0.758333333	480	9489.048368422	9488.996163368	456	450	0.986842105
+case_052	19	24	11.977788362	0.499074515	1	3	24	24	0.499074515	-0.499074515	480	479	0.997916667	24	18	0.750000000	479	362	0.755741127	479	9481.001144315	9480.951132672	491	484	0.985743381
+case_053	22	24	2.967696017	0.123654001	1	11	24	24	0.123654001	-0.123654001	480	476	0.991666667	24	23	0.958333333	476	355	0.745798319	476	9478.653084767	9478.640615456	452	451	0.997787611
+case_054	24	24	8.700215908	0.362508996	0	0	24	24	0.362508996	-0.362508996	480	480	1.000000000	24	22	0.916666667	480	384	0.800000000	480	9483.186217249	9483.149966349	456	445	0.975877193
+case_055	25	24	5.355243020	0.223135126	1	5	24	24	0.223135126	-0.223135126	480	480	1.000000000	24	21	0.875000000	480	338	0.704166667	480	9483.254834637	9483.232521124	456	453	0.993421053
+case_056	18	24	13.790494920	0.574603955	0	0	24	24	0.574603955	-0.574603955	480	479	0.997916667	24	19	0.791666667	479	348	0.726513570	479	9484.431406416	9484.373826062	455	447	0.982417582
+case_057	22	24	5.008496027	0.208687334	0	0	24	24	0.208687334	-0.208687334	480	478	0.995833333	24	23	0.958333333	478	365	0.763598326	478	9480.527223189	9480.506267139	454	453	0.997797357
+case_058	25	24	4.142684494	0.172611854	1	19	24	24	0.172611854	-0.172611854	480	480	1.000000000	24	23	0.958333333	480	369	0.768750000	480	9484.309634956	9484.292373771	456	455	0.997807018
+case_059	23	24	10.513147963	0.438047832	0	0	24	24	0.438047832	-0.438047832	480	480	1.000000000	24	19	0.791666667	480	348	0.725000000	480	9484.238997349	9484.195192566	456	450	0.986842105
+case_060	17	24	0.539267842	0.022469493	1	24	24	24	0.022469493	-0.022469493	480	473	0.985416667	24	24	1.000000000	473	375	0.792811839	473	9474.150389026	9474.148108823	449	449	1.000000000
+case_061	16	24	18.323077899	0.763461579	0	0	24	24	0.763461579	-0.763461579	480	480	1.000000000	24	17	0.708333333	480	342	0.712500000	480	9475.491756605	9475.415410447	492	476	0.967479675
+case_062	14	24	5.082559957	0.211773332	1	15	24	24	0.211773332	-0.211773332	480	480	1.000000000	24	23	0.958333333	480	352	0.733333333	480	9482.775947450	9482.754770116	456	453	0.993421053
+case_063	15	24	6.285562807	0.261898450	1	11	24	24	0.261898450	-0.261898450	480	476	0.991666667	24	22	0.916666667	476	377	0.792016807	476	9476.502627259	9476.476217331	452	446	0.986725664
+case_064	12	24	7.793198913	0.324716621	1	14	24	24	0.324716621	-0.324716621	480	477	0.993750000	24	23	0.958333333	477	333	0.698113208	477	9475.308509306	9475.275833420	471	468	0.993630573
+case_065	15	24	12.247740998	0.510322542	0	0	24	24	0.510322542	-0.510322542	480	480	1.000000000	24	18	0.750000000	480	383	0.797916667	480	9484.232162796	9484.181130542	456	449	0.984649123
+case_066	16	24	6.192415293	0.258017304	0	0	24	24	0.258017304	-0.258017304	480	480	1.000000000	24	21	0.875000000	480	393	0.818750000	480	9483.562772116	9483.536970386	474	471	0.993670886
+case_067	17	24	15.314347684	0.638097820	0	0	24	24	0.638097820	-0.638097820	480	479	0.997916667	24	20	0.833333333	479	357	0.745302714	479	9469.414218501	9469.350275504	473	463	0.978858351
+case_068	17	24	14.847894468	0.618662270	0	0	24	24	0.618662270	-0.618662270	480	480	1.000000000	24	19	0.791666667	480	338	0.704166667	480	9486.834477168	9486.772610941	456	445	0.975877193
+case_069	18	24	10.562609476	0.440108728	0	0	24	24	0.440108728	-0.440108728	480	479	0.997916667	24	19	0.791666667	479	385	0.803757829	479	9487.120359185	9487.076256431	455	449	0.986813187
+case_070	24	24	2.280622508	0.095025938	1	24	24	24	0.095025938	-0.095025938	480	476	0.991666667	24	24	1.000000000	476	381	0.800420168	476	9477.265043590	9477.255461143	452	452	1.000000000
+case_071	24	24	6.744841142	0.281035048	0	0	24	24	0.281035048	-0.281035048	480	480	1.000000000	24	22	0.916666667	480	373	0.777083333	480	9485.094612134	9485.066508629	456	453	0.993421053
+case_072	17	24	2.005355449	0.083556477	1	9	24	24	0.083556477	-0.083556477	480	478	0.995833333	24	23	0.958333333	478	361	0.755230126	478	9475.043177757	9475.034787149	472	471	0.997881356
+case_073	15	24	8.580089935	0.357503747	1	3	24	24	0.357503747	-0.357503747	480	477	0.993750000	24	21	0.875000000	477	376	0.788259958	477	9470.514505557	9470.478530337	471	466	0.989384289
+case_074	15	24	14.386974335	0.599457264	0	0	24	24	0.599457264	-0.599457264	480	478	0.995833333	24	20	0.833333333	478	345	0.721757322	478	9463.410017343	9463.349820798	471	457	0.970276008
+case_075	16	24	3.614196414	0.150591517	1	6	24	24	0.150591517	-0.150591517	480	478	0.995833333	24	22	0.916666667	478	356	0.744769874	478	9480.769879486	9480.754757326	454	452	0.995594714
+case_076	22	24	5.396436546	0.224851523	1	4	24	24	0.224851523	-0.224851523	480	477	0.993750000	24	22	0.916666667	477	366	0.767295597	477	9479.935717170	9479.913090602	453	450	0.993377483
+case_077	24	24	7.871823196	0.327992633	0	0	24	24	0.327992633	-0.327992633	480	480	1.000000000	24	20	0.833333333	480	376	0.783333333	480	9484.635272915	9484.602473651	456	452	0.991228070
+case_078	28	24	19.085377210	0.795224050	0	0	24	24	0.795224050	-0.795224050	480	479	0.997916667	24	18	0.750000000	479	338	0.705636743	479	9484.897514179	9484.817825757	455	436	0.958241758
+case_079	20	24	7.624825106	0.317701046	1	18	24	24	0.317701046	-0.317701046	480	479	0.997916667	24	22	0.916666667	479	352	0.734864301	479	9473.519303318	9473.487466888	473	470	0.993657505
+case_080	13	24	17.398640752	0.724943365	1	6	24	24	0.724943365	-0.724943365	480	478	0.995833333	24	17	0.708333333	478	356	0.744769874	478	9485.466003952	9485.393206291	454	439	0.966960352
+case_081	13	24	9.845937111	0.410247380	0	0	24	24	0.410247380	-0.410247380	480	479	0.997916667	24	21	0.875000000	479	352	0.734864301	479	9485.830540154	9485.789429769	455	450	0.989010989
+case_082	16	24	28.455881343	1.185661723	1	7	24	24	1.185661723	-1.185661723	480	477	0.993750000	24	16	0.666666667	477	328	0.687631027	477	9472.165230265	9472.045918393	471	451	0.957537155
+case_083	19	17	0.099510162	0.005853539	1	17	17	17	0.005853539	-0.005853539	340	330	0.970588235	17	17	1.000000000	330	241	0.730303030	330	9463.047109516	9463.046506424	313	313	1.000000000
+case_084	23	11	0.701877409	0.063807037	1	11	11	11	0.063807037	-0.063807037	220	215	0.977272727	11	11	1.000000000	215	163	0.758139535	215	9467.740751223	9467.734222131	204	204	1.000000000
+case_085	18	24	10.909207049	0.454550294	1	11	24	24	0.454550294	-0.454550294	480	475	0.989583333	24	21	0.875000000	475	373	0.785263158	475	9463.544781446	9463.498847943	469	464	0.989339019
+case_086	16	24	7.114907964	0.296454499	1	8	24	24	0.296454499	-0.296454499	480	480	1.000000000	24	21	0.875000000	480	376	0.783333333	480	9488.089166432	9488.059520982	456	450	0.986842105
+case_087	12	24	11.005250253	0.458552094	1	3	24	24	0.458552094	-0.458552094	480	480	1.000000000	24	20	0.833333333	480	377	0.785416667	480	9486.949239349	9486.903384140	456	448	0.982456140
+case_088	14	24	14.441297598	0.601720733	1	7	24	24	0.601720733	-0.601720733	480	480	1.000000000	24	17	0.708333333	480	393	0.818750000	480	9489.376179659	9489.316007586	456	445	0.975877193
+case_089	12	24	18.530864832	0.772119368	0	0	24	24	0.772119368	-0.772119368	480	480	1.000000000	24	19	0.791666667	480	353	0.735416667	480	9468.948623045	9468.871411109	474	462	0.974683544
+case_090	17	24	7.802819239	0.325117468	0	0	24	24	0.325117468	-0.325117468	480	480	1.000000000	24	22	0.916666667	480	344	0.716666667	480	9485.508067545	9485.475555798	456	450	0.986842105
+case_091	14	24	14.468345118	0.602847713	0	0	24	24	0.602847713	-0.602847713	480	479	0.997916667	24	21	0.875000000	479	326	0.680584551	479	9483.682717450	9483.622306824	455	437	0.960439560
+case_092	15	24	0.690675976	0.028778166	1	24	24	24	0.028778166	-0.028778166	480	475	0.989583333	24	24	1.000000000	475	379	0.797894737	475	9474.294883704	9474.291975594	451	451	1.000000000
+case_093	17	24	18.748704203	0.781196008	0	0	24	24	0.781196008	-0.781196008	480	480	1.000000000	24	17	0.708333333	480	366	0.762500000	480	9479.038790268	9478.960670668	474	463	0.976793249
+case_094	16	8	0.179222634	0.022402829	1	8	8	8	0.022402829	-0.022402829	160	155	0.968750000	8	8	1.000000000	155	123	0.793548387	155	9463.391503847	9463.389191297	147	147	1.000000000
+case_095	17	15	0.955572988	0.063704866	1	15	15	15	0.063704866	-0.063704866	300	290	0.966666667	15	15	1.000000000	290	228	0.786206897	290	9464.078747327	9464.072157169	275	275	1.000000000
+case_096	15	11	2.966251965	0.269659270	0	0	11	11	0.269659270	-0.269659270	220	213	0.968181818	11	10	0.909090909	213	140	0.657276995	213	9462.336499702	9462.308647570	202	201	0.995049505
+case_097	22	11	0.023667264	0.002151569	1	11	11	11	0.002151569	-0.002151569	220	211	0.959090909	11	11	1.000000000	211	147	0.696682464	211	9456.299221880	9456.298997545	200	200	1.000000000
+case_098	20	8	5.732133494	0.716516687	1	1	8	8	0.716516687	-0.716516687	160	152	0.950000000	8	5	0.625000000	152	100	0.657894737	152	9458.117830898	9458.042408089	144	141	0.979166667
+case_099	22	24	12.111447118	0.504643630	0	0	24	24	0.504643630	-0.504643630	480	464	0.966666667	24	21	0.875000000	464	343	0.739224138	464	9462.414465266	9462.362260752	440	437	0.993181818
diff --git a/linenoise.c b/linenoise.c
index 9639422efe..68146c505e 100644
--- a/linenoise.c
+++ b/linenoise.c
@@ -109,12 +109,10 @@
 #include <stdio.h>
 #include <errno.h>
 #include <string.h>
-#include <stdlib.h>
 #include <ctype.h>
 #include <sys/stat.h>
 #include <sys/types.h>
 #include <sys/ioctl.h>
-#include <unistd.h>
 #include <stdint.h>
 #include "linenoise.h"
 
@@ -122,7 +120,6 @@
 #define LINENOISE_MAX_LINE (1024*1024)      // That will get dynamically allocated
 #define LINENOISE_INITIAL_BUFLEN 4096
 #define PASTE_FOLD_THRESHOLD 200            // Min bytes to fold a single-line paste.
-#define PASTE_FOLD_CONTEXT 8                // Context chars kept around generic folds.
 #define HISTORY_FOLD_THRESHOLD 4096         // Min bytes to fold single-line history.
 #define HISTORY_FOLD_MULTILINE_LINES 16     // Min lines to fold shorter history.
 #define HISTORY_FOLD_CONTEXT 96             // Context chars kept around history folds.
@@ -486,14 +483,12 @@ static int utf8SingleCharWidth(const char *s, size_t len) {
 }
 
 enum KEY_ACTION{
-	KEY_NULL = 0,	    /* NULL */
 	CTRL_A = 1,         /* Ctrl+a */
 	CTRL_B = 2,         /* Ctrl-b */
 	CTRL_C = 3,         /* Ctrl-c */
 	CTRL_D = 4,         /* Ctrl-d */
 	CTRL_E = 5,         /* Ctrl-e */
 	CTRL_F = 6,         /* Ctrl-f */
-	CTRL_H = 8,         /* Ctrl-h */
 	TAB = 9,            /* Tab */
 	CTRL_K = 11,        /* Ctrl+k */
 	CTRL_L = 12,        /* Ctrl+l */
@@ -514,24 +509,7 @@ int linenoiseHistoryAdd(const char *line);
 #define REFRESH_ALL (REFRESH_CLEAN|REFRESH_WRITE) // Do both.
 static void refreshLine(struct linenoiseState *l);
 
-/* Debugging macro. */
-#if 0
-FILE *lndebug_fp = NULL;
-#define lndebug(...) \
-    do { \
-        if (lndebug_fp == NULL) { \
-            lndebug_fp = fopen("/tmp/lndebug.txt","a"); \
-            fprintf(lndebug_fp, \
-            "[%d %d %d] p: %d, rows: %d, rpos: %d, max: %d, oldmax: %d\n", \
-            (int)l->len,(int)l->pos,(int)l->oldpos,plen,rows,rpos, \
-            (int)l->oldrows,old_rows); \
-        } \
-        fprintf(lndebug_fp, ", " __VA_ARGS__); \
-        fflush(lndebug_fp); \
-    } while (0)
-#else
 #define lndebug(fmt, ...)
-#endif
 
 /* ======================= Low level terminal handling ====================== */
 
diff --git a/metal/activations.metal b/metal/activations.metal
new file mode 100644
index 0000000000..4e2bcf7119
--- /dev/null
+++ b/metal/activations.metal
@@ -0,0 +1,35 @@
+// Shared activation functions. This source is loaded before every kernel file.
+
+// Keep finite activation behavior aligned with the stable CPU implementation.
+// exp(-abs(x)) avoids overflow without a divergent branch on the sign.
+static inline float ds4_sigmoid_stable(float x) {
+    const float e = exp(-fabs(x));
+    const float numer = x >= 0.0f ? 1.0f : e;
+    return numer / (1.0f + e);
+}
+
+static inline float4 ds4_sigmoid_stable(float4 x) {
+    const float4 e = exp(-fabs(x));
+    const float4 numer = select(e, float4(1.0f), x >= 0.0f);
+    return numer / (1.0f + e);
+}
+
+static inline float ds4_silu(float x) {
+    // Metal fast-math may flush exp(x) before x can lift a subnormal tail
+    // back into the normal range. Here 1 + exp(x) rounds to 1, so fold |x|
+    // into the exponent and produce the final magnitude without a subnormal
+    // intermediate. The precise calls also prevent unsafe fast-math rewrites.
+    if (x < -87.0f) {
+        return -precise::exp(x + precise::log(-x));
+    }
+    return x * ds4_sigmoid_stable(x);
+}
+
+static inline float4 ds4_silu(float4 x) {
+    float4 result = x * ds4_sigmoid_stable(x);
+    if (x.x < -87.0f) result.x = ds4_silu(x.x);
+    if (x.y < -87.0f) result.y = ds4_silu(x.y);
+    if (x.z < -87.0f) result.z = ds4_silu(x.z);
+    if (x.w < -87.0f) result.w = ds4_silu(x.w);
+    return result;
+}
diff --git a/metal/argsort.metal b/metal/argsort.metal
index 77c473f964..8f84860ba2 100644
--- a/metal/argsort.metal
+++ b/metal/argsort.metal
@@ -273,3 +273,129 @@ kernel void kernel_argsort_merge_f32_i32(
 
 // Host-visible merge variant used by DS4 top-k selection.
 template [[host_name("kernel_argsort_merge_f32_i32_desc")]] kernel argsort_merge_t kernel_argsort_merge_f32_i32<DS4_SORT_ORDER_DESC>;
+
+/* Decode only needs the best vocabulary id, not a sorted candidate list.
+ * Stage 1 scans disjoint chunks and publishes one lexicographic winner
+ * per threadgroup.  Stage 2 reduces those pairs after a dispatch boundary,
+ * which is the required device-wide ordering without atomics. */
+struct ds4_metal_args_dsv4_argmax_top1 {
+    int32_t n_vocab;
+    int32_t n_tg;
+};
+
+static inline void ds4_argmax_top1_merge(
+        float ov, int oi, thread float *bv, thread int *bi) {
+    if (ov > *bv ||
+        (ov == *bv && (*bi < 0 || (oi >= 0 && oi < *bi)))) {
+        *bv = ov;
+        *bi = oi;
+    }
+}
+
+kernel void kernel_dsv4_argmax_top1_stage1_f32(
+        constant ds4_metal_args_dsv4_argmax_top1 &args,
+        device const float   *src0,
+        device float         *scratch_v,
+        device int32_t       *scratch_i,
+        threadgroup char     *shmem_raw [[threadgroup(0)]],
+        uint                  tgpig [[threadgroup_position_in_grid]],
+        ushort                tiitg [[thread_index_in_threadgroup]],
+        ushort                tiisg [[thread_index_in_simdgroup]],
+        ushort                sgitg [[simdgroup_index_in_threadgroup]]) {
+    threadgroup float *tg_v = (threadgroup float *)shmem_raw;
+    threadgroup int32_t *tg_i =
+        (threadgroup int32_t *)(tg_v + 8);
+
+    const uint n = (uint)args.n_vocab;
+    const uint ntg = (uint)args.n_tg;
+    const uint chunk = (n + ntg - 1u) / ntg;
+    const uint begin = tgpig * chunk;
+    const uint end = min(n, begin + chunk);
+
+    float bv = -INFINITY;
+    int bi = -1;
+    for (uint i = begin + (uint)tiitg; i < end; i += 256u) {
+        const float v = src0[i];
+        if (v > bv) {
+            bv = v;
+            bi = (int)i;
+        }
+    }
+    for (ushort off = 16u; off > 0u; off >>= 1u) {
+        ds4_argmax_top1_merge(simd_shuffle_xor(bv, off),
+                              simd_shuffle_xor(bi, off),
+                              &bv, &bi);
+    }
+    if (tiisg == 0u) {
+        tg_v[sgitg] = bv;
+        tg_i[sgitg] = bi;
+    }
+    threadgroup_barrier(mem_flags::mem_threadgroup);
+    if (sgitg == 0u) {
+        if (tiisg >= 8u) {
+            bv = -INFINITY;
+            bi = -1;
+        } else {
+            bv = tg_v[tiisg];
+            bi = tg_i[tiisg];
+        }
+        for (ushort off = 4u; off > 0u; off >>= 1u) {
+            ds4_argmax_top1_merge(simd_shuffle_xor(bv, off),
+                                  simd_shuffle_xor(bi, off),
+                                  &bv, &bi);
+        }
+        if (tiisg == 0u) {
+            scratch_v[tgpig] = bv;
+            scratch_i[tgpig] = bi;
+        }
+    }
+}
+
+kernel void kernel_dsv4_argmax_top1_stage2_f32(
+        constant ds4_metal_args_dsv4_argmax_top1 &args,
+        device const float   *scratch_v,
+        device const int32_t *scratch_i,
+        device int32_t       *dst,
+        threadgroup char     *shmem_raw [[threadgroup(0)]],
+        ushort                tiitg [[thread_index_in_threadgroup]],
+        ushort                tiisg [[thread_index_in_simdgroup]],
+        ushort                sgitg [[simdgroup_index_in_threadgroup]]) {
+    threadgroup float *tg_v = (threadgroup float *)shmem_raw;
+    threadgroup int32_t *tg_i =
+        (threadgroup int32_t *)(tg_v + 8);
+
+    const uint ntg = (uint)args.n_tg;
+    float bv = -INFINITY;
+    int bi = -1;
+    if ((uint)tiitg < ntg) {
+        bv = scratch_v[tiitg];
+        bi = scratch_i[tiitg];
+    }
+    for (ushort off = 16u; off > 0u; off >>= 1u) {
+        ds4_argmax_top1_merge(simd_shuffle_xor(bv, off),
+                              simd_shuffle_xor(bi, off),
+                              &bv, &bi);
+    }
+    if (tiisg == 0u) {
+        tg_v[sgitg] = bv;
+        tg_i[sgitg] = bi;
+    }
+    threadgroup_barrier(mem_flags::mem_threadgroup);
+    if (sgitg == 0u) {
+        if (tiisg >= 8u) {
+            bv = -INFINITY;
+            bi = -1;
+        } else {
+            bv = tg_v[tiisg];
+            bi = tg_i[tiisg];
+        }
+        for (ushort off = 4u; off > 0u; off >>= 1u) {
+            ds4_argmax_top1_merge(simd_shuffle_xor(bv, off),
+                                  simd_shuffle_xor(bi, off),
+                                  &bv, &bi);
+        }
+        /* sample_argmax starts at index zero with -INF.  Preserve that edge
+         * contract when every logit is NaN or -INF. */
+        if (tiisg == 0u) dst[0] = bi >= 0 ? bi : 0;
+    }
+}
diff --git a/metal/dense.metal b/metal/dense.metal
index 403597424b..b1495aec91 100644
--- a/metal/dense.metal
+++ b/metal/dense.metal
@@ -590,7 +590,7 @@ void kernel_dsv4_shared_gate_up_swiglu_q8_0_impl(
                 g = min(g, clamp_value);
                 u = clamp(u, -clamp_value, clamp_value);
             }
-            const float silu = g / (1.0f + exp(-g));
+            const float silu = ds4_silu(g);
             mid_f32[out_row] = silu * u;
         }
     }
@@ -1103,7 +1103,7 @@ kernel void kernel_dsv4_router_shared_gate_up_q8_0(
                 g = min(g, clamp_value);
                 u = clamp(u, -clamp_value, clamp_value);
             }
-            const float silu = g / (1.0f + exp(-g));
+            const float silu = ds4_silu(g);
             mid_f32[out_row] = silu * u;
         }
     }
@@ -1995,6 +1995,44 @@ void dequantize_dense_q4_K(device const ds4_dense_block_q4_K *xb, short il, thre
     reg = (type4x4)reg_f;
 }
 
+/*
+ * One-shot resident Q4_K -> F16 materialization used by the attn_q_b
+ * prefill cache.  Keeping the production dequantizer here is intentional:
+ * kernel_mul_mm_q4_K_f32 computes the same float4x4 values and rounds them
+ * while storing into its half threadgroup tile.  Instantiating the helper
+ * with half4x4 performs that same single rounding here, so the cached matrix
+ * contains exactly the half values the native Q4 kernel would otherwise
+ * rebuild for every 64x32 output tile.
+ *
+ * Each thread expands one consecutive 16-value chunk.  Four explicit half4
+ * stores preserve the half4x4 layout without relying on device-address-space
+ * matrix stores, which are not accepted by every supported Metal compiler.
+ */
+kernel void kernel_dequantize_q4_K_f16(
+        device const ds4_dense_block_q4_K *src [[buffer(0)]],
+        device       half4                *dst [[buffer(1)]],
+        constant     uint                 &chunks_per_row [[buffer(2)]],
+        constant     uint                 &row_count [[buffer(3)]],
+        uint2 gid [[thread_position_in_grid]]) {
+    if (gid.x >= chunks_per_row || gid.y >= row_count) return;
+
+    const uint block = gid.x >> 4;
+    const short il = (short)(gid.x & 15u);
+    const uint blocks_per_row = chunks_per_row >> 4;
+    device const ds4_dense_block_q4_K *xb =
+        src + (uint64_t)gid.y * blocks_per_row + block;
+
+    half4x4 values;
+    dequantize_dense_q4_K(xb, il, values);
+
+    const uint64_t out4 =
+        ((uint64_t)gid.y * chunks_per_row + gid.x) * 4u;
+    dst[out4 + 0u] = values[0];
+    dst[out4 + 1u] = values[1];
+    dst[out4 + 2u] = values[2];
+    dst[out4 + 3u] = values[3];
+}
+
 /*
  * Bit-identical twin of dequantize_q8_0 for the MPP staging loop: same
  * half(float(qs[i]) * d) per element, but the 16 consecutive int8 lanes are
@@ -2298,7 +2336,7 @@ void kernel_mul_mv_ext_q8_0_pair_swiglu_f32_impl(
                 g = min(g, clamp_value);
                 u = clamp(u, -clamp_value, clamp_value);
             }
-            const float silu = g / (1.0f + exp(-g));
+            const float silu = ds4_silu(g);
             mid_f32[i01] = silu * u;
         }
     }
@@ -2494,7 +2532,7 @@ template [[host_name("kernel_mul_mm_q8_0_f32_nax_direct_rhs_n128")]] kernel mul_
 // Tiled matrix-matrix kernel used for prompt batches larger than 8. DS4 uses
 // this to turn prefill into large simdgroup matrix operations; each block_q
 // contains 16*nl weights.
-template<typename S0, typename S0_4x4, typename S0_8x8, typename S1, typename S1_2x4, typename S1_8x8, typename block_q, short nl, void (*dequantize_func)(device const block_q *, short, thread S0_4x4 &), typename T0, typename T0_4x4, typename T1, typename T1_2x4>
+template<typename S0, typename S0_4x4, typename S0_8x8, typename S1, typename S1_2x4, typename S1_8x8, typename block_q, short nl, void (*dequantize_func)(device const block_q *, short, thread S0_4x4 &), typename T0, typename T0_4x4, typename T1, typename T1_2x4, bool CULL_TAIL_SIMDGROUPS = false>
 kernel void kernel_mul_mm(
         constant ds4_metal_args_mul_mm & args,
         device const char * src0,
@@ -2522,6 +2560,12 @@ kernel void kernel_mul_mm(
     // if this block is of 64x32 shape or smaller
     const short nr0 = (args.ne0 - r0 < NR0) ? (args.ne0 - r0) : NR0;
     const short nr1 = (args.ne1 - r1 < NR1) ? (args.ne1 - r1) : NR1;
+    // SIMDgroups 0/1 own token rows 0..15 and SIMDgroups 2/3 own rows
+    // 16..31.  Every thread still participates in cooperative A/B staging and
+    // every threadgroup barrier; on a short final Q4_K tile only the waves with
+    // valid token rows construct fragments, execute MMA, and store results.
+    const bool mma_active =
+        !CULL_TAIL_SIMDGROUPS || 16*(short)(sgitg/2) < nr1;
 
     // a thread shouldn't load data outside of the matrix
     const short lr0 = ((short)tiitg/NL0) < nr0 ? ((short)tiitg/NL0) : nr0 - 1; // 0 .. 63
@@ -2552,8 +2596,10 @@ kernel void kernel_mul_mm(
 
     simdgroup_float8x8 mc[8];
 
-    for (short i = 0; i < 8; i++){
-        mc[i] = make_filled_simdgroup_matrix<float, 8>(0.f);
+    if (mma_active) {
+        for (short i = 0; i < 8; i++){
+            mc[i] = make_filled_simdgroup_matrix<float, 8>(0.f);
+        }
     }
 
     for (int loop_k = 0; loop_k < args.ne00; loop_k += NK) {
@@ -2628,27 +2674,29 @@ kernel void kernel_mul_mm(
         threadgroup const S0 * lsma = (sa + 4*64*(sgitg%2));
         threadgroup const S1 * lsmb = (sb + 2*64*(sgitg/2));
 
-        FOR_UNROLL (short ik = 0; ik < NK/8; ik++) {
-            simdgroup_barrier(mem_flags::mem_none);
+        if (mma_active) {
+            FOR_UNROLL (short ik = 0; ik < NK/8; ik++) {
+                simdgroup_barrier(mem_flags::mem_none);
 
-            FOR_UNROLL (short i = 0; i < 4; i++) {
-                simdgroup_load(ma[i], lsma + 64*i, 8, 0, false);
-            }
+                FOR_UNROLL (short i = 0; i < 4; i++) {
+                    simdgroup_load(ma[i], lsma + 64*i, 8, 0, false);
+                }
 
-            simdgroup_barrier(mem_flags::mem_none);
+                simdgroup_barrier(mem_flags::mem_none);
 
-            FOR_UNROLL (short i = 0; i < 2; i++) {
-                simdgroup_load(mb[i], lsmb + 64*i, 8, 0, false);
-            }
+                FOR_UNROLL (short i = 0; i < 2; i++) {
+                    simdgroup_load(mb[i], lsmb + 64*i, 8, 0, false);
+                }
 
-            simdgroup_barrier(mem_flags::mem_none);
+                simdgroup_barrier(mem_flags::mem_none);
 
-            FOR_UNROLL (short i = 0; i < 8; i++){
-                simdgroup_multiply_accumulate(mc[i], mb[i/4], ma[i%4], mc[i]);
-            }
+                FOR_UNROLL (short i = 0; i < 8; i++){
+                    simdgroup_multiply_accumulate(mc[i], mb[i/4], ma[i%4], mc[i]);
+                }
 
-            lsma += 8*64;
-            lsmb += 4*64;
+                lsma += 8*64;
+                lsmb += 4*64;
+            }
         }
     }
 
@@ -2658,8 +2706,10 @@ kernel void kernel_mul_mm(
             (r0 + 32*(sgitg &  1)) + \
             (r1 + 16*(sgitg >> 1)) * args.ne0 + im*args.ne1*args.ne0;
 
-        for (short i = 0; i < 8; i++) {
-            simdgroup_store(mc[i], C + 8*(i%4) + 8*args.ne0*(i/4), args.ne0, 0, false);
+        if (mma_active) {
+            for (short i = 0; i < 8; i++) {
+                simdgroup_store(mc[i], C + 8*(i%4) + 8*args.ne0*(i/4), args.ne0, 0, false);
+            }
         }
     } else {
         // block is smaller than 64x32, we should avoid writing data outside of the matrix
@@ -2667,8 +2717,10 @@ kernel void kernel_mul_mm(
 
         threadgroup float * temp_str = ((threadgroup float *) shmem) + 32*(sgitg&1) + (16*(sgitg >> 1))*NR0;
 
-        for (short i = 0; i < 8; i++) {
-            simdgroup_store(mc[i], temp_str + 8*(i%4) + 8*NR0*(i/4), NR0, 0, false);
+        if (mma_active) {
+            for (short i = 0; i < 8; i++) {
+                simdgroup_store(mc[i], temp_str + 8*(i%4) + 8*NR0*(i/4), NR0, 0, false);
+            }
         }
 
         threadgroup_barrier(mem_flags::mem_threadgroup);
@@ -2907,9 +2959,21 @@ kernel void kernel_mul_mm_f16_f32_scaled(
 }
 
 typedef decltype(kernel_mul_mm<half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, float4x4, 1, dequantize_f32, float, float4x4, float, float2x4>) mul_mm_t;
+typedef decltype(kernel_mul_mm<half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, ds4_dense_block_q4_K, 16, dequantize_dense_q4_K, float, float4x4, float, float2x4, true>) mul_mm_q4_K_tail_cull_t;
 
 // Host-visible prefill matmul variants for F16 and Q8_0 weights.
 template [[host_name("kernel_mul_mm_f16_f32")]]  kernel mul_mm_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, half4x4, 1, dequantize_f16,  half,  half4x4,  float, float2x4>;
+// Resident Q4 sidecars and stream-local transient expansions can pair their
+// F16 weights with an RHS rounded to F16 once by the producer.  This retains
+// the exact legacy simdgroup-MMA and FP32 accumulation schedule while avoiding
+// the repeated F32 load/conversion performed once per 64-row output tile.
+template [[host_name("kernel_mul_mm_f16_f16_rhs")]] kernel mul_mm_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, half4x4, 1, dequantize_f16,  half,  half4x4,  half,  half2x4>;
 template [[host_name("kernel_mul_mm_q8_0_f32")]] kernel mul_mm_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_q8_0, 2, dequantize_q8_0, float, float4x4, float, float2x4>;
 template [[host_name("kernel_mul_mm_q4_0_f32")]] kernel mul_mm_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, ds4_dense_block_q4_0, 2, dequantize_dense_q4_0, float, float4x4, float, float2x4>;
 template [[host_name("kernel_mul_mm_q4_K_f32")]] kernel mul_mm_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, ds4_dense_block_q4_K, 16, dequantize_dense_q4_K, float, float4x4, float, float2x4>;
+template [[host_name("kernel_mul_mm_q4_K_f32_tail_cull")]] kernel mul_mm_q4_K_tail_cull_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, ds4_dense_block_q4_K, 16, dequantize_dense_q4_K, float, float4x4, float, float2x4, true>;
+// Q4_K output projection with a pre-materialized F16 RHS.  The ordinary F32
+// variant performs this same F32-to-F16 conversion every time a 64-row weight
+// tile revisits the activation matrix; this variant lets the host perform it
+// once while preserving the MMA tile and accumulation order.
+template [[host_name("kernel_mul_mm_q4_K_f16_rhs")]] kernel mul_mm_t kernel_mul_mm<half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, ds4_dense_block_q4_K, 16, dequantize_dense_q4_K, float, float4x4, half, half2x4>;
diff --git a/metal/dsv4_hc.metal b/metal/dsv4_hc.metal
index c161a03a8b..d0a6a9b8d2 100644
--- a/metal/dsv4_hc.metal
+++ b/metal/dsv4_hc.metal
@@ -692,6 +692,306 @@ kernel void kernel_dsv4_hc_expand4(
     }
 }
 
+// Resident prefill attention-output tail for the exact legacy Q8_0 tile:
+// materialize each 64x32 F32 result in threadgroup memory and immediately
+// apply the HC=4 epilogue.  This preserves the existing F32 boundary and HC
+// statement order while removing the global attn_out round trip.
+kernel void kernel_dsv4_attn_out_q8_mm_hc_expand4_batch(
+        constant ds4_metal_args_mul_mm          & mm [[buffer(0)]],
+        device  const char * weight                  [[buffer(1)]],
+        device  const char * input                   [[buffer(2)]],
+        device  const char * residual                [[buffer(3)]],
+        device  const char * post                    [[buffer(4)]],
+        device  const char * comb                    [[buffer(5)]],
+        device        char * dst                     [[buffer(6)]],
+        constant ds4_metal_args_dsv4_hc_expand & hc [[buffer(7)]],
+        threadgroup   char * shmem                   [[threadgroup(0)]],
+        uint3  tgpig [[threadgroup_position_in_grid]],
+        ushort tiitg [[thread_index_in_threadgroup]],
+        ushort sgitg [[simdgroup_index_in_threadgroup]]) {
+    constexpr int NR0 = 64;
+    constexpr int NR1 = 32;
+    constexpr int NK = 32;
+    constexpr int NL0 = NK / 16;
+    constexpr int NL1 = NK / 8;
+
+    if (hc.n_hc != 4 || mm.ne0 != hc.n_embd || mm.ne1 != hc.n_tokens ||
+        mm.ne00 != 8192 || mm.ne0 != 4096 || (mm.ne1 & 31) != 0) {
+        return;
+    }
+
+    threadgroup half * sa = (threadgroup half *)shmem;
+    threadgroup half * sb = (threadgroup half *)(shmem + 4096);
+
+    const int im = tgpig.z;
+    const int r0 = tgpig.y * NR0;
+    const int r1 = tgpig.x * NR1;
+    const short lr0 = (short)tiitg / NL0;
+    const short lr1 = (short)tiitg / NL1;
+    const short il0 = tiitg % NL0;
+    short il = il0;
+
+    const int i12 = im % mm.ne12;
+    const int i13 = im / mm.ne12;
+    const uint64_t offset0 =
+        (i12 / mm.r2) * mm.nb02 + (i13 / mm.r3) * mm.nb03;
+    const short offset1 = il0 / 2;
+    device const block_q8_0 * x =
+        (device const block_q8_0 *)(weight + mm.nb01 * (r0 + lr0) + offset0) +
+        offset1;
+
+    const short iy = 8 * (tiitg % NL1);
+    device const float * y = (device const float *)(input +
+        mm.nb13 * i13 + mm.nb12 * i12 + mm.nb11 * (r1 + lr1) + mm.nb10 * iy);
+
+    simdgroup_half8x8 ma[4];
+    simdgroup_half8x8 mb[2];
+    simdgroup_float8x8 mc[8];
+    FOR_UNROLL (short i = 0; i < 8; ++i) {
+        mc[i] = make_filled_simdgroup_matrix<float, 8>(0.0f);
+    }
+
+    for (int loop_k = 0; loop_k < mm.ne00; loop_k += NK) {
+        half4x4 temp_a;
+        dequantize_q8_0(x, il, temp_a);
+
+        threadgroup_barrier(mem_flags::mem_threadgroup);
+
+        FOR_UNROLL (short i = 0; i < 16; ++i) {
+            const short sx = 2 * il0 + i / 8;
+            const short sy = (tiitg / NL0) / 8;
+            const short lx = (tiitg / NL0) % 8;
+            const short ly = i % 8;
+            const short ib = 8 * sx + sy;
+            *(sa + 64 * ib + 8 * ly + lx) = temp_a[i / 4][i % 4];
+        }
+
+        const short sx = tiitg % NL1;
+        const short sy = (tiitg / NL1) / 8;
+        const short ly = (tiitg / NL1) % 8;
+        const short ib = 4 * sx + sy;
+        *(threadgroup half2x4 *)(sb + 64 * ib + 8 * ly) =
+            (half2x4)(*((device float2x4 *)y));
+
+        il = (il + 2 < 2) ? il + 2 : il % 2;
+        x = (il < 2) ? x + 1 : x;
+        y += NK;
+
+        threadgroup_barrier(mem_flags::mem_threadgroup);
+
+        threadgroup const half * lsma = sa + 4 * 64 * (sgitg % 2);
+        threadgroup const half * lsmb = sb + 2 * 64 * (sgitg / 2);
+
+        FOR_UNROLL (short ik = 0; ik < NK / 8; ++ik) {
+            simdgroup_barrier(mem_flags::mem_none);
+            FOR_UNROLL (short i = 0; i < 4; ++i) {
+                simdgroup_load(ma[i], lsma + 64 * i, 8, 0, false);
+            }
+            simdgroup_barrier(mem_flags::mem_none);
+            FOR_UNROLL (short i = 0; i < 2; ++i) {
+                simdgroup_load(mb[i], lsmb + 64 * i, 8, 0, false);
+            }
+            simdgroup_barrier(mem_flags::mem_none);
+            FOR_UNROLL (short i = 0; i < 8; ++i) {
+                simdgroup_multiply_accumulate(mc[i], mb[i / 4], ma[i % 4], mc[i]);
+            }
+            lsma += 8 * 64;
+            lsmb += 4 * 64;
+        }
+    }
+
+    threadgroup_barrier(mem_flags::mem_threadgroup);
+    threadgroup float * tile = (threadgroup float *)shmem;
+    threadgroup float * tile_sg =
+        tile + 32 * (sgitg & 1) + 16 * (sgitg >> 1) * NR0;
+    FOR_UNROLL (short i = 0; i < 8; ++i) {
+        simdgroup_store(mc[i],
+                        tile_sg + 8 * (i % 4) + 8 * NR0 * (i / 4),
+                        NR0, 0, false);
+    }
+    threadgroup_barrier(mem_flags::mem_threadgroup);
+
+    for (uint e = tiitg; e < (uint)(NR0 * NR1); e += 128u) {
+        const int64_t t = r1 + e / NR0;
+        const int64_t d = r0 + e % NR0;
+        const float block_v = tile[e];
+
+        const float rv0 = *((device const float *)(residual +
+            d * hc.nb_res0 + 0 * hc.nb_res1 + t * hc.nb_res2));
+        const float rv1 = *((device const float *)(residual +
+            d * hc.nb_res0 + 1 * hc.nb_res1 + t * hc.nb_res2));
+        const float rv2 = *((device const float *)(residual +
+            d * hc.nb_res0 + 2 * hc.nb_res1 + t * hc.nb_res2));
+        const float rv3 = *((device const float *)(residual +
+            d * hc.nb_res0 + 3 * hc.nb_res1 + t * hc.nb_res2));
+
+        for (int64_t dst_hc = 0; dst_hc < 4; ++dst_hc) {
+            float acc = block_v * *((device const float *)(post +
+                dst_hc * hc.nb_post0 + t * hc.nb_post1));
+            acc += *((device const float *)(comb +
+                dst_hc * hc.nb_comb0 + 0 * hc.nb_comb1 + t * hc.nb_comb2)) * rv0;
+            acc += *((device const float *)(comb +
+                dst_hc * hc.nb_comb0 + 1 * hc.nb_comb1 + t * hc.nb_comb2)) * rv1;
+            acc += *((device const float *)(comb +
+                dst_hc * hc.nb_comb0 + 2 * hc.nb_comb1 + t * hc.nb_comb2)) * rv2;
+            acc += *((device const float *)(comb +
+                dst_hc * hc.nb_comb0 + 3 * hc.nb_comb1 + t * hc.nb_comb2)) * rv3;
+            *((device float *)(dst + d * hc.nb0 + dst_hc * hc.nb1 +
+                               t * hc.nb2)) = acc;
+        }
+    }
+}
+
+// Q4_K counterpart of the resident output-B/HC tail.  It consumes the same
+// pre-materialized F16 RHS as kernel_mul_mm_q4_K_f16_rhs, keeps its Q4_K
+// dequantization and FP32 MMA order, and applies the identical HC=4 epilogue
+// directly from the threadgroup F32 tile.
+kernel void kernel_dsv4_attn_out_q4_K_f16_rhs_mm_hc_expand4_batch(
+        constant ds4_metal_args_mul_mm          & mm [[buffer(0)]],
+        device  const char * weight                  [[buffer(1)]],
+        device  const char * input                   [[buffer(2)]],
+        device  const char * residual                [[buffer(3)]],
+        device  const char * post                    [[buffer(4)]],
+        device  const char * comb                    [[buffer(5)]],
+        device        char * dst                     [[buffer(6)]],
+        constant ds4_metal_args_dsv4_hc_expand & hc [[buffer(7)]],
+        threadgroup   char * shmem                   [[threadgroup(0)]],
+        uint3  tgpig [[threadgroup_position_in_grid]],
+        ushort tiitg [[thread_index_in_threadgroup]],
+        ushort sgitg [[simdgroup_index_in_threadgroup]]) {
+    constexpr int NR0 = 64;
+    constexpr int NR1 = 32;
+    constexpr int NK = 32;
+    constexpr int NL0 = NK / 16;
+    constexpr int NL1 = NK / 8;
+    constexpr short NL = 16;
+
+    if (hc.n_hc != 4 || mm.ne0 != hc.n_embd || mm.ne1 != hc.n_tokens ||
+        mm.ne00 != 8192 || mm.ne0 != 4096 || (mm.ne1 & 31) != 0) {
+        return;
+    }
+
+    threadgroup half * sa = (threadgroup half *)shmem;
+    threadgroup half * sb = (threadgroup half *)(shmem + 4096);
+
+    const int im = tgpig.z;
+    const int r0 = tgpig.y * NR0;
+    const int r1 = tgpig.x * NR1;
+    const short lr0 = (short)tiitg / NL0;
+    const short lr1 = (short)tiitg / NL1;
+    const short il0 = tiitg % NL0;
+    short il = il0;
+
+    const int i12 = im % mm.ne12;
+    const int i13 = im / mm.ne12;
+    const uint64_t offset0 =
+        (i12 / mm.r2) * mm.nb02 + (i13 / mm.r3) * mm.nb03;
+    device const ds4_dense_block_q4_K * x =
+        (device const ds4_dense_block_q4_K *)(
+            weight + mm.nb01 * (r0 + lr0) + offset0);
+
+    const short iy = 8 * (tiitg % NL1);
+    device const half * y = (device const half *)(input +
+        mm.nb13 * i13 + mm.nb12 * i12 + mm.nb11 * (r1 + lr1) + mm.nb10 * iy);
+
+    simdgroup_half8x8 ma[4];
+    simdgroup_half8x8 mb[2];
+    simdgroup_float8x8 mc[8];
+    FOR_UNROLL (short i = 0; i < 8; ++i) {
+        mc[i] = make_filled_simdgroup_matrix<float, 8>(0.0f);
+    }
+
+    for (int loop_k = 0; loop_k < mm.ne00; loop_k += NK) {
+        half4x4 temp_a;
+        dequantize_dense_q4_K(x, il, temp_a);
+
+        threadgroup_barrier(mem_flags::mem_threadgroup);
+
+        FOR_UNROLL (short i = 0; i < 16; ++i) {
+            const short sx = 2 * il0 + i / 8;
+            const short sy = (tiitg / NL0) / 8;
+            const short lx = (tiitg / NL0) % 8;
+            const short ly = i % 8;
+            const short ib = 8 * sx + sy;
+            *(sa + 64 * ib + 8 * ly + lx) = temp_a[i / 4][i % 4];
+        }
+
+        const short sx = tiitg % NL1;
+        const short sy = (tiitg / NL1) / 8;
+        const short ly = (tiitg / NL1) % 8;
+        const short ib = 4 * sx + sy;
+        *(threadgroup half2x4 *)(sb + 64 * ib + 8 * ly) =
+            *((device half2x4 *)y);
+
+        il = (il + 2 < NL) ? il + 2 : il % 2;
+        x = (il < 2) ? x + 1 : x;
+        y += NK;
+
+        threadgroup_barrier(mem_flags::mem_threadgroup);
+
+        threadgroup const half * lsma = sa + 4 * 64 * (sgitg % 2);
+        threadgroup const half * lsmb = sb + 2 * 64 * (sgitg / 2);
+
+        FOR_UNROLL (short ik = 0; ik < NK / 8; ++ik) {
+            simdgroup_barrier(mem_flags::mem_none);
+            FOR_UNROLL (short i = 0; i < 4; ++i) {
+                simdgroup_load(ma[i], lsma + 64 * i, 8, 0, false);
+            }
+            simdgroup_barrier(mem_flags::mem_none);
+            FOR_UNROLL (short i = 0; i < 2; ++i) {
+                simdgroup_load(mb[i], lsmb + 64 * i, 8, 0, false);
+            }
+            simdgroup_barrier(mem_flags::mem_none);
+            FOR_UNROLL (short i = 0; i < 8; ++i) {
+                simdgroup_multiply_accumulate(mc[i], mb[i / 4], ma[i % 4], mc[i]);
+            }
+            lsma += 8 * 64;
+            lsmb += 4 * 64;
+        }
+    }
+
+    threadgroup_barrier(mem_flags::mem_threadgroup);
+    threadgroup float * tile = (threadgroup float *)shmem;
+    threadgroup float * tile_sg =
+        tile + 32 * (sgitg & 1) + 16 * (sgitg >> 1) * NR0;
+    FOR_UNROLL (short i = 0; i < 8; ++i) {
+        simdgroup_store(mc[i],
+                        tile_sg + 8 * (i % 4) + 8 * NR0 * (i / 4),
+                        NR0, 0, false);
+    }
+    threadgroup_barrier(mem_flags::mem_threadgroup);
+
+    for (uint e = tiitg; e < (uint)(NR0 * NR1); e += 128u) {
+        const int64_t t = r1 + e / NR0;
+        const int64_t d = r0 + e % NR0;
+        const float block_v = tile[e];
+
+        const float rv0 = *((device const float *)(residual +
+            d * hc.nb_res0 + 0 * hc.nb_res1 + t * hc.nb_res2));
+        const float rv1 = *((device const float *)(residual +
+            d * hc.nb_res0 + 1 * hc.nb_res1 + t * hc.nb_res2));
+        const float rv2 = *((device const float *)(residual +
+            d * hc.nb_res0 + 2 * hc.nb_res1 + t * hc.nb_res2));
+        const float rv3 = *((device const float *)(residual +
+            d * hc.nb_res0 + 3 * hc.nb_res1 + t * hc.nb_res2));
+
+        for (int64_t dst_hc = 0; dst_hc < 4; ++dst_hc) {
+            float acc = block_v * *((device const float *)(post +
+                dst_hc * hc.nb_post0 + t * hc.nb_post1));
+            acc += *((device const float *)(comb +
+                dst_hc * hc.nb_comb0 + 0 * hc.nb_comb1 + t * hc.nb_comb2)) * rv0;
+            acc += *((device const float *)(comb +
+                dst_hc * hc.nb_comb0 + 1 * hc.nb_comb1 + t * hc.nb_comb2)) * rv1;
+            acc += *((device const float *)(comb +
+                dst_hc * hc.nb_comb0 + 2 * hc.nb_comb1 + t * hc.nb_comb2)) * rv2;
+            acc += *((device const float *)(comb +
+                dst_hc * hc.nb_comb0 + 3 * hc.nb_comb1 + t * hc.nb_comb2)) * rv3;
+            *((device float *)(dst + d * hc.nb0 + dst_hc * hc.nb1 +
+                               t * hc.nb2)) = acc;
+        }
+    }
+}
+
 // Decode-time FFN tail fusion:
 //
 //     shared_out = shared_mid @ Wshared_down
@@ -1070,6 +1370,38 @@ kernel void kernel_dsv4_hc_weighted_sum(
     *((device float *) (dst + d*args.nb0 + t*args.nb1)) = acc;
 }
 
+// DSpark needs both the complete prompt matrix and its final reduced row.
+// Keep this as a separate entry point so the ordinary HC weighted-sum shader
+// retains its original ABI and instruction stream. The extra store reuses the
+// same accumulator before it leaves the register and replaces a 16 KiB blit.
+kernel void kernel_dsv4_hc_weighted_sum_capture_last(
+        constant ds4_metal_args_dsv4_hc_weighted_sum & args,
+        device  const char * x,
+        device  const char * weights,
+        device        char * dst,
+        device        char * last_dst,
+        uint gid [[thread_position_in_grid]]) {
+    const int64_t n_elem = args.n_embd * args.n_tokens;
+    if ((int64_t) gid >= n_elem) {
+        return;
+    }
+
+    const int64_t d = ((int64_t) gid) % args.n_embd;
+    const int64_t t = ((int64_t) gid) / args.n_embd;
+
+    float acc = 0.0f;
+    for (int64_t h = 0; h < args.n_hc; ++h) {
+        const float xv = *((device const float *) (x       + d*args.nb_x0 + h*args.nb_x1 + t*args.nb_x2));
+        const float wv = *((device const float *) (weights + h*args.nb_w0 + t*args.nb_w1));
+        acc += xv * wv;
+    }
+
+    *((device float *) (dst + d*args.nb0 + t*args.nb1)) = acc;
+    if (t + 1 == args.n_tokens) {
+        *((device float *) (last_dst + d*args.nb0)) = acc;
+    }
+}
+
 // The one-row HC=4 output head historically materializes four device-F32
 // stages across separate launches. Collapse those launches into one tiny
 // two-thread group while preserving the scalar/vector lane mapping and every
@@ -1106,7 +1438,6 @@ kernel void kernel_dsv4_output_hc_weights4(
     }
 }
 
-
 struct ds4_metal_args_hc_norm_mix {
     int32_t n;
     int32_t out_dim;
@@ -1202,7 +1533,6 @@ kernel void kernel_dsv4_hc_rms_norm_mix_f16(
             FOR_UNROLL (short i = 0; i < NF4; ++i) {
                 sumq += dot(float4(xb4[i]), yl4[i]);
             }
-
             sumf_mv[row] += sumq;
         }
     }
diff --git a/metal/dsv4_misc.metal b/metal/dsv4_misc.metal
index a9b2397f96..6b896e75f5 100644
--- a/metal/dsv4_misc.metal
+++ b/metal/dsv4_misc.metal
@@ -518,16 +518,6 @@ kernel void kernel_dsv4_router_weights_one(
     w[tid] = p[s[tid]] / sum * 1.5f;
 }
 
-static inline float ds4_glm_router_sigmoid(float x) {
-    if (x >= 0.0f) {
-        const float e = exp(-x);
-        return 1.0f / (1.0f + e);
-    } else {
-        const float e = exp(x);
-        return e / (1.0f + e);
-    }
-}
-
 static inline bool ds4_glm_router_better(
         threadgroup const float *scores,
         int32_t                  a,
@@ -2065,139 +2055,6 @@ kernel void kernel_glm_indexer_scores_batch(
     if (tid == 0) *dst = score;
 }
 
-kernel void kernel_glm_indexer_scores_tiled_f32(
-        constant ds4_metal_args_glm_indexer_scores_batch & args,
-        device const char *q,
-        device const char *weights,
-        device const char *indexer_key_cache,
-        device char *scores,
-        threadgroup float *shared [[threadgroup(0)]],
-        uint2  tgpig [[threadgroup_position_in_grid]],
-        ushort tid   [[thread_index_in_threadgroup]],
-        ushort lane  [[thread_index_in_simdgroup]],
-        ushort sg    [[simdgroup_index_in_threadgroup]]) {
-    constexpr uint TM = 8;
-    constexpr uint TN = 32;
-    constexpr uint TS = 8;
-    constexpr uint D  = 128;
-
-    const uint row_base = tgpig.x * TN;
-    const uint token_base = tgpig.y * TM;
-
-    threadgroup float *qtg = shared;
-    threadgroup float *ktg = qtg + TM*D;
-    threadgroup float *dot = ktg + TN*D;
-
-    const uint last_token = min(token_base + TM, args.n_tokens);
-    const uint max_visible = last_token > token_base ?
-        glm_indexer_batch_visible_rows(args, last_token - 1u) : 0u;
-
-    if (row_base >= max_visible) {
-        for (uint i = tid; i < TM*TN; i += 128) {
-            const uint tr = i / TN;
-            const uint rc = i - tr*TN;
-            const uint token = token_base + tr;
-            const uint row = row_base + rc;
-            if (token < args.n_tokens && row < args.n_rows) {
-                device float *dst = (device float *)(scores +
-                    (uint64_t)token * args.score_token_stride) + row;
-                *dst = -INFINITY;
-            }
-        }
-        return;
-    }
-
-    for (uint i = tid; i < TN*D; i += 128) {
-        const uint rc = i / D;
-        const uint d = i - rc*D;
-        const uint row = row_base + rc;
-        float v = 0.0f;
-        if (row < args.n_rows) {
-            v = glm_cache_load_f32_or_f16(indexer_key_cache,
-                                          (uint64_t)row * args.head_dim + d,
-                                          args.cache_f16);
-        }
-        ktg[i] = v;
-    }
-
-    const uint cell0 = lane;
-    const uint cell1 = lane + 32u;
-    const uint token_row0 = cell0 >> 3;
-    const uint token_row1 = cell1 >> 3;
-    const uint sub0 = cell0 & 7u;
-    const uint sub1 = cell1 & 7u;
-    const uint col0 = (uint)sg * TS + sub0;
-    const uint col1 = (uint)sg * TS + sub1;
-    const uint token0 = token_base + token_row0;
-    const uint token1 = token_base + token_row1;
-    const uint row0 = row_base + col0;
-    const uint row1 = row_base + col1;
-
-    float acc0 = 0.0f;
-    float acc1 = 0.0f;
-
-    threadgroup_barrier(mem_flags::mem_threadgroup);
-
-    for (uint head = 0; head < args.n_head; head++) {
-        for (uint i = tid; i < TM*D; i += 128) {
-            const uint tr = i / D;
-            const uint d = i - tr*D;
-            const uint token = token_base + tr;
-            float v = 0.0f;
-            if (token < args.n_tokens) {
-                device const float *qrow = (device const float *)(q +
-                    (uint64_t)token * args.q_token_stride +
-                    (uint64_t)head  * args.q_head_stride);
-                v = qrow[d];
-            }
-            qtg[i] = v;
-        }
-
-        threadgroup_barrier(mem_flags::mem_threadgroup);
-
-        simdgroup_float8x8 mdot = make_filled_simdgroup_matrix<float, 8>(0.0f);
-        for (uint db = 0; db < D/TS; db++) {
-            simdgroup_float8x8 mq;
-            simdgroup_float8x8 mk;
-            simdgroup_load(mq, qtg + db*TS, D, 0, false);
-            simdgroup_load(mk, ktg + ((uint)sg * TS) * D + db*TS, D, 0, true);
-            simdgroup_multiply_accumulate(mdot, mq, mk, mdot);
-        }
-
-        simdgroup_store(mdot, dot + (uint)sg * TS, TN, 0, false);
-
-        threadgroup_barrier(mem_flags::mem_threadgroup);
-
-        if (token0 < args.n_tokens && row0 < args.n_rows) {
-            device const float *w = (device const float *)(weights +
-                (uint64_t)token0 * args.weights_token_stride);
-            const float s = dot[token_row0*TN + col0];
-            acc0 += max(s * args.scale, 0.0f) * w[head];
-        }
-        if (token1 < args.n_tokens && row1 < args.n_rows) {
-            device const float *w = (device const float *)(weights +
-                (uint64_t)token1 * args.weights_token_stride);
-            const float s = dot[token_row1*TN + col1];
-            acc1 += max(s * args.scale, 0.0f) * w[head];
-        }
-
-        threadgroup_barrier(mem_flags::mem_threadgroup);
-    }
-
-    if (token0 < args.n_tokens && row0 < args.n_rows) {
-        const uint visible = glm_indexer_batch_visible_rows(args, token0);
-        device float *dst = (device float *)(scores +
-            (uint64_t)token0 * args.score_token_stride) + row0;
-        *dst = row0 < visible ? acc0 : -INFINITY;
-    }
-    if (token1 < args.n_tokens && row1 < args.n_rows) {
-        const uint visible = glm_indexer_batch_visible_rows(args, token1);
-        device float *dst = (device float *)(scores +
-            (uint64_t)token1 * args.score_token_stride) + row1;
-        *dst = row1 < visible ? acc1 : -INFINITY;
-    }
-}
-
 kernel void kernel_glm_indexer_scores_tiled(
         constant ds4_metal_args_glm_indexer_scores_batch & args,
         device const char *q,
@@ -2607,34 +2464,6 @@ kernel void kernel_glm_qk_lowrank_q8_0_batch_glm52_t4(
     }
 }
 
-kernel void kernel_glm_value_project_q8_0(
-        constant ds4_metal_args_glm_qk_lowrank & args,
-        device const char *weight,
-        device const char *lora,
-        device char *heads,
-        threadgroup float *x [[threadgroup(0)]],
-        uint tid [[thread_index_in_threadgroup]],
-        ushort3 ntg_u [[threads_per_threadgroup]],
-        uint3 tgpig [[threadgroup_position_in_grid]]) {
-    const uint head = tgpig.x;
-    if (head >= args.n_head) return;
-    const uint nth = ntg_u.x;
-    device const float *src =
-        (device const float *)(lora + (uint64_t)head * args.kv_lora_dim * sizeof(float));
-    for (uint j = tid; j < args.kv_lora_dim; j += nth) {
-        x[j] = src[j];
-    }
-    threadgroup_barrier(mem_flags::mem_threadgroup);
-
-    device float *out =
-        (device float *)(heads + (uint64_t)head * args.qk_dim * sizeof(float));
-    for (uint d = tid; d < args.qk_dim; d += nth) {
-        device const char *row =
-            weight + ((uint64_t)head * args.qk_dim + d) * args.row_bytes;
-        out[d] = glm_quant_dot_row_tg_f32(args.weight_type, row, x, args.kv_lora_dim);
-    }
-}
-
 kernel void kernel_glm_value_project_q8_0_batch_heads(
         constant ds4_metal_args_glm_qk_lowrank_batch & args,
         device const char *weight,
@@ -4783,7 +4612,7 @@ kernel void kernel_glm_router_select_one(
 
     const uint n_expert = min(args.n_expert, 512u);
     const bool active = tid < n_expert;
-    const float p = active ? ds4_glm_router_sigmoid(token_logits[tid]) : 0.0f;
+    const float p = active ? ds4_sigmoid_stable(token_logits[tid]) : 0.0f;
     if (active) token_probs[tid] = p;
     sel_scores[tid] = active ? p + bias[tid] : -INFINITY;
     idx[tid] = (int32_t)tid;
@@ -5506,118 +5335,6 @@ kernel void kernel_dsv4_sort_i32_rows_asc(
     }
 }
 
-static inline void dsv4_attend_f32_row_as_f16(
-        device const char *kv,
-        uint64_t row_stride,
-        uint row,
-        half4 q0,
-        half4 q1,
-        half4 q2,
-        half4 q3,
-        float scale,
-        ushort lane,
-        thread float &M,
-        thread float &S,
-        thread float4 &o0,
-        thread float4 &o1,
-        thread float4 &o2,
-        thread float4 &o3) {
-    device const float4 *kv4 = (device const float4 *)(kv + (uint64_t)row * row_stride);
-    const half4 k0 = (half4)kv4[lane +  0];
-    const half4 k1 = (half4)kv4[lane + 32];
-    const half4 k2 = (half4)kv4[lane + 64];
-    const half4 k3 = (half4)kv4[lane + 96];
-
-    float score = dot((float4)q0, (float4)k0) +
-                  dot((float4)q1, (float4)k1) +
-                  dot((float4)q2, (float4)k2) +
-                  dot((float4)q3, (float4)k3);
-    score = simd_sum(score) * scale;
-
-    const float old_m = M;
-    const float new_m = max(M, score);
-    const float old_scale = exp(old_m - new_m);
-    const float row_scale = exp(score - new_m);
-
-    S = S * old_scale + row_scale;
-    o0 *= old_scale;
-    o1 *= old_scale;
-    o2 *= old_scale;
-    o3 *= old_scale;
-
-    o0 += (float4)k0 * row_scale;
-    o1 += (float4)k1 * row_scale;
-    o2 += (float4)k2 * row_scale;
-    o3 += (float4)k3 * row_scale;
-    M = new_m;
-}
-
-static inline void dsv4_attend_shared_f32_row_as_f16(
-        threadgroup const float4 *kv4,
-        half4 q0,
-        half4 q1,
-        half4 q2,
-        half4 q3,
-        float scale,
-        ushort lane,
-        thread float &M,
-        thread float &S,
-        thread float4 &o0,
-        thread float4 &o1,
-        thread float4 &o2,
-        thread float4 &o3) {
-    const half4 k0 = (half4)kv4[lane +  0];
-    const half4 k1 = (half4)kv4[lane + 32];
-    const half4 k2 = (half4)kv4[lane + 64];
-    const half4 k3 = (half4)kv4[lane + 96];
-
-    float score = dot((float4)q0, (float4)k0) +
-                  dot((float4)q1, (float4)k1) +
-                  dot((float4)q2, (float4)k2) +
-                  dot((float4)q3, (float4)k3);
-    score = simd_sum(score) * scale;
-
-    const float old_m = M;
-    const float new_m = max(M, score);
-    const float old_scale = exp(old_m - new_m);
-    const float row_scale = exp(score - new_m);
-
-    S = S * old_scale + row_scale;
-    o0 *= old_scale;
-    o1 *= old_scale;
-    o2 *= old_scale;
-    o3 *= old_scale;
-
-    o0 += (float4)k0 * row_scale;
-    o1 += (float4)k1 * row_scale;
-    o2 += (float4)k2 * row_scale;
-    o3 += (float4)k3 * row_scale;
-    M = new_m;
-}
-
-static inline void dsv4_attend_shared_f32_row_as_f16_at(
-        threadgroup const float4 *kv4,
-        uint row_in_tg,
-        half4 q0,
-        half4 q1,
-        half4 q2,
-        half4 q3,
-        float scale,
-        ushort lane,
-        thread float &M,
-        thread float &S,
-        thread float4 &o0,
-        thread float4 &o1,
-        thread float4 &o2,
-        thread float4 &o3) {
-    dsv4_attend_shared_f32_row_as_f16(kv4 + row_in_tg * 128u,
-                                      q0, q1, q2, q3,
-                                      scale,
-                                      lane,
-                                      M, S,
-                                      o0, o1, o2, o3);
-}
-
 static inline void dsv4_attend_shared_h4_row(
         threadgroup const half4 *kv4,
         half4 q0,
@@ -6251,26 +5968,6 @@ kernel void kernel_dsv4_indexed_mixed_attention_heads8_split_reduce(
     }
 }
 
-static inline float dsv4_indexer_dot128_shared_q(
-        float4 c0,
-        float4 c1,
-        float4 c2,
-        float4 c3,
-        threadgroup const float4 *q4,
-        ushort lane) {
-    float sum = 0.0f;
-    if (lane < 8) {
-        const ushort ib = lane >> 1;
-        const ushort il = lane & 1;
-        const ushort base = ib*8 + il*4;
-        sum += dot(c0, q4[base + 0]);
-        sum += dot(c1, q4[base + 1]);
-        sum += dot(c2, q4[base + 2]);
-        sum += dot(c3, q4[base + 3]);
-    }
-    return simd_sum(sum);
-}
-
 // Tiled prefill score builder for the sparse-compressed attention indexer.
 //
 // The kernel covers an 8-token by 32-compressed-row rectangle: K is copied into
@@ -7077,3 +6774,245 @@ kernel void kernel_dsv4_softmax_pool_ratio4_direct(
 
     dst[ic * args.head_dim + id] = acc/sum;
 }
+
+/* DSpark confidence/Markov tail.  The public result occupies the first
+ * 64 bytes of a shared buffer; this private state starts immediately after
+ * it.  Keeping the previous token and the 128 group candidates here lets a
+ * complete draft block use one CPU readback instead of one per row. */
+#define DS4_METAL_DSPARK_MAX_DRAFTS 6u
+#define DS4_METAL_DSPARK_MARKOV_GROUPS 128u
+
+struct ds4_metal_dspark_device_args {
+    uint  vocab;
+    uint  rank_blocks;
+    uint  hidden_dim;
+    uint  n_drafts;
+    uint  draft;
+    uint  reuse_confidence0;
+    float confidence_threshold;
+    float confidence0;
+};
+
+struct ds4_metal_dspark_device_result {
+    int   tokens[DS4_METAL_DSPARK_MAX_DRAFTS];
+    float confidence_logits[DS4_METAL_DSPARK_MAX_DRAFTS];
+    uint  proposal_len;
+    uint  confidence_len;
+    uint  status;
+    uint  reserved;
+};
+
+struct ds4_metal_dspark_device_state {
+    uint  prev_token;
+    uint  status;
+    uint  active;
+    uint  proposal_len;
+    uint  confidence_len;
+    uint  pad0;
+    uint  pad1;
+    uint  pad2;
+    int   tokens[DS4_METAL_DSPARK_MAX_DRAFTS];
+    float confidence_logits[DS4_METAL_DSPARK_MAX_DRAFTS];
+    float group_values[DS4_METAL_DSPARK_MARKOV_GROUPS];
+    uint  group_indices[DS4_METAL_DSPARK_MARKOV_GROUPS];
+};
+
+kernel void kernel_dsv4_dspark_device_proposal_init(
+        constant uint &first_prev_token [[buffer(0)]],
+        device ds4_metal_dspark_device_state *state [[buffer(1)]],
+        uint gid [[thread_position_in_grid]]) {
+    if (gid != 0u) return;
+    state->prev_token = first_prev_token;
+    state->status = 1u;
+    state->active = 1u;
+    state->proposal_len = 0u;
+    state->confidence_len = 0u;
+    for (uint i = 0u; i < DS4_METAL_DSPARK_MAX_DRAFTS; i++) {
+        state->tokens[i] = -1;
+        state->confidence_logits[i] = 0.0f;
+    }
+}
+
+static inline float ds4_metal_dspark_q8_value(
+        device const block_q8_0 *row,
+        uint index) {
+    const uint b = index >> 5u;
+    return float(row[b].d) * float(row[b].qs[index & 31u]);
+}
+
+kernel void kernel_dsv4_dspark_device_confidence_q8(
+        constant ds4_metal_dspark_device_args &args [[buffer(0)]],
+        device const float *hidden_rows [[buffer(1)]],
+        device const block_q8_0 *w1 [[buffer(2)]],
+        device const block_q8_0 *confidence [[buffer(3)]],
+        device ds4_metal_dspark_device_state *state [[buffer(4)]],
+        uint gid [[thread_position_in_grid]]) {
+    if (gid != 0u || state->status == 0u || state->active == 0u ||
+        args.draft >= args.n_drafts) return;
+
+    float acc = 0.0f;
+    if (args.draft == 0u && args.reuse_confidence0 != 0u) {
+        acc = args.confidence0;
+    } else {
+        const uint hidden_blocks = args.hidden_dim >> 5u;
+        const uint feature_blocks = hidden_blocks + args.rank_blocks;
+        device const block_q8_0 *w1_row =
+            w1 + (ulong)state->prev_token * args.rank_blocks;
+        device const float *hidden =
+            hidden_rows + (ulong)args.draft * args.hidden_dim;
+
+        for (uint b = 0u; b < feature_blocks; b++) {
+            float values[32];
+            float amax = 0.0f;
+            for (uint k = 0u; k < 32u; k++) {
+                const uint feature = b * 32u + k;
+                const float v = feature < args.hidden_dim
+                    ? hidden[feature]
+                    : ds4_metal_dspark_q8_value(
+                          w1_row, feature - args.hidden_dim);
+                values[k] = v;
+                amax = max(amax, abs(v));
+            }
+            const float xscale = amax / 127.0f;
+            const float inv = xscale != 0.0f ? 1.0f / xscale : 0.0f;
+            int isum = 0;
+            for (uint k = 0u; k < 32u; k++) {
+                int q = int(rint(values[k] * inv));
+                q = clamp(q, -128, 127);
+                isum += int(confidence[b].qs[k]) * q;
+            }
+            acc += float(confidence[b].d) * xscale * float(isum);
+        }
+    }
+
+    state->confidence_logits[args.draft] = acc;
+    state->confidence_len = args.draft + 1u;
+    const float e = exp(acc >= 0.0f ? -acc : acc);
+    const float probability = acc >= 0.0f
+        ? 1.0f / (1.0f + e)
+        : e / (1.0f + e);
+    if (probability < args.confidence_threshold) state->active = 0u;
+}
+
+static inline bool ds4_metal_dspark_score_better(
+        float av, uint ai, float bv, uint bi) {
+    return av > bv || (av == bv && ai < bi);
+}
+
+kernel void kernel_dsv4_dspark_device_markov_scan_q8(
+        constant ds4_metal_dspark_device_args &args [[buffer(0)]],
+        device const float *logits [[buffer(1)]],
+        device const block_q8_0 *w1 [[buffer(2)]],
+        device const block_q8_0 *w2 [[buffer(3)]],
+        device ds4_metal_dspark_device_state *state [[buffer(4)]],
+        uint tid [[thread_index_in_threadgroup]],
+        uint group [[threadgroup_position_in_grid]]) {
+    threadgroup float markov_state[256];
+    threadgroup float values[256];
+    threadgroup uint indices[256];
+
+    if (state->active == 0u || state->status == 0u) {
+        if (tid == 0u && group < DS4_METAL_DSPARK_MARKOV_GROUPS) {
+            state->group_values[group] = -INFINITY;
+            state->group_indices[group] = 0u;
+        }
+        return;
+    }
+
+    const uint rank = args.rank_blocks * 32u;
+    device const block_q8_0 *w1_row =
+        w1 + (ulong)state->prev_token * args.rank_blocks;
+    if (tid < rank) {
+        markov_state[tid] = ds4_metal_dspark_q8_value(w1_row, tid);
+    }
+    threadgroup_barrier(mem_flags::mem_threadgroup);
+
+    float best_v = -INFINITY;
+    uint best_i = 0u;
+    const uint first = group * 256u + tid;
+    const uint stride = DS4_METAL_DSPARK_MARKOV_GROUPS * 256u;
+    for (uint i = first; i < args.vocab; i += stride) {
+        device const block_q8_0 *row =
+            w2 + (ulong)i * args.rank_blocks;
+        float acc = 0.0f;
+        for (uint b = 0u; b < args.rank_blocks; b++) {
+            float s = 0.0f;
+            for (uint k = 0u; k < 32u; k++) {
+                s += float(row[b].qs[k]) * markov_state[b * 32u + k];
+            }
+            acc += float(row[b].d) * s;
+        }
+        const float v = logits[i] + acc;
+        if (ds4_metal_dspark_score_better(v, i, best_v, best_i)) {
+            best_v = v;
+            best_i = i;
+        }
+    }
+
+    values[tid] = best_v;
+    indices[tid] = best_i;
+    threadgroup_barrier(mem_flags::mem_threadgroup);
+    for (uint step = 128u; step != 0u; step >>= 1u) {
+        if (tid < step &&
+            ds4_metal_dspark_score_better(
+                    values[tid + step], indices[tid + step],
+                    values[tid], indices[tid])) {
+            values[tid] = values[tid + step];
+            indices[tid] = indices[tid + step];
+        }
+        threadgroup_barrier(mem_flags::mem_threadgroup);
+    }
+    if (tid == 0u) {
+        state->group_values[group] = values[0];
+        state->group_indices[group] = indices[0];
+    }
+}
+
+kernel void kernel_dsv4_dspark_device_markov_reduce(
+        constant ds4_metal_dspark_device_args &args [[buffer(0)]],
+        device ds4_metal_dspark_device_state *state [[buffer(1)]],
+        uint tid [[thread_index_in_threadgroup]]) {
+    threadgroup float values[DS4_METAL_DSPARK_MARKOV_GROUPS];
+    threadgroup uint indices[DS4_METAL_DSPARK_MARKOV_GROUPS];
+    if (state->active == 0u || state->status == 0u) return;
+
+    values[tid] = state->group_values[tid];
+    indices[tid] = state->group_indices[tid];
+    threadgroup_barrier(mem_flags::mem_threadgroup);
+    for (uint step = DS4_METAL_DSPARK_MARKOV_GROUPS / 2u;
+         step != 0u; step >>= 1u) {
+        if (tid < step &&
+            ds4_metal_dspark_score_better(
+                    values[tid + step], indices[tid + step],
+                    values[tid], indices[tid])) {
+            values[tid] = values[tid + step];
+            indices[tid] = indices[tid + step];
+        }
+        threadgroup_barrier(mem_flags::mem_threadgroup);
+    }
+    if (tid == 0u) {
+        const uint token = indices[0];
+        if (!isfinite(values[0]) || token >= args.vocab) {
+            state->status = 0u;
+            return;
+        }
+        state->tokens[args.draft] = int(token);
+        state->prev_token = token;
+        state->proposal_len = args.draft + 1u;
+    }
+}
+
+kernel void kernel_dsv4_dspark_device_proposal_export(
+        device ds4_metal_dspark_device_result *out [[buffer(0)]],
+        device const ds4_metal_dspark_device_state *state [[buffer(1)]],
+        uint gid [[thread_position_in_grid]]) {
+    if (gid != 0u) return;
+    for (uint i = 0u; i < DS4_METAL_DSPARK_MAX_DRAFTS; i++) {
+        out->tokens[i] = state->tokens[i];
+        out->confidence_logits[i] = state->confidence_logits[i];
+    }
+    out->proposal_len = state->proposal_len;
+    out->confidence_len = state->confidence_len;
+    out->status = state->status;
+    out->reserved = 0u;
+}
diff --git a/metal/glu.metal b/metal/glu.metal
index fd5c6fd096..80af3117a4 100644
--- a/metal/glu.metal
+++ b/metal/glu.metal
@@ -11,34 +11,6 @@ struct ds4_metal_args_glu {
     float    limit;
 };
 
-// SwiGLU activation for the FFN inner state. DS4 clamps the shared expert with
-// the same swiglu_limit used by routed experts.
-kernel void kernel_swiglu_f32(
-        constant ds4_metal_args_glu & args,
-        device const char * src0,
-        device const char * src1,
-        device       char * dst,
-        uint tgpig[[threadgroup_position_in_grid]],
-        uint tpitg[[thread_position_in_threadgroup]],
-        uint   ntg[[threads_per_threadgroup]]) {
-    device const float * src0_row = (device const float *) ((device const char *) src0 + tgpig*args.nb01) + args.i00;
-    device const float * src1_row = (device const float *) ((device const char *) src1 + tgpig*args.nb11) + args.i10;
-    device       float * dst_row  = (device       float *) ((device       char *) dst  + tgpig*args.nb1);
-
-    for (int i0 = tpitg; i0 < args.ne0; i0 += ntg) {
-        float x0 = src0_row[i0];
-        float x1 = src1_row[i0];
-        if (args.limit > 1.0e-6f) {
-            x0 = min(x0, args.limit);
-            x1 = clamp(x1, -args.limit, args.limit);
-        }
-
-        const float silu = x0 / (1.0f + exp(-x0));
-
-        dst_row[i0] = silu*x1*args.alpha;
-    }
-}
-
 kernel void kernel_swiglu_flat_f32(
         constant ds4_metal_args_glu & args,
         device const char * src0,
@@ -58,6 +30,6 @@ kernel void kernel_swiglu_flat_f32(
         x1 = clamp(x1, -args.limit, args.limit);
     }
 
-    const float silu = x0 / (1.0f + exp(-x0));
+    const float silu = ds4_silu(x0);
     dst_f32[i] = silu*x1*args.alpha;
 }
diff --git a/metal/moe.metal b/metal/moe.metal
index 9d0840d7da..930d6fb622 100644
--- a/metal/moe.metal
+++ b/metal/moe.metal
@@ -375,6 +375,266 @@ static constant ulong ds4_metal_iq2xxs_grid[256] = {
 #define ksigns_iq2xs ds4_metal_ksigns_iq2xs
 #define iq2xxs_grid ds4_metal_iq2xxs_grid
 
+// iq2xxs_grid entry (row) x byte: exact binary16 of 0.25 * grid byte.
+static constant ushort ds4_metal_iq2xxs_half_values[256][8] = {
+    { 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u },
+    { 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u },
+    { 0x4640u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u },
+    { 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u },
+    { 0x4960u, 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u },
+    { 0x4640u, 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u },
+    { 0x4000u, 0x4640u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u },
+    { 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u },
+    { 0x4960u, 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u },
+    { 0x4000u, 0x4960u, 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u },
+    { 0x4960u, 0x4960u, 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u },
+    { 0x4640u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4000u },
+    { 0x4000u, 0x4640u, 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4000u },
+    { 0x4000u, 0x4000u, 0x4640u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4000u },
+    { 0x4000u, 0x4960u, 0x4640u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4000u },
+    { 0x4640u, 0x4000u, 0x4960u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4000u },
+    { 0x4000u, 0x4640u, 0x4960u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4000u },
+    { 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4000u },
+    { 0x4960u, 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4000u },
+    { 0x4960u, 0x4960u, 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4000u },
+    { 0x4960u, 0x4000u, 0x4960u, 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4000u },
+    { 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4000u },
+    { 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4000u },
+    { 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4000u },
+    { 0x4640u, 0x4640u, 0x4640u, 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4000u },
+    { 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4640u, 0x4000u, 0x4000u, 0x4000u },
+    { 0x4000u, 0x4640u, 0x4000u, 0x4960u, 0x4640u, 0x4000u, 0x4000u, 0x4000u },
+    { 0x4000u, 0x4960u, 0x4640u, 0x4960u, 0x4640u, 0x4000u, 0x4000u, 0x4000u },
+    { 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4000u },
+    { 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4000u },
+    { 0x4960u, 0x4000u, 0x4960u, 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4000u },
+    { 0x4960u, 0x4000u, 0x4000u, 0x4960u, 0x4960u, 0x4000u, 0x4000u, 0x4000u },
+    { 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4000u },
+    { 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4000u },
+    { 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4000u },
+    { 0x4640u, 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4000u },
+    { 0x4000u, 0x4640u, 0x4960u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4000u },
+    { 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4640u, 0x4000u, 0x4000u },
+    { 0x4960u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4640u, 0x4000u, 0x4000u },
+    { 0x4000u, 0x4960u, 0x4000u, 0x4640u, 0x4000u, 0x4640u, 0x4000u, 0x4000u },
+    { 0x4000u, 0x4000u, 0x4960u, 0x4640u, 0x4000u, 0x4640u, 0x4000u, 0x4000u },
+    { 0x4640u, 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4640u, 0x4000u, 0x4000u },
+    { 0x4000u, 0x4640u, 0x4000u, 0x4960u, 0x4000u, 0x4640u, 0x4000u, 0x4000u },
+    { 0x4000u, 0x4000u, 0x4640u, 0x4960u, 0x4000u, 0x4640u, 0x4000u, 0x4000u },
+    { 0x4000u, 0x4640u, 0x4960u, 0x4960u, 0x4000u, 0x4640u, 0x4000u, 0x4000u },
+    { 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4640u, 0x4000u, 0x4000u },
+    { 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4640u, 0x4000u, 0x4000u },
+    { 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4640u, 0x4640u, 0x4000u, 0x4000u },
+    { 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4640u, 0x4640u, 0x4000u, 0x4000u },
+    { 0x4960u, 0x4640u, 0x4000u, 0x4640u, 0x4640u, 0x4640u, 0x4000u, 0x4000u },
+    { 0x4640u, 0x4960u, 0x4960u, 0x4640u, 0x4640u, 0x4640u, 0x4000u, 0x4000u },
+    { 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4640u, 0x4640u, 0x4000u, 0x4000u },
+    { 0x4640u, 0x4000u, 0x4640u, 0x4960u, 0x4640u, 0x4640u, 0x4000u, 0x4000u },
+    { 0x4640u, 0x4960u, 0x4000u, 0x4000u, 0x4960u, 0x4640u, 0x4000u, 0x4000u },
+    { 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4960u, 0x4640u, 0x4000u, 0x4000u },
+    { 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4960u, 0x4640u, 0x4000u, 0x4000u },
+    { 0x4000u, 0x4640u, 0x4000u, 0x4960u, 0x4960u, 0x4640u, 0x4000u, 0x4000u },
+    { 0x4000u, 0x4640u, 0x4960u, 0x4960u, 0x4960u, 0x4640u, 0x4000u, 0x4000u },
+    { 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4000u },
+    { 0x4640u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4000u },
+    { 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4000u },
+    { 0x4000u, 0x4640u, 0x4640u, 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4000u },
+    { 0x4000u, 0x4960u, 0x4960u, 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4000u },
+    { 0x4640u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4960u, 0x4000u, 0x4000u },
+    { 0x4000u, 0x4640u, 0x4000u, 0x4640u, 0x4000u, 0x4960u, 0x4000u, 0x4000u },
+    { 0x4000u, 0x4000u, 0x4640u, 0x4640u, 0x4000u, 0x4960u, 0x4000u, 0x4000u },
+    { 0x4960u, 0x4000u, 0x4640u, 0x4640u, 0x4000u, 0x4960u, 0x4000u, 0x4000u },
+    { 0x4000u, 0x4960u, 0x4000u, 0x4960u, 0x4000u, 0x4960u, 0x4000u, 0x4000u },
+    { 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4640u, 0x4960u, 0x4000u, 0x4000u },
+    { 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4640u, 0x4960u, 0x4000u, 0x4000u },
+    { 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4960u, 0x4000u, 0x4000u },
+    { 0x4000u, 0x4640u, 0x4640u, 0x4000u, 0x4960u, 0x4960u, 0x4000u, 0x4000u },
+    { 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4000u },
+    { 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4000u },
+    { 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4000u },
+    { 0x4640u, 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4000u },
+    { 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4640u, 0x4000u },
+    { 0x4000u, 0x4000u, 0x4960u, 0x4640u, 0x4000u, 0x4000u, 0x4640u, 0x4000u },
+    { 0x4000u, 0x4640u, 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4640u, 0x4000u },
+    { 0x4000u, 0x4000u, 0x4640u, 0x4960u, 0x4000u, 0x4000u, 0x4640u, 0x4000u },
+    { 0x4640u, 0x4640u, 0x4640u, 0x4960u, 0x4000u, 0x4000u, 0x4640u, 0x4000u },
+    { 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4640u, 0x4000u },
+    { 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4640u, 0x4000u },
+    { 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4640u, 0x4000u, 0x4640u, 0x4000u },
+    { 0x4000u, 0x4000u, 0x4640u, 0x4640u, 0x4640u, 0x4000u, 0x4640u, 0x4000u },
+    { 0x4960u, 0x4960u, 0x4640u, 0x4640u, 0x4640u, 0x4000u, 0x4640u, 0x4000u },
+    { 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4640u, 0x4000u, 0x4640u, 0x4000u },
+    { 0x4000u, 0x4640u, 0x4960u, 0x4000u, 0x4960u, 0x4000u, 0x4640u, 0x4000u },
+    { 0x4640u, 0x4640u, 0x4000u, 0x4640u, 0x4960u, 0x4000u, 0x4640u, 0x4000u },
+    { 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4640u, 0x4000u },
+    { 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4640u, 0x4000u },
+    { 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4640u, 0x4640u, 0x4000u },
+    { 0x4640u, 0x4640u, 0x4960u, 0x4000u, 0x4000u, 0x4640u, 0x4640u, 0x4000u },
+    { 0x4640u, 0x4960u, 0x4000u, 0x4640u, 0x4000u, 0x4640u, 0x4640u, 0x4000u },
+    { 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4640u, 0x4640u, 0x4000u },
+    { 0x4000u, 0x4960u, 0x4640u, 0x4000u, 0x4640u, 0x4640u, 0x4640u, 0x4000u },
+    { 0x4960u, 0x4000u, 0x4960u, 0x4640u, 0x4640u, 0x4640u, 0x4640u, 0x4000u },
+    { 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4640u, 0x4640u, 0x4000u },
+    { 0x4960u, 0x4640u, 0x4640u, 0x4000u, 0x4960u, 0x4640u, 0x4640u, 0x4000u },
+    { 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4640u, 0x4000u },
+    { 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4640u, 0x4000u },
+    { 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4960u, 0x4640u, 0x4000u },
+    { 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4960u, 0x4640u, 0x4000u },
+    { 0x4640u, 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4960u, 0x4640u, 0x4000u },
+    { 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4960u, 0x4640u, 0x4000u },
+    { 0x4640u, 0x4640u, 0x4000u, 0x4000u, 0x4640u, 0x4960u, 0x4640u, 0x4000u },
+    { 0x4000u, 0x4000u, 0x4960u, 0x4960u, 0x4640u, 0x4960u, 0x4640u, 0x4000u },
+    { 0x4640u, 0x4000u, 0x4640u, 0x4640u, 0x4960u, 0x4960u, 0x4640u, 0x4000u },
+    { 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4000u },
+    { 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4000u },
+    { 0x4960u, 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4000u },
+    { 0x4000u, 0x4640u, 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4960u, 0x4000u },
+    { 0x4640u, 0x4000u, 0x4960u, 0x4640u, 0x4000u, 0x4000u, 0x4960u, 0x4000u },
+    { 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4960u, 0x4000u },
+    { 0x4960u, 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4960u, 0x4000u },
+    { 0x4640u, 0x4960u, 0x4960u, 0x4000u, 0x4640u, 0x4000u, 0x4960u, 0x4000u },
+    { 0x4000u, 0x4960u, 0x4000u, 0x4640u, 0x4640u, 0x4000u, 0x4960u, 0x4000u },
+    { 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4960u, 0x4000u },
+    { 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4960u, 0x4000u },
+    { 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4960u, 0x4000u },
+    { 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4960u, 0x4000u },
+    { 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4640u, 0x4960u, 0x4000u },
+    { 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4640u, 0x4960u, 0x4000u },
+    { 0x4960u, 0x4640u, 0x4640u, 0x4640u, 0x4000u, 0x4640u, 0x4960u, 0x4000u },
+    { 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4640u, 0x4960u, 0x4000u },
+    { 0x4640u, 0x4000u, 0x4000u, 0x4640u, 0x4640u, 0x4640u, 0x4960u, 0x4000u },
+    { 0x4000u, 0x4640u, 0x4960u, 0x4640u, 0x4640u, 0x4640u, 0x4960u, 0x4000u },
+    { 0x4000u, 0x4000u, 0x4640u, 0x4960u, 0x4960u, 0x4640u, 0x4960u, 0x4000u },
+    { 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4960u, 0x4000u },
+    { 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4960u, 0x4960u, 0x4000u },
+    { 0x4000u, 0x4640u, 0x4640u, 0x4960u, 0x4000u, 0x4960u, 0x4960u, 0x4000u },
+    { 0x4000u, 0x4640u, 0x4000u, 0x4640u, 0x4960u, 0x4960u, 0x4960u, 0x4000u },
+    { 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4640u },
+    { 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4640u },
+    { 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4640u },
+    { 0x4000u, 0x4960u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4640u },
+    { 0x4640u, 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4640u },
+    { 0x4000u, 0x4640u, 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4640u },
+    { 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4640u },
+    { 0x4000u, 0x4960u, 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4640u },
+    { 0x4960u, 0x4640u, 0x4640u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4640u },
+    { 0x4000u, 0x4000u, 0x4960u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4640u },
+    { 0x4640u, 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4640u },
+    { 0x4000u, 0x4640u, 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4640u },
+    { 0x4000u, 0x4000u, 0x4640u, 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4640u },
+    { 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4640u },
+    { 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4640u },
+    { 0x4640u, 0x4000u, 0x4960u, 0x4640u, 0x4640u, 0x4000u, 0x4000u, 0x4640u },
+    { 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4640u, 0x4000u, 0x4000u, 0x4640u },
+    { 0x4640u, 0x4640u, 0x4000u, 0x4960u, 0x4640u, 0x4000u, 0x4000u, 0x4640u },
+    { 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4640u },
+    { 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4640u },
+    { 0x4000u, 0x4960u, 0x4000u, 0x4640u, 0x4960u, 0x4000u, 0x4000u, 0x4640u },
+    { 0x4960u, 0x4640u, 0x4640u, 0x4640u, 0x4960u, 0x4000u, 0x4000u, 0x4640u },
+    { 0x4000u, 0x4960u, 0x4960u, 0x4640u, 0x4960u, 0x4000u, 0x4000u, 0x4640u },
+    { 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4640u },
+    { 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4640u },
+    { 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4640u },
+    { 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4640u, 0x4000u, 0x4640u },
+    { 0x4640u, 0x4960u, 0x4640u, 0x4960u, 0x4000u, 0x4640u, 0x4000u, 0x4640u },
+    { 0x4960u, 0x4000u, 0x4640u, 0x4000u, 0x4640u, 0x4640u, 0x4000u, 0x4640u },
+    { 0x4000u, 0x4640u, 0x4960u, 0x4000u, 0x4640u, 0x4640u, 0x4000u, 0x4640u },
+    { 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4640u, 0x4000u, 0x4640u },
+    { 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4640u },
+    { 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4640u },
+    { 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4640u },
+    { 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4960u, 0x4000u, 0x4640u },
+    { 0x4640u, 0x4640u, 0x4000u, 0x4640u, 0x4000u, 0x4960u, 0x4000u, 0x4640u },
+    { 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4960u, 0x4000u, 0x4640u },
+    { 0x4000u, 0x4960u, 0x4640u, 0x4640u, 0x4640u, 0x4960u, 0x4000u, 0x4640u },
+    { 0x4640u, 0x4000u, 0x4960u, 0x4640u, 0x4640u, 0x4960u, 0x4000u, 0x4640u },
+    { 0x4960u, 0x4000u, 0x4000u, 0x4960u, 0x4640u, 0x4960u, 0x4000u, 0x4640u },
+    { 0x4640u, 0x4640u, 0x4000u, 0x4640u, 0x4960u, 0x4960u, 0x4000u, 0x4640u },
+    { 0x4000u, 0x4000u, 0x4640u, 0x4960u, 0x4960u, 0x4960u, 0x4000u, 0x4640u },
+    { 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4640u },
+    { 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4640u },
+    { 0x4640u, 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4640u },
+    { 0x4640u, 0x4960u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4640u },
+    { 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4640u },
+    { 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4640u, 0x4640u },
+    { 0x4000u, 0x4960u, 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4640u, 0x4640u },
+    { 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4640u, 0x4640u },
+    { 0x4960u, 0x4000u, 0x4000u, 0x4640u, 0x4640u, 0x4000u, 0x4640u, 0x4640u },
+    { 0x4000u, 0x4640u, 0x4960u, 0x4960u, 0x4640u, 0x4000u, 0x4640u, 0x4640u },
+    { 0x4640u, 0x4000u, 0x4640u, 0x4960u, 0x4960u, 0x4000u, 0x4640u, 0x4640u },
+    { 0x4000u, 0x4000u, 0x4640u, 0x4960u, 0x4000u, 0x4640u, 0x4640u, 0x4640u },
+    { 0x4960u, 0x4000u, 0x4640u, 0x4960u, 0x4000u, 0x4640u, 0x4640u, 0x4640u },
+    { 0x4960u, 0x4960u, 0x4000u, 0x4000u, 0x4640u, 0x4640u, 0x4640u, 0x4640u },
+    { 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4640u, 0x4640u, 0x4640u },
+    { 0x4000u, 0x4640u, 0x4640u, 0x4640u, 0x4960u, 0x4640u, 0x4640u, 0x4640u },
+    { 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4640u, 0x4640u },
+    { 0x4640u, 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4960u, 0x4640u, 0x4640u },
+    { 0x4640u, 0x4960u, 0x4640u, 0x4000u, 0x4000u, 0x4960u, 0x4640u, 0x4640u },
+    { 0x4000u, 0x4640u, 0x4960u, 0x4640u, 0x4000u, 0x4960u, 0x4640u, 0x4640u },
+    { 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4640u, 0x4960u, 0x4640u, 0x4640u },
+    { 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4960u, 0x4960u, 0x4640u, 0x4640u },
+    { 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4640u },
+    { 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4640u },
+    { 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4960u, 0x4640u },
+    { 0x4000u, 0x4960u, 0x4960u, 0x4640u, 0x4000u, 0x4000u, 0x4960u, 0x4640u },
+    { 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4960u, 0x4640u },
+    { 0x4640u, 0x4640u, 0x4640u, 0x4640u, 0x4640u, 0x4000u, 0x4960u, 0x4640u },
+    { 0x4000u, 0x4960u, 0x4640u, 0x4000u, 0x4960u, 0x4000u, 0x4960u, 0x4640u },
+    { 0x4000u, 0x4000u, 0x4960u, 0x4640u, 0x4960u, 0x4000u, 0x4960u, 0x4640u },
+    { 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4960u, 0x4640u },
+    { 0x4640u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4960u, 0x4640u },
+    { 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4640u, 0x4640u, 0x4960u, 0x4640u },
+    { 0x4960u, 0x4000u, 0x4640u, 0x4000u, 0x4640u, 0x4640u, 0x4960u, 0x4640u },
+    { 0x4000u, 0x4640u, 0x4000u, 0x4960u, 0x4640u, 0x4640u, 0x4960u, 0x4640u },
+    { 0x4960u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4960u, 0x4960u, 0x4640u },
+    { 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4960u },
+    { 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4960u },
+    { 0x4960u, 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4960u },
+    { 0x4640u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4960u },
+    { 0x4960u, 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4960u },
+    { 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4960u },
+    { 0x4000u, 0x4960u, 0x4640u, 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4960u },
+    { 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4640u, 0x4000u, 0x4000u, 0x4960u },
+    { 0x4640u, 0x4000u, 0x4640u, 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4960u },
+    { 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4960u },
+    { 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4960u },
+    { 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4960u },
+    { 0x4640u, 0x4640u, 0x4640u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4960u },
+    { 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4640u, 0x4000u, 0x4960u },
+    { 0x4000u, 0x4000u, 0x4960u, 0x4640u, 0x4000u, 0x4640u, 0x4000u, 0x4960u },
+    { 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4640u, 0x4000u, 0x4960u },
+    { 0x4960u, 0x4640u, 0x4000u, 0x4640u, 0x4640u, 0x4640u, 0x4000u, 0x4960u },
+    { 0x4000u, 0x4640u, 0x4640u, 0x4960u, 0x4640u, 0x4640u, 0x4000u, 0x4960u },
+    { 0x4640u, 0x4960u, 0x4000u, 0x4000u, 0x4960u, 0x4640u, 0x4000u, 0x4960u },
+    { 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4960u, 0x4640u, 0x4000u, 0x4960u },
+    { 0x4000u, 0x4000u, 0x4960u, 0x4640u, 0x4960u, 0x4640u, 0x4000u, 0x4960u },
+    { 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4960u },
+    { 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4640u, 0x4960u, 0x4000u, 0x4960u },
+    { 0x4640u, 0x4000u, 0x4640u, 0x4000u, 0x4960u, 0x4960u, 0x4000u, 0x4960u },
+    { 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4960u },
+    { 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4960u },
+    { 0x4000u, 0x4640u, 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4960u },
+    { 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4640u, 0x4960u },
+    { 0x4640u, 0x4000u, 0x4960u, 0x4960u, 0x4000u, 0x4000u, 0x4640u, 0x4960u },
+    { 0x4960u, 0x4640u, 0x4640u, 0x4000u, 0x4640u, 0x4000u, 0x4640u, 0x4960u },
+    { 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4640u, 0x4000u, 0x4640u, 0x4960u },
+    { 0x4640u, 0x4640u, 0x4000u, 0x4640u, 0x4960u, 0x4000u, 0x4640u, 0x4960u },
+    { 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4640u, 0x4640u, 0x4960u },
+    { 0x4960u, 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4640u, 0x4640u, 0x4960u },
+    { 0x4000u, 0x4640u, 0x4000u, 0x4640u, 0x4000u, 0x4640u, 0x4640u, 0x4960u },
+    { 0x4640u, 0x4000u, 0x4640u, 0x4640u, 0x4640u, 0x4640u, 0x4640u, 0x4960u },
+    { 0x4640u, 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4960u, 0x4640u, 0x4960u },
+    { 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4640u, 0x4960u, 0x4640u, 0x4960u },
+    { 0x4960u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4960u },
+    { 0x4000u, 0x4000u, 0x4640u, 0x4640u, 0x4000u, 0x4000u, 0x4960u, 0x4960u },
+    { 0x4640u, 0x4640u, 0x4000u, 0x4960u, 0x4000u, 0x4000u, 0x4960u, 0x4960u },
+    { 0x4640u, 0x4960u, 0x4000u, 0x4000u, 0x4640u, 0x4000u, 0x4960u, 0x4960u },
+    { 0x4000u, 0x4000u, 0x4000u, 0x4000u, 0x4960u, 0x4000u, 0x4960u, 0x4960u },
+    { 0x4000u, 0x4960u, 0x4640u, 0x4000u, 0x4000u, 0x4640u, 0x4960u, 0x4960u },
+    { 0x4000u, 0x4000u, 0x4640u, 0x4640u, 0x4000u, 0x4960u, 0x4960u, 0x4960u },
+    { 0x4000u, 0x4640u, 0x4000u, 0x4000u, 0x4640u, 0x4960u, 0x4960u, 0x4960u },
+};
+
 struct block_q2_K {
     uchar scales[QK_K/16];
     uchar qs[QK_K/4];
@@ -442,7 +702,7 @@ static inline float ds4_glm_swiglu(float gate, float up, float limit) {
         gate = min(gate, limit);
         up = clamp(up, -limit, limit);
     }
-    return (gate / (1.0f + exp(-gate))) * up;
+    return ds4_silu(gate) * up;
 }
 
 
@@ -506,7 +766,7 @@ kernel void kernel_dsv4_moe_swiglu_weight(
                 up_row[i] = u;
             }
         }
-        const float silu = g / (1.0f + exp(-g));
+        const float silu = ds4_silu(g);
         mid_row[i] = silu * u * route_weight;
     }
 }
@@ -544,7 +804,7 @@ kernel void kernel_dsv4_moe_swiglu_weight_f16(
                 up_row[i] = u;
             }
         }
-        const float silu = g / (1.0f + exp(-g));
+        const float silu = ds4_silu(g);
         mid_row[i] = (half)(silu * u * route_weight);
     }
 }
@@ -620,20 +880,6 @@ void dequantize_q2_K(device const block_q2_K *xb, short il, thread type4x4 & reg
     }
 }
 
-static inline float ds4_glm_q2_K_value(device const block_q2_K *blocks, uint k) {
-    const uint block = k / QK_K;
-    const uint idx = k - block * QK_K;
-    device const block_q2_K *xb = blocks + block;
-    const uint group = idx / 16u;
-    const uint l = idx - group * 16u;
-    const uint q_base = 32u * (group / 8u) + 16u * (group & 1u);
-    const uint shift = ((group / 2u) & 3u) * 2u;
-    const uint q = ((uint)xb->qs[q_base + l] >> shift) & 0x03u;
-    const uint sc = (uint)xb->scales[group];
-    return (float)xb->d * (float)(sc & 0x0fu) * (float)q -
-           (float)xb->dmin * (float)(sc >> 4u);
-}
-
 static inline uchar2 get_scale_min_k4_just2(int j, int k, device const uchar * q) {
     return j < 4 ? uchar2{uchar(q[j+0+k] & 63), uchar(q[j+4+k] & 63)}
                  : uchar2{uchar((q[j+4+k] & 0xF) | ((q[j-4+k] & 0xc0) >> 2)),
@@ -654,115 +900,6 @@ static inline float ds4_glm_q4_K_value(device const block_q4_K *blocks, uint k)
            (float)xb->dmin * (float)sm.y;
 }
 
-static inline float ds4_glm_q5_K_value(device const block_q5_K *blocks, uint k) {
-    const uint block = k / QK_K;
-    const uint idx = k - block * QK_K;
-    device const block_q5_K *xb = blocks + block;
-    const uint group = idx / 32u;
-    const uint l = idx - group * 32u;
-    const uchar2 sm = get_scale_min_k4_just2((int)group, 0, xb->scales);
-    const uint ql_base = (group >> 1u) * 32u + l;
-    const uint shift = (group & 1u) * 4u;
-    uint q = (xb->qs[ql_base] >> shift) & 0x0Fu;
-    q += (xb->qh[l] & (uchar)(1u << group)) ? 16u : 0u;
-    return (float)xb->d * (float)sm.x * (float)q -
-           (float)xb->dmin * (float)sm.y;
-}
-
-static inline float ds4_glm_q6_K_value(device const block_q6_K *blocks, uint k) {
-    const uint block = k / QK_K;
-    const uint idx = k - block * QK_K;
-    device const block_q6_K *xb = blocks + block;
-    const uint n128 = idx >> 7u;
-    const uint r = idx & 127u;
-    const uint l = r & 31u;
-    const uint quarter = r >> 5u;
-    const uint ql_base = n128 * 64u;
-    const uint qh_base = n128 * 32u;
-    const uint sc_base = n128 * 8u;
-    uint q;
-    int sc;
-
-    if (quarter == 0u) {
-        q = (xb->ql[ql_base + l] & 0x0Fu) | (((xb->qh[qh_base + l] >> 0u) & 3u) << 4u);
-        sc = (int)xb->scales[sc_base + l / 16u + 0u];
-    } else if (quarter == 1u) {
-        q = (xb->ql[ql_base + 32u + l] & 0x0Fu) | (((xb->qh[qh_base + l] >> 2u) & 3u) << 4u);
-        sc = (int)xb->scales[sc_base + l / 16u + 2u];
-    } else if (quarter == 2u) {
-        q = (xb->ql[ql_base + l] >> 4u) | (((xb->qh[qh_base + l] >> 4u) & 3u) << 4u);
-        sc = (int)xb->scales[sc_base + l / 16u + 4u];
-    } else {
-        q = (xb->ql[ql_base + 32u + l] >> 4u) | (((xb->qh[qh_base + l] >> 6u) & 3u) << 4u);
-        sc = (int)xb->scales[sc_base + l / 16u + 6u];
-    }
-
-    return (float)xb->d * (float)sc * (float)((int)q - 32);
-}
-
-kernel void kernel_glm_q4_K_pair_swiglu_f32(
-        constant ds4_metal_glm_routed_moe_args &args,
-        device const char *gate,
-        device const char *up,
-        device const float *x,
-        device const int32_t *selected,
-        device const float *weights,
-        device float *mid,
-        threadgroup float *scratch [[threadgroup(0)]],
-        uint3 tgpig [[threadgroup_position_in_grid]],
-        uint tid [[thread_index_in_threadgroup]]) {
-    const uint ntg = 256u;
-    const uint row = tgpig.x;
-    const uint slot = tgpig.y;
-    const uint token = tgpig.z;
-    if (row >= args.mid_dim || slot >= args.n_expert_used || token >= args.n_tokens) return;
-
-    const uint64_t selected_off = (uint64_t)token * args.n_expert_used + slot;
-    const uint64_t mid_off = (uint64_t)token * args.mid_token_stride +
-                             (uint64_t)slot * args.mid_dim + row;
-    const int expert = selected[selected_off];
-    if (!ds4_tp_owns_expert(expert, args.n_total_expert,
-                            args.tp_rank, args.tp_world)) {
-        if (tid == 0u) mid[mid_off] = 0.0f;
-        return;
-    }
-    const int local_expert = expert - args.tp_expert_base;
-
-    device const block_q4_K *gate_row =
-        (device const block_q4_K *)(gate +
-            (uint64_t)(uint)local_expert * args.gate_expert_bytes +
-            (uint64_t)row * args.gate_row_bytes);
-    device const block_q4_K *up_row =
-        (device const block_q4_K *)(up +
-            (uint64_t)(uint)local_expert * args.up_expert_bytes +
-            (uint64_t)row * args.up_row_bytes);
-
-    float acc_gate = 0.0f;
-    float acc_up = 0.0f;
-    device const float *token_x = x + (uint64_t)token * args.in_dim;
-    for (uint k = tid; k < args.in_dim; k += ntg) {
-        const float xv = token_x[k];
-        acc_gate += ds4_glm_q4_K_value(gate_row, k) * xv;
-        acc_up += ds4_glm_q4_K_value(up_row, k) * xv;
-    }
-
-    scratch[tid] = acc_gate;
-    scratch[ntg + tid] = acc_up;
-    threadgroup_barrier(mem_flags::mem_threadgroup);
-    for (uint stride = ntg >> 1u; stride > 0u; stride >>= 1u) {
-        if (tid < stride) {
-            scratch[tid] += scratch[tid + stride];
-            scratch[ntg + tid] += scratch[ntg + tid + stride];
-        }
-        threadgroup_barrier(mem_flags::mem_threadgroup);
-    }
-
-    if (tid == 0u) {
-        mid[mid_off] = ds4_glm_swiglu(scratch[0], scratch[ntg],
-                                      args.swiglu_clamp) * weights[selected_off];
-    }
-}
-
 template <short N_R0>
 static inline void glm_q2_K_pair_swiglu_simd_f32_impl(
         ds4_metal_glm_routed_moe_args args,
@@ -1386,71 +1523,6 @@ kernel void kernel_glm_q4_K_pair_swiglu4_f32(
         expert - args.tp_expert_base, tiisg, sgitg);
 }
 
-kernel void kernel_glm_q4_K_pair_swiglu2_mapped_f32(
-        constant ds4_metal_glm_routed_moe_args &args,
-        device const char *gate,
-        device const char *up,
-        device const float *x,
-        device const uint32_t *htpe,
-        device const int32_t *hids,
-        device const float *weights,
-        device float *mid,
-        threadgroup float *scratch [[threadgroup(0)]],
-        uint3 tgpig [[threadgroup_position_in_grid]],
-        ushort tiisg [[thread_index_in_simdgroup]],
-        ushort sgitg [[simdgroup_index_in_threadgroup]]) {
-    const uint expert = tgpig.z;
-    if (expert >= args.n_total_expert) return;
-    if (!ds4_tp_owns_expert((int)expert, args.n_total_expert,
-                            args.tp_rank, args.tp_world)) return;
-    const uint count = htpe[expert];
-    const uint map_base = tgpig.y * 32u;
-    for (uint i = 0; i < 32u; i++) {
-        const uint map_row = map_base + i;
-        if (map_row >= count) break;
-        const int id = hids[(uint64_t)expert * args.n_tokens + map_row];
-        if (id < 0) continue;
-        const uint token = (uint)id / args.n_expert_used;
-        const uint slot = (uint)id - token * args.n_expert_used;
-        if (slot >= args.n_expert_used || token >= args.n_tokens) continue;
-        const uint64_t selected_off = (uint64_t)token * args.n_expert_used + slot;
-        glm_q4_K_pair_swiglu_simd_f32_impl<N_R0_Q4_K>(
-            args, gate, up, x, weights, mid, scratch,
-            tgpig, slot, token, selected_off,
-            (int)expert - args.tp_expert_base, tiisg, sgitg);
-    }
-}
-
-kernel void kernel_glm_q4_K_pair_swiglu2_mapped_row_f32(
-        constant ds4_metal_glm_routed_moe_args &args,
-        device const char *gate,
-        device const char *up,
-        device const float *x,
-        device const uint32_t *htpe,
-        device const int32_t *hids,
-        device const float *weights,
-        device float *mid,
-        threadgroup float *scratch [[threadgroup(0)]],
-        uint3 tgpig [[threadgroup_position_in_grid]],
-        ushort tiisg [[thread_index_in_simdgroup]],
-        ushort sgitg [[simdgroup_index_in_threadgroup]]) {
-    const uint expert = tgpig.z;
-    const uint map_row = tgpig.y;
-    if (expert >= args.n_total_expert || map_row >= htpe[expert]) return;
-    if (!ds4_tp_owns_expert((int)expert, args.n_total_expert,
-                            args.tp_rank, args.tp_world)) return;
-    const int id = hids[(uint64_t)expert * args.n_tokens + map_row];
-    if (id < 0) return;
-    const uint token = (uint)id / args.n_expert_used;
-    const uint slot = (uint)id - token * args.n_expert_used;
-    if (slot >= args.n_expert_used || token >= args.n_tokens) return;
-    const uint64_t selected_off = (uint64_t)token * args.n_expert_used + slot;
-    glm_q4_K_pair_swiglu_simd_f32_impl<N_R0_Q4_K>(
-        args, gate, up, x, weights, mid, scratch,
-        tgpig, slot, token, selected_off,
-        (int)expert - args.tp_expert_base, tiisg, sgitg);
-}
-
 static inline void glm_q5_K_pair_swiglu_f32_impl(
         constant ds4_metal_glm_routed_moe_args &args,
         device const char *gate,
@@ -1650,65 +1722,6 @@ kernel void kernel_glm_q5_K_pair_swiglu_f32(
         tgpig, slot, token, selected_off, expert, tiisg, sgitg);
 }
 
-kernel void kernel_glm_q5_K_pair_swiglu_mapped_f32(
-        constant ds4_metal_glm_routed_moe_args &args,
-        device const char *gate,
-        device const char *up,
-        device const float *x,
-        device const uint32_t *htpe,
-        device const int32_t *hids,
-        device const float *weights,
-        device float *mid,
-        threadgroup float *scratch [[threadgroup(0)]],
-        uint3 tgpig [[threadgroup_position_in_grid]],
-        ushort tiisg [[thread_index_in_simdgroup]],
-        ushort sgitg [[simdgroup_index_in_threadgroup]]) {
-    const uint expert = tgpig.z;
-    if (expert >= args.n_total_expert) return;
-    const uint count = htpe[expert];
-    const uint map_base = tgpig.y * 32u;
-    for (uint i = 0; i < 32u; i++) {
-        const uint map_row = map_base + i;
-        if (map_row >= count) break;
-        const int id = hids[(uint64_t)expert * args.n_tokens + map_row];
-        if (id < 0) continue;
-        const uint token = (uint)id / args.n_expert_used;
-        const uint slot = (uint)id - token * args.n_expert_used;
-        if (slot >= args.n_expert_used || token >= args.n_tokens) continue;
-        const uint64_t selected_off = (uint64_t)token * args.n_expert_used + slot;
-        glm_q5_K_pair_swiglu_f32_impl(
-            args, gate, up, x, weights, mid, scratch,
-            tgpig, slot, token, selected_off, (int)expert, tiisg, sgitg);
-    }
-}
-
-kernel void kernel_glm_q5_K_pair_swiglu_mapped_row_f32(
-        constant ds4_metal_glm_routed_moe_args &args,
-        device const char *gate,
-        device const char *up,
-        device const float *x,
-        device const uint32_t *htpe,
-        device const int32_t *hids,
-        device const float *weights,
-        device float *mid,
-        threadgroup float *scratch [[threadgroup(0)]],
-        uint3 tgpig [[threadgroup_position_in_grid]],
-        ushort tiisg [[thread_index_in_simdgroup]],
-        ushort sgitg [[simdgroup_index_in_threadgroup]]) {
-    const uint expert = tgpig.z;
-    const uint map_row = tgpig.y;
-    if (expert >= args.n_total_expert || map_row >= htpe[expert]) return;
-    const int id = hids[(uint64_t)expert * args.n_tokens + map_row];
-    if (id < 0) return;
-    const uint token = (uint)id / args.n_expert_used;
-    const uint slot = (uint)id - token * args.n_expert_used;
-    if (slot >= args.n_expert_used || token >= args.n_tokens) return;
-    const uint64_t selected_off = (uint64_t)token * args.n_expert_used + slot;
-    glm_q5_K_pair_swiglu_f32_impl(
-        args, gate, up, x, weights, mid, scratch,
-        tgpig, slot, token, selected_off, (int)expert, tiisg, sgitg);
-}
-
 kernel void kernel_glm_q5_K_down_f32(
         constant ds4_metal_glm_routed_moe_args &args,
         device const char *down,
@@ -2575,6 +2588,9 @@ void dequantize_q6_K(device const block_q6_K *xb, short il, thread type4x4 &reg)
     }
 }
 
+/* 0.25 * grid byte is exact in binary16, so the table replaces the u64
+ * grid load, byte extract, and float conversion. Sign is a bit flip;
+ * the remaining rounding is the same f32->f16 store as before. */
 template <typename type4x4>
 void dequantize_iq2_xxs(device const block_iq2_xxs * xb, short il, thread type4x4 & reg) {
     const float d = xb->d;
@@ -2584,16 +2600,18 @@ void dequantize_iq2_xxs(device const block_iq2_xxs * xb, short il, thread type4x
     const uint32_t aux32_g = q2[0] | (q2[1] << 16);
     const uint32_t aux32_s = q2[2] | (q2[3] << 16);
     thread const uint8_t * aux8 = (thread const uint8_t *)&aux32_g;
-    const float dl = d * (0.5f + (aux32_s >> 28)) * 0.25f;
-    constant uint8_t * grid = (constant uint8_t *)(iq2xxs_grid + aux8[2*il+0]);
+    const float dl = d * (0.5f + (aux32_s >> 28));
+    constant const ushort * values = ds4_metal_iq2xxs_half_values[aux8[2*il+0]];
     uint8_t signs = ksigns_iq2xs[(aux32_s >> 14*il) & 127];
-    for (int i = 0; i < 8; ++i) {
-        reg[i/4][i%4] = dl * grid[i] * (signs & kmask_iq2xs[i] ? -1.f : 1.f);
+    FOR_UNROLL (int i = 0; i < 8; ++i) {
+        const ushort bits = values[i] ^ (signs & kmask_iq2xs[i] ? 0x8000u : 0x0000u);
+        reg[i/4][i%4] = dl * (float)as_type<half>(bits);
     }
-    grid = (constant uint8_t *)(iq2xxs_grid + aux8[2*il+1]);
+    values = ds4_metal_iq2xxs_half_values[aux8[2*il+1]];
     signs = ksigns_iq2xs[(aux32_s >> (14*il+7)) & 127];
-    for (int i = 0; i < 8; ++i) {
-        reg[2+i/4][i%4] = dl * grid[i] * (signs & kmask_iq2xs[i] ? -1.f : 1.f);
+    FOR_UNROLL (int i = 0; i < 8; ++i) {
+        const ushort bits = values[i] ^ (signs & kmask_iq2xs[i] ? 0x8000u : 0x0000u);
+        reg[2+i/4][i%4] = dl * (float)as_type<half>(bits);
     }
 }
 
@@ -2649,10 +2667,6 @@ struct ds4_metal_q4_expert_table {
     array<device const char *, 384> experts [[id(0)]];
 };
 
-struct ds4_metal_expert_address_table {
-    device const uint64_t *addrs;
-};
-
 struct ds4_metal_stream_expert_validate_args {
     uint32_t n_total_expert;
     uint32_t n_expert;
@@ -2699,6 +2713,23 @@ struct ds4_metal_args_mul_mm_id {
     int32_t  tp_expert_base;
 };
 
+// Exact-N Q4_K attention-output projection used only by the opt-in SSD
+// prefill path.  One threadgroup stages a pair of packed weight rows once
+// and sixteen independent SIMDgroups consume that pair for sixteen tokens.
+// Keep this layout in lock-step with ds4_gpu_q4_attn_exactn_args.
+struct ds4_metal_args_q4_attn_exactn {
+    uint32_t in_dim;
+    uint32_t out_rows;
+    uint32_t n_groups;
+    uint32_t n_tokens;
+    uint64_t weight_row_bytes;
+    uint64_t weight_group_bytes;
+    uint64_t input_group_bytes;
+    uint64_t input_token_bytes;
+    uint64_t output_group_bytes;
+    uint64_t output_token_bytes;
+};
+
 template<int nr0, typename args_t>
 void kernel_mul_mv_q2_K_f32_impl(
         args_t args,
@@ -2900,6 +2931,208 @@ void kernel_mul_mv_q4_K_f32_impl(
     (void)shmem;
 }
 
+// This is the classic Q4_K matvec inner loop with only the weight address
+// space changed from device to threadgroup.  Its lane-to-K mapping, scalar
+// operation order and simd_sum reduction are deliberately kept identical to
+// kernel_mul_mv_q4_K_f32_impl: the exact-N oracle requires bitwise equality.
+template<int nr0, bool use_preexpanded_scales = false>
+void kernel_mul_mv_q4_K_staged_exactn_impl(
+        uint32_t in_dim,
+        uint64_t weight_row_bytes,
+        threadgroup const char *src0,
+        threadgroup const ushort4 *preexpanded_scales,
+        device const char *src1,
+        device       char *dst,
+        uint32_t valid_rows,
+        ushort tiisg) {
+    constexpr uint16_t kmask1 = 0x3f3f;
+    constexpr uint16_t kmask2 = 0x0f0f;
+    constexpr uint16_t kmask3 = 0xc0c0;
+
+    const short ix = tiisg / 8;
+    const short it = tiisg % 8;
+    const short iq = it / 4;
+    const short ir = it % 4;
+    const int nb = in_dim / QK_K;
+
+    threadgroup const block_q4_K *x =
+        (threadgroup const block_q4_K *)src0;
+    device const float *y = (device const float *)src1;
+
+    float yl[16];
+    float yh[16];
+    float sumf[nr0] = {0.f};
+
+    device const float *y4 = y + ix * QK_K + 64 * iq + 8 * ir;
+
+    uint16_t sc16[4];
+
+    for (int ib = ix; ib < nb; ib += 4) {
+        float4 sumy = {0.f, 0.f, 0.f, 0.f};
+
+        FOR_UNROLL (short i = 0; i < 8; ++i) {
+            yl[i + 0] = y4[i +   0]; sumy[0] += yl[i + 0];
+            yl[i + 8] = y4[i +  32]; sumy[1] += yl[i + 8];
+            yh[i + 0] = y4[i + 128]; sumy[2] += yh[i + 0];
+            yh[i + 8] = y4[i + 160]; sumy[3] += yh[i + 8];
+        }
+
+        threadgroup const uint16_t *sc =
+            (threadgroup const uint16_t *)x[ib].scales + iq;
+        threadgroup const uint16_t *q1 =
+            (threadgroup const uint16_t *)x[ib].qs + 16 * iq + 4 * ir;
+        threadgroup const half *dh = &x[ib].d;
+
+        for (short row = 0; row < nr0 && row < valid_rows; row++) {
+            if (use_preexpanded_scales) {
+                const uint32_t meta_index =
+                    ((uint32_t)row * (uint32_t)nb + (uint32_t)ib) * 2u +
+                    (uint32_t)iq;
+                const ushort4 expanded = preexpanded_scales[meta_index];
+                sc16[0] = expanded[0];
+                sc16[1] = expanded[1];
+                sc16[2] = expanded[2];
+                sc16[3] = expanded[3];
+            } else {
+                sc16[0] = sc[0] & kmask1;
+                sc16[1] = sc[2] & kmask1;
+                sc16[2] =
+                    ((sc[4] >> 0) & kmask2) | ((sc[0] & kmask3) >> 2);
+                sc16[3] =
+                    ((sc[4] >> 4) & kmask2) | ((sc[2] & kmask3) >> 2);
+            }
+            thread const uint8_t *sc8 = (thread const uint8_t *)sc16;
+
+            threadgroup const uint16_t *q2 = q1 + 32;
+
+            float4 acc1 = {0.f, 0.f, 0.f, 0.f};
+            float4 acc2 = {0.f, 0.f, 0.f, 0.f};
+
+            FOR_UNROLL (short i = 0; i < 4; ++i) {
+                acc1[0] += yl[2 * i + 0] * (q1[i] & 0x000F);
+                acc1[1] += yl[2 * i + 1] * (q1[i] & 0x0F00);
+                acc1[2] += yl[2 * i + 8] * (q1[i] & 0x00F0);
+                acc1[3] += yl[2 * i + 9] * (q1[i] & 0xF000);
+                acc2[0] += yh[2 * i + 0] * (q2[i] & 0x000F);
+                acc2[1] += yh[2 * i + 1] * (q2[i] & 0x0F00);
+                acc2[2] += yh[2 * i + 8] * (q2[i] & 0x00F0);
+                acc2[3] += yh[2 * i + 9] * (q2[i] & 0xF000);
+            }
+
+            sumf[row] += dh[0] * ((acc1[0] + 1.f / 256.f * acc1[1]) * sc8[0] +
+                                  (acc1[2] + 1.f / 256.f * acc1[3]) * sc8[1] * 1.f / 16.f +
+                                  (acc2[0] + 1.f / 256.f * acc2[1]) * sc8[4] +
+                                  (acc2[2] + 1.f / 256.f * acc2[3]) * sc8[5] * 1.f / 16.f) -
+                         dh[1] * (sumy[0] * sc8[2] + sumy[1] * sc8[3] + sumy[2] * sc8[6] + sumy[3] * sc8[7]);
+
+            q1 += weight_row_bytes / 2;
+            sc += weight_row_bytes / 2;
+            dh += weight_row_bytes / 2;
+        }
+
+        y4 += 4 * QK_K;
+    }
+
+    device float *dst_f32 = (device float *)dst;
+    for (int row = 0; row < nr0 && row < valid_rows; ++row) {
+        float sum_all = simd_sum(sumf[row]);
+        if (tiisg == 0) {
+            dst_f32[row] = sum_all;
+        }
+    }
+}
+
+// Grid: x = output-row pairs, y = 16-token tiles, z = independent groups.
+// All 512 threads join the raw packed-row load and barrier.  The optimized
+// specialization also expands the integer scale/min metadata before that same
+// barrier so all token SIMDgroups can reuse it without changing FP arithmetic.
+template<bool preexpand_scales>
+kernel void kernel_dsv4_attn_out_q4_K_ssd_prefill_exactn_impl(
+        constant ds4_metal_args_q4_attn_exactn &args,
+        device const char *weights,
+        device const char *input,
+        device       char *output,
+        threadgroup  char *staged [[threadgroup(0)]],
+        uint3  tgpig [[threadgroup_position_in_grid]],
+        ushort tiitg [[thread_index_in_threadgroup]],
+        ushort tiisg [[thread_index_in_simdgroup]],
+        ushort sgitg [[simdgroup_index_in_threadgroup]]) {
+    const uint32_t first_row = tgpig.x * 2u;
+    const uint32_t valid_rows = min(2u, args.out_rows - first_row);
+    const uint64_t group_base = (uint64_t)tgpig.z * args.weight_group_bytes;
+
+    const uint64_t row0_base =
+        group_base + (uint64_t)first_row * args.weight_row_bytes;
+    for (uint64_t i = tiitg; i < args.weight_row_bytes; i += 32u * 16u) {
+        staged[i] = weights[row0_base + i];
+    }
+    const uint64_t row1_base = row0_base + args.weight_row_bytes;
+    for (uint64_t i = tiitg; i < args.weight_row_bytes; i += 32u * 16u) {
+        staged[args.weight_row_bytes + i] =
+            valid_rows == 2u ? weights[row1_base + i] : 0;
+    }
+
+    const uint32_t blocks_per_row = args.in_dim / QK_K;
+    threadgroup ushort4 *scale_meta =
+        (threadgroup ushort4 *)(staged + 2u * args.weight_row_bytes);
+    if (preexpand_scales) {
+        const uint32_t meta_count = valid_rows * blocks_per_row * 2u;
+        for (uint32_t meta_index = tiitg;
+             meta_index < meta_count;
+             meta_index += 32u * 16u) {
+            const uint32_t iq = meta_index & 1u;
+            const uint32_t block_index = meta_index >> 1u;
+            const uint32_t row = block_index / blocks_per_row;
+            const uint32_t ib = block_index - row * blocks_per_row;
+            device const block_q4_K *xb =
+                (device const block_q4_K *)(weights + row0_base +
+                    (uint64_t)row * args.weight_row_bytes) + ib;
+            device const uint16_t *sc =
+                (device const uint16_t *)xb->scales + iq;
+            const ushort4 expanded = ushort4(
+                sc[0] & 0x3f3fu,
+                sc[2] & 0x3f3fu,
+                ((sc[4] >> 0) & 0x0f0fu) |
+                    ((sc[0] & 0xc0c0u) >> 2),
+                ((sc[4] >> 4) & 0x0f0fu) |
+                    ((sc[2] & 0xc0c0u) >> 2));
+            scale_meta[meta_index] = expanded;
+        }
+    }
+    threadgroup_barrier(mem_flags::mem_threadgroup);
+
+    const uint32_t token = tgpig.y * 16u + sgitg;
+    if (token >= args.n_tokens) return;
+
+    device const char *token_input =
+        input + (uint64_t)token * args.input_token_bytes +
+        (uint64_t)tgpig.z * args.input_group_bytes;
+    device char *token_output =
+        output + (uint64_t)token * args.output_token_bytes +
+        (uint64_t)tgpig.z * args.output_group_bytes +
+        (uint64_t)first_row * sizeof(float);
+
+    kernel_mul_mv_q4_K_staged_exactn_impl<2, preexpand_scales>(
+        args.in_dim,
+        args.weight_row_bytes,
+        staged,
+        scale_meta,
+        token_input,
+        token_output,
+        valid_rows,
+        tiisg);
+}
+
+typedef decltype(
+    kernel_dsv4_attn_out_q4_K_ssd_prefill_exactn_impl<false>)
+    q4_attn_ssd_prefill_exactn_t;
+template [[host_name("kernel_dsv4_attn_out_q4_K_ssd_prefill_exactn_f32")]]
+kernel q4_attn_ssd_prefill_exactn_t
+    kernel_dsv4_attn_out_q4_K_ssd_prefill_exactn_impl<false>;
+template [[host_name("kernel_dsv4_attn_out_q4_K_ssd_prefill_exactn_scale_meta_f32")]]
+kernel q4_attn_ssd_prefill_exactn_t
+    kernel_dsv4_attn_out_q4_K_ssd_prefill_exactn_impl<true>;
+
 template<int nr0, typename args_t>
 void kernel_mul_mv_mxfp4_f32_impl(
         args_t args,
@@ -3075,12 +3308,14 @@ void kernel_mul_mv_iq2_xxs_f32_impl(
     threadgroup uint64_t * svalues = (threadgroup uint64_t *)(shmem);
     threadgroup uint8_t  * ssigns  = (threadgroup uint8_t  *)(svalues + 256);
     {
-        int nval = 4;
-        int pos  = (32*sgitg + tiisg)*nval;
-        for (int i = 0; i < nval; ++i) svalues[pos + i] = ds4_metal_iq2xxs_grid[pos + i];
-        nval = 2;
-        pos  = (32*sgitg + tiisg)*nval;
-        for (int i = 0; i < nval; ++i) ssigns[pos+i] = ds4_metal_ksigns_iq2xs[pos+i];
+        const uint tid = 32u*(uint)sgitg + (uint)tiisg;
+        const uint nth = 32u*(uint)NSG;
+        for (uint i = tid; i < 256u; i += nth) {
+            svalues[i] = ds4_metal_iq2xxs_grid[i];
+        }
+        for (uint i = tid; i < 128u; i += nth) {
+            ssigns[i] = ds4_metal_ksigns_iq2xs[i];
+        }
         threadgroup_barrier(mem_flags::mem_threadgroup);
     }
 
@@ -3247,47 +3482,182 @@ void kernel_mul_mv_iq2_xxs_pair_f32_impl(
     }
 }
 
-typedef void (kernel_mul_mv2_disp_t)(
-        ds4_metal_args_mul_mv args,
-        device const char * src0,
-        device const char * src1,
-        device       char * dst,
-        threadgroup  char * shmem,
-        uint3  tgpig,
-        ushort tiisg,
-        ushort sgitg);
-
-template<kernel_mul_mv2_disp_t disp_fn>
-void mmv_fn(
+// Address-table decode specialization for the DeepSeek Flash IQ2 gate/up
+// shape.  It deliberately keeps the exact dot-product, simd_sum, scale,
+// clamp, exp, and route-weight order of kernel_mul_mv_iq2_xxs_pair_f32_impl
+// followed by the canonical fused SwiGLU epilogue.  The only removed work is
+// the round trip through the gate/up destination buffers: downstream consumes
+// `mid` exclusively on this dispatch.
+template<int nr0>
+void kernel_mul_mv_iq2_xxs_pair_swiglu_mid_only_4096x2048_impl(
         ds4_metal_args_mul_mv args,
-        device const char * src0,
+        constant ds4_metal_dsv4_moe_swiglu_weight_args & act,
+        device const char * src0_gate,
+        device const char * src0_up,
         device const char * src1,
-        device       char * dst,
+        device       char * dst_mid,
+        device const char * weights,
+        uint64_t pair_row,
         threadgroup  char * shmem,
         uint3  tgpig,
-        ushort tiitg,
         ushort tiisg,
         ushort sgitg) {
-    disp_fn(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);
-}
+    constexpr int ne00 = 4096;
+    constexpr int ne0 = 2048;
+    constexpr int nb = ne00 / QK_K;
+    const short NSG = FC_mul_mv_nsg;
 
-typedef decltype(mmv_fn<kernel_mul_mv_q2_K_f32_impl<N_R0_Q2_K>>) mul_mv_id_disp_fn_t;
+    const int r0 = tgpig.x;
+    const int r1 = tgpig.y;
+    const int im = tgpig.z;
+    const int first_row = (r0 * NSG + sgitg) * nr0;
 
-// Decode-time expert matvec. The ids tensor selects the routed expert for each
-// slot, then this wrapper invokes the quantized row kernel for Q8_0, Q2_K, or
-// IQ2_XXS weights without materializing per-expert dispatches on the CPU.
-template<mul_mv_id_disp_fn_t disp_fn>
-kernel void kernel_mul_mv_id(
-        constant ds4_metal_args_mul_mv_id & args,
-        device const char * src0s,
-        device const char * src1,
-        device       char * dst,
-        device const char * ids,
-        threadgroup  char * shmem [[threadgroup(0)]],
-        uint3  tgpig[[threadgroup_position_in_grid]],
-        ushort tiitg[[thread_index_in_threadgroup]],
-        ushort tiisg[[thread_index_in_simdgroup]],
-        ushort sgitg[[simdgroup_index_in_threadgroup]]) {
+    const uint i12 = im % args.ne12;
+    const uint i13 = im / args.ne12;
+    const uint64_t offset0 = first_row * args.nb01 +
+        (i12 / args.r2) * args.nb02 + (i13 / args.r3) * args.nb03;
+    const uint64_t offset1 = r1 * args.nb11 + i12 * args.nb12 + i13 * args.nb13;
+
+    device const block_iq2_xxs *xg =
+        (device const block_iq2_xxs *)(src0_gate + offset0);
+    device const block_iq2_xxs *xu =
+        (device const block_iq2_xxs *)(src0_up + offset0);
+    device const float *y = (device const float *)(src1 + offset1);
+
+    float yl[32];
+    float sumg[nr0] = {0.f};
+    float sumu[nr0] = {0.f};
+    constexpr int nb32 = nb * (QK_K / 32);
+
+    threadgroup uint64_t *svalues = (threadgroup uint64_t *)(shmem);
+    threadgroup uint8_t *ssigns = (threadgroup uint8_t *)(svalues + 256);
+    {
+        const uint tid = 32u * (uint)sgitg + (uint)tiisg;
+        const uint nth = 32u * (uint)NSG;
+        for (uint i = tid; i < 256u; i += nth) {
+            svalues[i] = ds4_metal_iq2xxs_grid[i];
+        }
+        for (uint i = tid; i < 128u; i += nth) {
+            ssigns[i] = ds4_metal_ksigns_iq2xs[i];
+        }
+        threadgroup_barrier(mem_flags::mem_threadgroup);
+    }
+
+    const int ix = tiisg;
+    device const float *y4 = y + 32 * ix;
+    for (int ib32 = ix; ib32 < nb32; ib32 += 32) {
+        FOR_UNROLL (short i = 0; i < 32; ++i) {
+            yl[i] = y4[i];
+        }
+
+        const int ibl = ib32 / (QK_K / 32);
+        const int ib = ib32 % (QK_K / 32);
+        device const block_iq2_xxs *xgr = xg + ibl;
+        device const block_iq2_xxs *xur = xu + ibl;
+        device const uint16_t *qg = xgr->qs + 4 * ib;
+        device const uint16_t *qu = xur->qs + 4 * ib;
+        device const half *dhg = &xgr->d;
+        device const half *dhu = &xur->d;
+
+        for (short row = 0; row < nr0; ++row) {
+            device const uint8_t *aux8g = (device const uint8_t *)qg;
+            device const uint8_t *aux8u = (device const uint8_t *)qu;
+            const uint32_t aux32g = qg[2] | (qg[3] << 16);
+            const uint32_t aux32u = qu[2] | (qu[3] << 16);
+            const float dg = (float)dhg[0] * (0.5f + (aux32g >> 28));
+            const float du = (float)dhu[0] * (0.5f + (aux32u >> 28));
+
+            float sg = 0;
+            float su = 0;
+            FOR_UNROLL (short l = 0; l < 4; ++l) {
+                const threadgroup uint8_t *gridg =
+                    (const threadgroup uint8_t *)(svalues + aux8g[l]);
+                const threadgroup uint8_t *gridu =
+                    (const threadgroup uint8_t *)(svalues + aux8u[l]);
+                const uint8_t signg = ssigns[(aux32g >> 7 * l) & 127];
+                const uint8_t signu = ssigns[(aux32u >> 7 * l) & 127];
+                FOR_UNROLL (short j = 0; j < 8; ++j) {
+                    const float v = yl[8 * l + j];
+                    sg += v * gridg[j] *
+                        (signg & ds4_metal_kmask_iq2xs[j] ? -1.f : 1.f);
+                    su += v * gridu[j] *
+                        (signu & ds4_metal_kmask_iq2xs[j] ? -1.f : 1.f);
+                }
+            }
+            sumg[row] += dg * sg;
+            sumu[row] += du * su;
+
+            dhg += args.nb01 / 2;
+            dhu += args.nb01 / 2;
+            qg += args.nb01 / 2;
+            qu += args.nb01 / 2;
+        }
+        y4 += 32 * 32;
+    }
+
+    device float *mid_f32 =
+        (device float *)(dst_mid + pair_row * act.mid_row_stride);
+    device const float *route_w =
+        (device const float *)(weights + pair_row * act.weight_stride);
+    const float c = act.clamp_value;
+    const float route_weight = route_w[0];
+    for (int row = 0; row < nr0 && first_row + row < ne0; ++row) {
+        const float sum_gate = simd_sum(sumg[row]);
+        const float sum_up = simd_sum(sumu[row]);
+        if (tiisg == 0) {
+            float g = sum_gate * 0.25f;
+            float u = sum_up * 0.25f;
+            if (c > 1.0e-6f) {
+                g = min(g, c);
+                u = clamp(u, -c, c);
+            }
+            const float silu = ds4_silu(g);
+            mid_f32[first_row + row] = silu * u * route_weight;
+        }
+    }
+}
+
+typedef void (kernel_mul_mv2_disp_t)(
+        ds4_metal_args_mul_mv args,
+        device const char * src0,
+        device const char * src1,
+        device       char * dst,
+        threadgroup  char * shmem,
+        uint3  tgpig,
+        ushort tiisg,
+        ushort sgitg);
+
+template<kernel_mul_mv2_disp_t disp_fn>
+void mmv_fn(
+        ds4_metal_args_mul_mv args,
+        device const char * src0,
+        device const char * src1,
+        device       char * dst,
+        threadgroup  char * shmem,
+        uint3  tgpig,
+        ushort tiitg,
+        ushort tiisg,
+        ushort sgitg) {
+    disp_fn(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);
+}
+
+typedef decltype(mmv_fn<kernel_mul_mv_q2_K_f32_impl<N_R0_Q2_K>>) mul_mv_id_disp_fn_t;
+
+// Decode-time expert matvec. The ids tensor selects the routed expert for each
+// slot, then this wrapper invokes the quantized row kernel for Q8_0, Q2_K, or
+// IQ2_XXS weights without materializing per-expert dispatches on the CPU.
+template<mul_mv_id_disp_fn_t disp_fn>
+kernel void kernel_mul_mv_id(
+        constant ds4_metal_args_mul_mv_id & args,
+        device const char * src0s,
+        device const char * src1,
+        device       char * dst,
+        device const char * ids,
+        threadgroup  char * shmem [[threadgroup(0)]],
+        uint3  tgpig[[threadgroup_position_in_grid]],
+        ushort tiitg[[thread_index_in_threadgroup]],
+        ushort tiisg[[thread_index_in_simdgroup]],
+        ushort sgitg[[simdgroup_index_in_threadgroup]]) {
     (void)tiitg;
 
     const int iid1 = tgpig.z/args.nei0;
@@ -3381,6 +3751,244 @@ kernel void kernel_mul_mv_q4_K_dense_f32(
     kernel_mul_mv_q4_K_f32_impl<N_R0_Q4_K>(args, src0, src1, dst, shmem, tgpig, tiisg, sgitg);
 }
 
+// Dense Q-A/KV pair. Both projections consume the same activation rows and
+// retain the standalone Q4_K reduction order. Combining them removes one
+// Metal dispatch/encoder transition while keeping independently-sized output
+// matrices and byte-identical arithmetic.
+kernel void kernel_mul_mv_q4_K_dense_pair_f32(
+        constant ds4_metal_args_mul_mv & args0,
+        constant ds4_metal_args_mul_mv & args1,
+        device const char * src0_a,
+        device const char * src0_b,
+        device const char * src1,
+        device       char * dst_a,
+        device       char * dst_b,
+        threadgroup  char * shmem [[threadgroup(0)]],
+        uint3  tgpig[[threadgroup_position_in_grid]],
+        ushort tiisg[[thread_index_in_simdgroup]],
+        ushort sgitg[[simdgroup_index_in_threadgroup]]) {
+    const int first_row =
+        (tgpig.x * FC_mul_mv_nsg + sgitg) * N_R0_Q4_K;
+    if (first_row < args0.ne0) {
+        kernel_mul_mv_q4_K_f32_impl<N_R0_Q4_K>(
+            args0, src0_a, src1, dst_a, shmem, tgpig, tiisg, sgitg);
+    }
+    if (first_row < args1.ne0) {
+        kernel_mul_mv_q4_K_f32_impl<N_R0_Q4_K>(
+            args1, src0_b, src1, dst_b, shmem, tgpig, tiisg, sgitg);
+    }
+}
+
+// Decode Q4_K Q-A/KV pair plus the attention/indexer F16 compressor pairs.
+// The Q4_K range calls the same per-row implementation as the standalone
+// dense-pair kernel.  With NSG=8 each simdgroup still owns an independent
+// two-row cohort, so changing the number of cohorts in a threadgroup does not
+// change the K walk or the simd reduction for any output row.  The shifted
+// compressor ranges are the standalone F16 pair/store body verbatim.  The
+// fused dispatch therefore removes encoder/dispatch transitions while
+// preserving the bits produced by all three original kernels.
+kernel void kernel_dsv4_q4_K_qkv_pair_quad_compressor_store(
+        constant ds4_metal_args_mul_mv & args0,
+        constant ds4_metal_args_mul_mv & args1,
+        constant ds4_metal_args_mul_mv & cargs,
+        constant ds4_metal_args_compressor_pair_store & store0,
+        constant ds4_metal_args_compressor_pair_store & store1,
+        constant uint & pair_tgs,
+        device const char * qw0,
+        device const char * qw1,
+        device const char * cw0a,
+        device const char * cw0b,
+        device const char * cw1a,
+        device const char * cw1b,
+        device const char * src1,
+        device       char * dst0,
+        device       char * dst1,
+        device       char * cdst_a0,
+        device       char * cdst_b0,
+        device       char * cdst_a1,
+        device       char * cdst_b1,
+        device const char * ape0,
+        device const char * ape1,
+        device       float * state0_kv,
+        device       float * state0_score,
+        device       float * state1_kv,
+        device       float * state1_score,
+        threadgroup  char * shmem [[threadgroup(0)]],
+        uint3  tgpig  [[threadgroup_position_in_grid]],
+        ushort tiitg  [[thread_index_in_threadgroup]],
+        ushort tiisg  [[thread_index_in_simdgroup]],
+        ushort sgitg  [[simdgroup_index_in_threadgroup]]) {
+    if (tgpig.x < pair_tgs) {
+        const int first_row =
+            (tgpig.x * FC_mul_mv_nsg + sgitg) * N_R0_Q4_K;
+        if (first_row < args0.ne0) {
+            kernel_mul_mv_q4_K_f32_impl<N_R0_Q4_K>(
+                args0, qw0, src1, dst0, shmem, tgpig, tiisg, sgitg);
+        }
+        if (first_row < args1.ne0) {
+            kernel_mul_mv_q4_K_f32_impl<N_R0_Q4_K>(
+                args1, qw1, src1, dst1, shmem, tgpig, tiisg, sgitg);
+        }
+        return;
+    }
+
+    constexpr short NR0 = 2;
+    const uint lx = tgpig.x - pair_tgs;
+    const uint tgs0 = ((uint)store0.width + NR0 - 1u) / NR0;
+    const uint tgs1 = ((uint)store1.width + NR0 - 1u) / NR0;
+    if (lx >= tgs0 + tgs1) return;
+    const bool second = lx >= tgs0;
+
+    uint3 local_tgpig = tgpig;
+    local_tgpig.x = second ? lx - tgs0 : lx;
+
+    ds4_metal_args_mul_mv largs = cargs;
+    largs.nr0 = NR0;
+    largs.ne01 = second ? (int32_t)store1.width : (int32_t)store0.width;
+
+    if (!second) {
+        kernel_mul_mv_f16_f32_pair_4_impl<NR0>(
+            largs, cw0a, cw0b, src1, cdst_a0, cdst_b0,
+            shmem, local_tgpig, tiisg, sgitg);
+    } else {
+        kernel_mul_mv_f16_f32_pair_4_impl<NR0>(
+            largs, cw1a, cw1b, src1, cdst_a1, cdst_b1,
+            shmem, local_tgpig, tiisg, sgitg);
+    }
+
+    threadgroup_barrier(mem_flags::mem_device);
+
+    constant ds4_metal_args_compressor_pair_store & store =
+        second ? store1 : store0;
+    if (tiitg >= NR0 || store.width == 0u || store.ratio == 0u) {
+        return;
+    }
+    const uint col = local_tgpig.x * (uint)NR0 + tiitg;
+    if (col >= store.width) return;
+
+    const uint pos_mod = store.pos % store.ratio;
+    const uint dst_row = store.ratio == 4u ? store.ratio + pos_mod : pos_mod;
+    const uint dst = dst_row * store.width + col;
+    const uint ape_i = pos_mod * store.width + col;
+
+    device volatile const float * projected_kv = second
+        ? (device volatile const float *)cdst_a1
+        : (device volatile const float *)cdst_a0;
+    device volatile const float * projected_score = second
+        ? (device volatile const float *)cdst_b1
+        : (device volatile const float *)cdst_b0;
+    device const char * ape = second ? ape1 : ape0;
+    device float * state_kv = second ? state1_kv : state0_kv;
+    device float * state_score = second ? state1_score : state0_score;
+
+    float ape_v;
+    if (store.ape_type == 1u) {
+        ape_v = (float)(((device const half *)ape)[ape_i]);
+    } else {
+        ape_v = ((device const float *)ape)[ape_i];
+    }
+
+    state_kv[dst] = projected_kv[col];
+    state_score[dst] = projected_score[col] + ape_v;
+}
+
+// ABI-compatible subset of ds4_metal_args_dsv4_hc_expand.  The Q4_K kernel
+// lives in this source file beside the classic Q4 implementation, while the
+// generic HC kernels are concatenated later from dsv4_hc.metal.
+struct ds4_metal_args_q4_hc_expand {
+    int64_t  n_embd;
+    int64_t  n_hc;
+    int64_t  n_tokens;
+    uint64_t nb_block0;
+    uint64_t nb_block1;
+    uint64_t nb_add0;
+    uint64_t nb_add1;
+    uint64_t nb_res0;
+    uint64_t nb_res1;
+    uint64_t nb_res2;
+    uint64_t nb_post0;
+    uint64_t nb_post1;
+    uint64_t nb_comb0;
+    uint64_t nb_comb1;
+    uint64_t nb_comb2;
+    uint64_t nb0;
+    uint64_t nb1;
+    uint64_t nb2;
+    int32_t  has_add;
+};
+
+// Decode attention-output tail for AProjQ4:
+//
+//     block_out = input @ Wob(Q4_K)
+//     out_hc    = HCPost(block_out, residual_hc, split)
+//
+// The matvec is the exact standalone classic Q4_K implementation.  Once its
+// stored F32 row is visible to the threadgroup, the owning simdgroup expands
+// the same value into the four HC streams.  Materializing block_out preserves
+// diagnostics and makes an A/B memcmp possible.
+kernel void kernel_dsv4_q4_K_hc_expand4(
+        constant ds4_metal_args_mul_mv & mv,
+        constant ds4_metal_args_q4_hc_expand & hc,
+        device const char * weight,
+        device const char * input,
+        device       char * block_out,
+        device const char * residual,
+        device const char * post,
+        device const char * comb,
+        device       char * dst,
+        threadgroup  char * shmem [[threadgroup(0)]],
+        uint3  tgpig [[threadgroup_position_in_grid]],
+        ushort tiisg [[thread_index_in_simdgroup]],
+        ushort sgitg [[simdgroup_index_in_threadgroup]]) {
+    if (hc.n_hc != 4 || hc.n_tokens != 1 || hc.has_add != 0 ||
+        mv.ne0 != hc.n_embd || (mv.ne0 & 1) != 0 ||
+        hc.nb_block0 != sizeof(float)) {
+        return;
+    }
+
+    const int first_row =
+        (tgpig.x * FC_mul_mv_nsg + sgitg) * N_R0_Q4_K;
+    if (first_row < mv.ne0) {
+        kernel_mul_mv_q4_K_f32_impl<N_R0_Q4_K>(
+            mv, weight, input, block_out, shmem,
+            tgpig, tiisg, sgitg);
+    }
+
+    threadgroup_barrier(mem_flags::mem_device);
+
+    if (tiisg != 0) return;
+    FOR_UNROLL(short row = 0; row < N_R0_Q4_K; ++row) {
+        const int d = first_row + row;
+        if (d >= mv.ne0) continue;
+
+        const float block_v = *((device const float *)(
+            block_out + (uint64_t)d * hc.nb_block0));
+        const float r0 = *((device const float *)(
+            residual + (uint64_t)d * hc.nb_res0 + 0 * hc.nb_res1));
+        const float r1 = *((device const float *)(
+            residual + (uint64_t)d * hc.nb_res0 + 1 * hc.nb_res1));
+        const float r2 = *((device const float *)(
+            residual + (uint64_t)d * hc.nb_res0 + 2 * hc.nb_res1));
+        const float r3 = *((device const float *)(
+            residual + (uint64_t)d * hc.nb_res0 + 3 * hc.nb_res1));
+
+        FOR_UNROLL(short dst_hc = 0; dst_hc < 4; ++dst_hc) {
+            float acc = block_v * *((device const float *)(
+                post + (uint64_t)dst_hc * hc.nb_post0));
+            acc += *((device const float *)(
+                comb + (uint64_t)dst_hc * hc.nb_comb0 + 0 * hc.nb_comb1)) * r0;
+            acc += *((device const float *)(
+                comb + (uint64_t)dst_hc * hc.nb_comb0 + 1 * hc.nb_comb1)) * r1;
+            acc += *((device const float *)(
+                comb + (uint64_t)dst_hc * hc.nb_comb0 + 2 * hc.nb_comb1)) * r2;
+            acc += *((device const float *)(
+                comb + (uint64_t)dst_hc * hc.nb_comb0 + 3 * hc.nb_comb1)) * r3;
+            *((device float *)(dst + (uint64_t)d * hc.nb0 +
+                                (uint64_t)dst_hc * hc.nb1)) = acc;
+        }
+    }
+}
+
 // DS4 attention output low projection, specialized for the fixed block
 // diagonal mapping used by the model:
 //
@@ -3444,6 +4052,83 @@ kernel void kernel_dsv4_attn_out_low_q8_0_f32(
         sgitg);
 }
 
+#define DS4_ATTN_OUT_LOW_Q8_STATIC_K 4096
+#define DS4_ATTN_OUT_LOW_Q8_STATIC_ROWS 1024
+#define DS4_ATTN_OUT_LOW_Q8_STATIC_GROUPS 8u
+#define DS4_ATTN_OUT_LOW_Q8_STATIC_BLOCKS 128
+#define DS4_ATTN_OUT_LOW_Q8_STATIC_ROW_BYTES 4352
+#define DS4_ATTN_OUT_LOW_Q8_STATIC_GROUP_BYTES 4456448
+
+static inline void ds4_attn_out_low_q8_static_impl(
+        device const char * src0s,
+        device const char * src1,
+        device       char * dst,
+        threadgroup  char * shmem,
+        uint3 tgpig,
+        ushort tiisg,
+        ushort sgitg) {
+    constexpr short NR0 = N_R0_Q8_0;
+    constexpr short NW = N_SIMDWIDTH;
+    constexpr short NQ = 8;
+    constexpr short NSG = 4;
+
+    // The z grid is flattened as pair = token * groups + group.
+    const uint pair = tgpig.z;
+    const uint group = pair % DS4_ATTN_OUT_LOW_Q8_STATIC_GROUPS;
+    const int r0 = (int)tgpig.x * NR0;
+    device const char *src0 = src0s +
+        (uint64_t)group * DS4_ATTN_OUT_LOW_Q8_STATIC_GROUP_BYTES;
+    device const float *y = (device const float *)(src1 +
+        (uint64_t)pair * DS4_ATTN_OUT_LOW_Q8_STATIC_K * sizeof(float));
+    device float *out = (device float *)(dst +
+        (uint64_t)pair * DS4_ATTN_OUT_LOW_Q8_STATIC_ROWS * sizeof(float));
+
+    device const block_q8_0 *ax[NR0];
+    FOR_UNROLL (short row = 0; row < NR0; ++row) {
+        ax[row] = (device const block_q8_0 *)(src0 +
+            (uint64_t)(r0 + row) * DS4_ATTN_OUT_LOW_Q8_STATIC_ROW_BYTES);
+    }
+
+    float sumf[NR0] = { 0.0f };
+    const short ix = tiisg / (NW / NQ);
+    const short il = tiisg % (NW / NQ);
+    const int ib0 = sgitg * NQ + ix;
+    device const float *yb = y + ib0 * QK8_0 + il * NQ;
+    float yl[NQ];
+
+    FOR_UNROLL (int ib = ib0; ib < DS4_ATTN_OUT_LOW_Q8_STATIC_BLOCKS;
+         ib += NSG * NQ) {
+        FOR_UNROLL (short i = 0; i < NQ; ++i) yl[i] = yb[i];
+        FOR_UNROLL (short row = 0; row < NR0; ++row) {
+            device const int8_t *qs = ax[row][ib].qs + il * NQ;
+            float sumq = 0.0f;
+            FOR_UNROLL (short i = 0; i < NQ; ++i) sumq += qs[i] * yl[i];
+            sumf[row] += sumq * ax[row][ib].d;
+        }
+        yb += NSG * NQ * QK8_0;
+    }
+
+    helper_mv_reduce_and_write<NR0>(
+        out, sumf, r0, DS4_ATTN_OUT_LOW_Q8_STATIC_ROWS,
+        tiisg, sgitg, shmem);
+}
+
+kernel void kernel_dsv4_attn_out_low_q8_0_flash_decode_static_f32(
+        constant ds4_metal_args_mul_mv_id & args,
+        device const char * src0s,
+        device const char * src1,
+        device       char * dst,
+        threadgroup  char * shmem [[threadgroup(0)]],
+        uint3  tgpig[[threadgroup_position_in_grid]],
+        ushort tiitg[[thread_index_in_threadgroup]],
+        ushort tiisg[[thread_index_in_simdgroup]],
+        ushort sgitg[[simdgroup_index_in_threadgroup]]) {
+    ds4_attn_out_low_q8_static_impl(
+        src0s, src1, dst, shmem, tgpig, tiisg, sgitg);
+    (void)args;
+    (void)tiitg;
+}
+
 kernel void kernel_dsv4_attn_out_low_q4_K_f32(
         constant ds4_metal_args_mul_mv_id & args,
         device const char * src0s,
@@ -3715,7 +4400,7 @@ kernel void kernel_mul_mv_id_iq2_xxs_pair_swiglu_f32(
         }
         dst_gate_f32[out_row] = gate;
         dst_up_f32[out_row] = up;
-        const float silu = g / (1.0f + exp(-g));
+        const float silu = ds4_silu(g);
         dst_mid_f32[out_row] = silu * u * route_weight;
     }
 
@@ -3867,7 +4552,7 @@ kernel void kernel_mul_mv_id_iq2_xxs_pair_swiglu_pack2_overlap_f32(
         }
         dst_gate_f32[out_row] = gate;
         dst_up_f32[out_row] = up;
-        const float silu = g / (1.0f + exp(-g));
+        const float silu = ds4_silu(g);
         dst_mid_f32[out_row] = silu * u * route_weight;
     }
 
@@ -3963,7 +4648,7 @@ kernel void kernel_mul_mv_slots6_iq2_xxs_pair_swiglu_f32(
                 g = min(g, c);
                 u = clamp(u, -c, c);
             }
-            const float silu = g / (1.0f + exp(-g));
+            const float silu = ds4_silu(g);
             mid_f32[out_row] = silu * u * route_weight;
         }
     }
@@ -4034,30 +4719,184 @@ kernel void kernel_mul_mv_addr_iq2_xxs_pair_swiglu_f32(
         tiisg,
         sgitg);
 
-    const short NSG = FC_mul_mv_nsg;
-    const int first_row = (tgpig.x * NSG + sgitg) * N_R0_IQ2_XXS;
-    device float *gate_f32 = (device float *)dst_gate_cur;
-    device float *up_f32 = (device float *)dst_up_cur;
-    const uint64_t pair_row = (uint64_t)i12 * (uint64_t)args.nei0 + (uint64_t)idx;
-    device float *mid_f32 = (device float *)(dst_mid + pair_row * act.mid_row_stride);
-    device const float *route_w = (device const float *)(weights + pair_row * act.weight_stride);
-    const float c = act.clamp_value;
-    const float route_weight = route_w[0];
-
-    if (tiisg == 0) {
-        for (int row = 0; row < N_R0_IQ2_XXS && first_row + row < args.ne0; ++row) {
-            const uint out_row = first_row + row;
-            float g = gate_f32[out_row];
-            float u = up_f32[out_row];
-            if (c > 1.0e-6f) {
-                g = min(g, c);
-                u = clamp(u, -c, c);
-            }
-            const float silu = g / (1.0f + exp(-g));
-            mid_f32[out_row] = silu * u * route_weight;
-        }
-    }
-
+    const short NSG = FC_mul_mv_nsg;
+    const int first_row = (tgpig.x * NSG + sgitg) * N_R0_IQ2_XXS;
+    device float *gate_f32 = (device float *)dst_gate_cur;
+    device float *up_f32 = (device float *)dst_up_cur;
+    const uint64_t pair_row = (uint64_t)i12 * (uint64_t)args.nei0 + (uint64_t)idx;
+    device float *mid_f32 = (device float *)(dst_mid + pair_row * act.mid_row_stride);
+    device const float *route_w = (device const float *)(weights + pair_row * act.weight_stride);
+    const float c = act.clamp_value;
+    const float route_weight = route_w[0];
+
+    if (tiisg == 0) {
+        for (int row = 0; row < N_R0_IQ2_XXS && first_row + row < args.ne0; ++row) {
+            const uint out_row = first_row + row;
+            float g = gate_f32[out_row];
+            float u = up_f32[out_row];
+            if (c > 1.0e-6f) {
+                g = min(g, c);
+                u = clamp(u, -c, c);
+            }
+            const float silu = ds4_silu(g);
+            mid_f32[out_row] = silu * u * route_weight;
+        }
+    }
+
+    (void)tiitg;
+}
+
+// M1 SSD-streaming candidate for the exact Flash 4096 -> 2048 gate/up shape.
+// `dst_gate` and `dst_up` remain in the ABI so the host can switch pipelines
+// without changing resource bindings; this kernel intentionally never writes
+// either buffer.  Host dispatch is opt-in and falls back to the kernel above
+// for every other shape or execution mode.
+kernel void kernel_mul_mv_addr_iq2_xxs_pair_swiglu_mid_only_4096x2048_f32(
+        constant ds4_metal_args_mul_mv_id & args,
+        constant ds4_metal_dsv4_moe_swiglu_weight_args & act,
+        device const uint64_t * gate_addrs,
+        device const uint64_t * up_addrs,
+        device const char * src1,
+        device       char * dst_gate,
+        device       char * dst_up,
+        device       char * dst_mid,
+        device const char * ids,
+        device const char * weights,
+        threadgroup  char * shmem [[threadgroup(0)]],
+        uint3  tgpig[[threadgroup_position_in_grid]],
+        ushort tiitg[[thread_index_in_threadgroup]],
+        ushort tiisg[[thread_index_in_simdgroup]],
+        ushort sgitg[[simdgroup_index_in_threadgroup]]) {
+    if (args.ne00 != 4096 || args.ne01 != 2048 || args.ne0 != 2048 ||
+        args.nr0 != N_R0_IQ2_XXS || args.nei1 != 1) {
+        return;
+    }
+
+    const int iid1 = tgpig.z / args.nei0;
+    const int idx = tgpig.z % args.nei0;
+    tgpig.z = 0;
+
+    const int32_t i02 = ((device const int32_t *)(ids + iid1 * args.nbi1))[idx];
+    if (i02 < 0 || i02 >= args.ne02 || i02 >= 384) {
+        return;
+    }
+    const uint64_t gate_addr = gate_addrs[(uint)i02];
+    const uint64_t up_addr = up_addrs[(uint)i02];
+    if (gate_addr == 0 || up_addr == 0) {
+        return;
+    }
+
+    const int64_t i11 = idx % args.ne11;
+    const int64_t i12 = iid1;
+    device const char *src0_gate_cur =
+        reinterpret_cast<device const char *>(gate_addr);
+    device const char *src0_up_cur =
+        reinterpret_cast<device const char *>(up_addr);
+    device const char *src1_cur = src1 + i11 * args.nb11 + i12 * args.nb12;
+    const uint64_t pair_row = (uint64_t)i12 * (uint64_t)args.nei0 + (uint64_t)idx;
+
+    ds4_metal_args_mul_mv args0 = {
+        args.ne00, args.ne01, 1,
+        args.nb00, args.nb01, args.nb02, args.nb02,
+        args.ne10, 1, 1,
+        args.nb10, args.nb11, args.nb12, args.nb12,
+        args.ne0, 1, args.nr0, 1, 1,
+    };
+    kernel_mul_mv_iq2_xxs_pair_swiglu_mid_only_4096x2048_impl<N_R0_IQ2_XXS>(
+        args0,
+        act,
+        src0_gate_cur,
+        src0_up_cur,
+        src1_cur,
+        dst_mid,
+        weights,
+        pair_row,
+        shmem,
+        tgpig,
+        tiisg,
+        sgitg);
+
+    (void)dst_gate;
+    (void)dst_up;
+    (void)tiitg;
+}
+
+// Same mid-only arithmetic for the ordinary SSD cache split.  Complementary
+// resident/missing masks may execute in separate command buffers; each active
+// slot owns a disjoint mid row, so no gate/up materialization is required.
+kernel void kernel_mul_mv_addr_iq2_xxs_pair_swiglu_mid_only_4096x2048_masked_f32(
+        constant ds4_metal_args_mul_mv_id & args,
+        constant ds4_metal_dsv4_moe_swiglu_weight_args & act,
+        constant ds4_metal_stream_expert_split_args & split,
+        device const uint64_t * gate_addrs,
+        device const uint64_t * up_addrs,
+        device const char * src1,
+        device       char * dst_gate,
+        device       char * dst_up,
+        device       char * dst_mid,
+        device const char * ids,
+        device const char * weights,
+        threadgroup  char * shmem [[threadgroup(0)]],
+        uint3  tgpig[[threadgroup_position_in_grid]],
+        ushort tiitg[[thread_index_in_threadgroup]],
+        ushort tiisg[[thread_index_in_simdgroup]],
+        ushort sgitg[[simdgroup_index_in_threadgroup]]) {
+    if (args.ne00 != 4096 || args.ne01 != 2048 || args.ne0 != 2048 ||
+        args.nr0 != N_R0_IQ2_XXS || args.nei1 != 1 ||
+        split.accumulate != 0u || split.active_mask == 0u ||
+        (split.active_mask & ~0x3fu) != 0u) {
+        return;
+    }
+
+    const int iid1 = tgpig.z / args.nei0;
+    const int idx = tgpig.z % args.nei0;
+    if ((split.active_mask & (1u << (uint)idx)) == 0) {
+        return;
+    }
+    tgpig.z = 0;
+
+    const int32_t i02 = ((device const int32_t *)(ids + iid1 * args.nbi1))[idx];
+    if (i02 < 0 || i02 >= args.ne02 || i02 >= 384) {
+        return;
+    }
+    const uint64_t gate_addr = gate_addrs[(uint)i02];
+    const uint64_t up_addr = up_addrs[(uint)i02];
+    if (gate_addr == 0 || up_addr == 0) {
+        return;
+    }
+
+    const int64_t i11 = idx % args.ne11;
+    const int64_t i12 = iid1;
+    device const char *src0_gate_cur =
+        reinterpret_cast<device const char *>(gate_addr);
+    device const char *src0_up_cur =
+        reinterpret_cast<device const char *>(up_addr);
+    device const char *src1_cur = src1 + i11 * args.nb11 + i12 * args.nb12;
+    const uint64_t pair_row = (uint64_t)i12 * (uint64_t)args.nei0 + (uint64_t)idx;
+
+    ds4_metal_args_mul_mv args0 = {
+        args.ne00, args.ne01, 1,
+        args.nb00, args.nb01, args.nb02, args.nb02,
+        args.ne10, 1, 1,
+        args.nb10, args.nb11, args.nb12, args.nb12,
+        args.ne0, 1, args.nr0, 1, 1,
+    };
+    kernel_mul_mv_iq2_xxs_pair_swiglu_mid_only_4096x2048_impl<N_R0_IQ2_XXS>(
+        args0,
+        act,
+        src0_gate_cur,
+        src0_up_cur,
+        src1_cur,
+        dst_mid,
+        weights,
+        pair_row,
+        shmem,
+        tgpig,
+        tiisg,
+        sgitg);
+
+    (void)dst_gate;
+    (void)dst_up;
     (void)tiitg;
 }
 
@@ -4200,7 +5039,7 @@ kernel void kernel_mul_mv_addr_iq2_xxs_pair_swiglu_masked_f32(
                 g = min(g, c);
                 u = clamp(u, -c, c);
             }
-            const float silu = g / (1.0f + exp(-g));
+            const float silu = ds4_silu(g);
             mid_f32[out_row] = silu * u * route_weight;
         }
     }
@@ -4485,7 +5324,7 @@ kernel void kernel_mul_mv_id_q4_K_pair_swiglu_f32(
             }
             gate_f32[out_row] = gate;
             up_f32[out_row] = up;
-            const float silu = g / (1.0f + exp(-g));
+            const float silu = ds4_silu(g);
             mid_f32[out_row] = silu * u * route_weight;
         }
     }
@@ -4583,7 +5422,7 @@ void kernel_mul_mv_mxfp4_pair_swiglu_impl(
                 }
                 gate_f32[out_row] = gate;
                 up_f32[out_row] = up;
-                mid_f32[out_row] = (g / (1.0f + exp(-g))) * u * route_weight;
+                mid_f32[out_row] = ds4_silu(g) * u * route_weight;
             }
         }
     }
@@ -4764,7 +5603,7 @@ void kernel_mul_mv_mxfp4_pair_swiglu_static_impl(
                 }
                 gate_f32[out_row] = gate;
                 up_f32[out_row] = up;
-                mid_f32[out_row] = (g / (1.0f + exp(-g))) * u * route_weight;
+                mid_f32[out_row] = ds4_silu(g) * u * route_weight;
             }
         }
     }
@@ -5068,7 +5907,7 @@ kernel void kernel_mul_mv_table_q4_K_pair_swiglu_f32(
             }
             gate_f32[out_row] = gate;
             up_f32[out_row] = up;
-            const float silu = g / (1.0f + exp(-g));
+            const float silu = ds4_silu(g);
             mid_f32[out_row] = silu * u * route_weight;
         }
     }
@@ -5159,7 +5998,7 @@ kernel void kernel_mul_mv_addr_q4_K_pair_swiglu_f32(
                 g = min(g, c);
                 u = clamp(u, -c, c);
             }
-            const float silu = g / (1.0f + exp(-g));
+            const float silu = ds4_silu(g);
             mid_f32[out_row] = silu * u * route_weight;
         }
     }
@@ -5306,7 +6145,7 @@ kernel void kernel_mul_mv_slots6_q4_K_pair_swiglu_f32(
                 g = min(g, c);
                 u = clamp(u, -c, c);
             }
-            const float silu = g / (1.0f + exp(-g));
+            const float silu = ds4_silu(g);
             mid_f32[out_row] = silu * u * route_weight;
         }
     }
@@ -5478,7 +6317,7 @@ kernel void kernel_mul_mv_group6_q4_K_pair_swiglu_f32(
                 g = min(g, c);
                 u = clamp(u, -c, c);
             }
-            const float silu = g / (1.0f + exp(-g));
+            const float silu = ds4_silu(g);
             mid_f32[out_row] = silu * u * route_weight;
         }
     }
@@ -5602,7 +6441,7 @@ kernel void kernel_mul_mv_group8_q4_K_pair_swiglu_f32(
                 g = min(g, c);
                 u = clamp(u, -c, c);
             }
-            const float silu = g / (1.0f + exp(-g));
+            const float silu = ds4_silu(g);
             mid_f32[out_row] = silu * u * route_weight;
         }
     }
@@ -5785,7 +6624,7 @@ kernel void kernel_mul_mv_group_q4_K_pair_swiglu_f32(
                 g = min(g, c);
                 u = clamp(u, -c, c);
             }
-            const float silu = g / (1.0f + exp(-g));
+            const float silu = ds4_silu(g);
             mid_f32[out_row] = silu * u * route_weight;
         }
     }
@@ -7750,8 +8589,7 @@ kernel void kernel_mul_mm_id_map0(
     const short ide = tpitg;
 
     uint32_t n_all = 0;
-
-    device int32_t * ids_i32 = (device int32_t *) hids + ide*args.ne21;
+    device int32_t * ids_i32 = (device int32_t *) hids;
 
     for (int i21 = 0; i21 < args.ne21; i21 += ntg) {
         if (i21 + tpitg < args.ne21) {
@@ -7774,36 +8612,80 @@ kernel void kernel_mul_mm_id_map0(
 
             threadgroup const uint16_t * sids = (threadgroup const uint16_t *) shmem + t*ne20;
 
-            short sel = 0;
+            uint32_t matches = 0;
             #pragma unroll(ne20)
             for (short i20 = 0; i20 < ne20; i20++) {
-                sel += (sids[i20] == ide)*(i20 + 1);
+                matches += sids[i20] == ide;
             }
-
-            ids_i32[n_all] = (i21 + t)*ne20 + sel - 1;
-
-            n_all += sel > 0;
+            n_all += matches;
         }
 
         threadgroup_barrier(mem_flags::mem_threadgroup);
     }
 
+    /* Store a packed route list, not a fixed n_tokens slice per expert.  A
+     * malformed/synthetic top-k list may select the same expert in multiple
+     * slots of one token.  The historical `sel += slot + 1` collapsed those
+     * routes and could even point at a different slot.  Counting every match
+     * and assigning prefix-sum ranges preserves the original (token, slot)
+     * identity while keeping total storage bounded by ne21 * ne20. */
+    threadgroup uint32_t * route_counts =
+        (threadgroup uint32_t *) shmem;
+    route_counts[ide] = n_all;
+    threadgroup_barrier(mem_flags::mem_threadgroup);
+
+    uint32_t route_base = 0;
+    uint32_t tile_base = 0;
+    for (ushort i = 0; i < ide; i++) {
+        const uint32_t count = route_counts[i];
+        route_base += count;
+        tile_base += (count + 31u) / 32u;
+    }
+
     device uint32_t * tpe_u32 = (device uint32_t *) (htpe);
-    tpe_u32[ide] = n_all;
+    tpe_u32[2u*ide + 0u] = n_all;
+    tpe_u32[2u*ide + 1u] = route_base;
+    threadgroup_barrier(mem_flags::mem_threadgroup);
+
+    uint32_t route = 0;
+    for (int i21 = 0; i21 < args.ne21; i21 += ntg) {
+        if (i21 + tpitg < args.ne21) {
+            device const int32_t * src2_i32 =
+                (device const int32_t *)(src2 +
+                    (i21 + tpitg)*args.nb21);
+            threadgroup uint16_t * sids =
+                (threadgroup uint16_t *) shmem + tpitg*ne20;
+
+            #pragma unroll(ne20)
+            for (short i20 = 0; i20 < ne20; i20++) {
+                sids[i20] = src2_i32[i20];
+            }
+        }
+
+        threadgroup_barrier(mem_flags::mem_threadgroup);
+
+        for (short t = 0; t < ntg; t++) {
+            if (i21 + t >= args.ne21) break;
+            threadgroup const uint16_t * sids =
+                (threadgroup const uint16_t *) shmem + t*ne20;
+
+            #pragma unroll(ne20)
+            for (short i20 = 0; i20 < ne20; i20++) {
+                if (sids[i20] == ide) {
+                    ids_i32[route_base + route++] =
+                        (i21 + t)*ne20 + i20;
+                }
+            }
+        }
+
+        threadgroup_barrier(mem_flags::mem_threadgroup);
+    }
 
     // Reuse the route-id staging memory after the map is complete to build a
     // compact list of non-empty 32-row matmul tiles. The old dispatch covered
     // every possible token tile for every expert, even though most experts
     // receive only a small fraction of the prompt rows.
-    threadgroup uint16_t * tile_counts = (threadgroup uint16_t *) shmem;
     const uint16_t n_tiles = (uint16_t)((n_all + 31u) / 32u);
-    tile_counts[ide] = n_tiles;
-    threadgroup_barrier(mem_flags::mem_threadgroup);
-
-    uint32_t tile_base = 0;
-    for (ushort i = 0; i < ide; i++) {
-        tile_base += tile_counts[i];
-    }
 
     device uint32_t * work_count = (device uint32_t *) work;
     device uint2 * work_items = (device uint2 *)(work + 8);
@@ -7847,8 +8729,8 @@ kernel void kernel_mul_mm_id_map_scatter_work(
         ushort   ntg[[threads_per_threadgroup]]) {
     threadgroup atomic_uint * counts =
         (threadgroup atomic_uint *) shmem;
-    threadgroup uint16_t * tile_counts =
-        (threadgroup uint16_t *)(shmem +
+    threadgroup uint32_t * route_bases =
+        (threadgroup uint32_t *)(shmem +
             (uint32_t)args.ne02*sizeof(uint32_t));
     device uint32_t * tpe_u32 = (device uint32_t *) htpe;
     device int32_t * ids_i32 = (device int32_t *) hids;
@@ -7869,32 +8751,56 @@ kernel void kernel_mul_mm_id_map_scatter_work(
                 continue;
             }
 
-            const uint32_t row = atomic_fetch_add_explicit(
-                counts + expert, 1u, memory_order_relaxed);
-            // Production top-k selections are unique. Keep malformed or
-            // synthetic duplicates from exceeding the fixed expert slice.
-            if (row < (uint32_t)args.ne21) {
-                ids_i32[(uint32_t)expert*args.ne21 + row] =
-                    i21*ne20 + i20;
-            }
+            atomic_fetch_add_explicit(counts + expert,
+                                      1u,
+                                      memory_order_relaxed);
         }
     }
 
     threadgroup_barrier(mem_flags::mem_threadgroup);
 
     const short ide = tpitg;
-    const uint32_t n_all = min(
-        atomic_load_explicit(counts + ide, memory_order_relaxed),
-        (uint32_t)args.ne21);
-    tpe_u32[ide] = n_all;
+    const uint32_t n_all =
+        atomic_load_explicit(counts + ide, memory_order_relaxed);
+    uint32_t route_base = 0;
+    for (ushort i = 0; i < ide; i++) {
+        route_base += atomic_load_explicit(counts + i,
+                                           memory_order_relaxed);
+    }
+    tpe_u32[2u*ide + 0u] = n_all;
+    tpe_u32[2u*ide + 1u] = route_base;
+    route_bases[ide] = route_base;
+    threadgroup_barrier(mem_flags::mem_threadgroup);
+
+    /* Reuse the counters as per-expert write cursors for a second, stable-ABI
+     * scatter.  The packed ranges have room for every route, including
+     * repeated expert IDs in different top-k slots. */
+    atomic_store_explicit(counts + ide, 0u, memory_order_relaxed);
+    threadgroup_barrier(mem_flags::mem_threadgroup);
+
+    for (int i21 = tpitg; i21 < args.ne21; i21 += ntg) {
+        device const int32_t * src2_i32 =
+            (device const int32_t *)(src2 + i21*args.nb21);
+
+        #pragma unroll(ne20)
+        for (short i20 = 0; i20 < ne20; i20++) {
+            const int32_t expert = src2_i32[i20];
+            if ((uint32_t)expert >= (uint32_t)args.ne02) continue;
+            const uint32_t row = atomic_fetch_add_explicit(
+                counts + expert, 1u, memory_order_relaxed);
+            const uint32_t base = route_bases[(uint32_t)expert];
+            ids_i32[base + row] = i21*ne20 + i20;
+        }
+    }
 
-    const uint16_t n_tiles = (uint16_t)((n_all + 31u) / 32u);
-    tile_counts[ide] = n_tiles;
     threadgroup_barrier(mem_flags::mem_threadgroup);
 
+    const uint16_t n_tiles = (uint16_t)((n_all + 31u) / 32u);
     uint32_t tile_base = 0;
     for (ushort i = 0; i < ide; i++) {
-        tile_base += tile_counts[i];
+        const uint32_t count_i =
+            atomic_load_explicit(counts + i, memory_order_relaxed);
+        tile_base += (count_i + 31u) / 32u;
     }
 
     device uint32_t * work_count = (device uint32_t *) work;
@@ -7958,7 +8864,8 @@ kernel void kernel_mul_mm_id(
     device const uint32_t * tpe_u32 = (device const uint32_t *) (htpe);
     device const int32_t  * ids_i32 = (device const int32_t  *) (hids);
 
-    const int32_t neh1 = tpe_u32[im];
+    const int32_t neh1 = tpe_u32[2u*(uint)im + 0u];
+    const uint32_t route_base = tpe_u32[2u*(uint)im + 1u];
 
     if (r1 >= neh1) {
         return;
@@ -7977,7 +8884,7 @@ kernel void kernel_mul_mm_id(
          * the downstream swiglu/sum stages stay unchanged. Each (token,slot)
          * row belongs to exactly one expert, so nothing else writes them. */
         for (short j = sgitg; j < nr1; j += 4) {
-            const int idj = ids_i32[im*args.ne21 + r1 + j];
+            const int idj = ids_i32[route_base + r1 + j];
 
             const short ide = idj % args.ne20;
             const short idt = idj / args.ne20;
@@ -7998,7 +8905,7 @@ kernel void kernel_mul_mm_id(
 
     short il = il0;
 
-    const int id = ids_i32[im*args.ne21 + r1 + lr1];
+    const int id = ids_i32[route_base + r1 + lr1];
 
     const short i11 = (id % args.ne20) % args.ne11;
     const short i12 = (id / args.ne20);
@@ -8022,7 +8929,7 @@ kernel void kernel_mul_mm_id(
 
     simdgroup_float8x8 mc[8];
 
-    for (short i = 0; i < 8; i++){
+    FOR_UNROLL (short i = 0; i < 8; i++) {
         mc[i] = make_filled_simdgroup_matrix<float, 8>(0.f);
     }
 
@@ -8030,7 +8937,7 @@ kernel void kernel_mul_mm_id(
         if (is_same<T0_4x4, block_q>::value && FC_mul_mm_bc_inp) {
             threadgroup_barrier(mem_flags::mem_threadgroup);
 
-            for (short i = 0; i < 16; i++) {
+            FOR_UNROLL (short i = 0; i < 16; i++) {
                 const short sx = 2*il0 + i/8;
                 const short sy = (tiitg/NL0)/8;
 
@@ -8061,7 +8968,7 @@ kernel void kernel_mul_mm_id(
         }
 
         if (FC_mul_mm_bc_inp) {
-            for (short i = 0; i < 8; ++i) {
+            FOR_UNROLL (short i = 0; i < 8; ++i) {
                 const short sx = (tiitg%NL1);
                 const short sy = (tiitg/NL1)/8;
 
@@ -8124,7 +9031,7 @@ kernel void kernel_mul_mm_id(
     threadgroup float * temp_str = ((threadgroup float *) shmem) + 32*(sgitg&1) + (16*(sgitg >> 1))*NR0;
 
     if (mma_active) {
-        for (short i = 0; i < 8; i++) {
+        FOR_UNROLL (short i = 0; i < 8; i++) {
             simdgroup_store(mc[i], temp_str + 8*(i%4) + 8*NR0*(i/4), NR0, 0, false);
         }
     }
@@ -8132,7 +9039,7 @@ kernel void kernel_mul_mm_id(
     threadgroup_barrier(mem_flags::mem_threadgroup);
 
     for (short j = sgitg; j < nr1; j += 4) {
-        const int idj = ids_i32[im*args.ne21 + r1 + j];
+        const int idj = ids_i32[route_base + r1 + j];
 
         const short ide = idj % args.ne20;
         const short idt = idj / args.ne20;
@@ -8155,10 +9062,144 @@ kernel void kernel_mul_mm_id(
     }
 }
 
+// Fixed-routing Q4_K attention output-A projection for the production
+// [token][group][K] -> [token][group][rank] layout.  The generic routed
+// matmul builds an expert-major map, work list, and scatter description even
+// though attention output-A always routes slot g to weight group g.  Keep the
+// same Q4 dequantization, F32->F16 staging, K-loop order, simdgroup MMA, and
+// final scatter arithmetic while deriving that fixed route directly from the
+// dispatch coordinates.
+kernel void kernel_attn_out_low_q4_K_legacy_direct(
+        constant ds4_metal_args_mul_mm_id & args,
+        device const char * src0,
+        device const char * src1,
+        device       char * dst,
+        threadgroup  char * shmem [[threadgroup(0)]],
+        uint3  tgpig [[threadgroup_position_in_grid]],
+        ushort tiitg [[thread_index_in_threadgroup]],
+        ushort tiisg [[thread_index_in_simdgroup]],
+        ushort sgitg [[simdgroup_index_in_threadgroup]]) {
+    constexpr int NR0 = 64;
+    constexpr int NR1 = 32;
+    constexpr int NK  = 32;
+    constexpr int NL0 = NK/16;
+    constexpr int NL1 = NK/8;
+    constexpr short Q4_NL = 16;
+    constexpr int SA_BYTES = NR0*NR1*(int)sizeof(half);
+
+    const int group = (int)tgpig.z;
+    const int r0 = (int)tgpig.y*NR0;
+    const int r1 = (int)tgpig.x*NR1;
+    if (group >= args.ne02 || r0 >= args.ne0 || r1 >= args.ne21) return;
+
+    const short nr0 = (args.ne0 - r0 < NR0) ? (args.ne0 - r0) : NR0;
+    const short nr1 = (args.ne21 - r1 < NR1) ? (args.ne21 - r1) : NR1;
+    const short lr0 = ((short)tiitg/NL0) < nr0 ?
+        ((short)tiitg/NL0) : nr0 - 1;
+    const short lr1 = ((short)tiitg/NL1) < nr1 ?
+        ((short)tiitg/NL1) : nr1 - 1;
+    const short il0 = tiitg % NL0;
+    short il = il0;
+
+    threadgroup half *sa = (threadgroup half *)shmem;
+    threadgroup half *sb =
+        (threadgroup half *)(shmem + SA_BYTES);
+
+    const uint64_t offset0 = (uint64_t)group*args.nb02;
+    const short offset1 = il0/Q4_NL;
+    device const block_q4_K *x =
+        (device const block_q4_K *)(src0 + args.nb01*(r0 + lr0) + offset0) +
+        offset1;
+
+    const short iy = 8*(tiitg % NL1);
+    device const float *y = (device const float *)(src1
+        + args.nb12*(r1 + lr1)
+        + args.nb11*group
+        + args.nb10*iy);
+
+    simdgroup_half8x8 ma[4];
+    simdgroup_half8x8 mb[2];
+    simdgroup_float8x8 mc[8];
+    FOR_UNROLL (short i = 0; i < 8; i++) {
+        mc[i] = make_filled_simdgroup_matrix<float, 8>(0.0f);
+    }
+
+    for (int loop_k = 0; loop_k < args.ne00; loop_k += NK) {
+        half4x4 temp_a;
+        dequantize_q4_K(x, il, temp_a);
+
+        threadgroup_barrier(mem_flags::mem_threadgroup);
+
+        FOR_UNROLL (short i = 0; i < 16; i++) {
+            const short sx = 2*il0 + i/8;
+            const short sy = (tiitg/NL0)/8;
+            const short lx = (tiitg/NL0)%8;
+            const short ly = i%8;
+            const short ib = 8*sx + sy;
+            *(sa + 64*ib + 8*ly + lx) = temp_a[i/4][i%4];
+        }
+
+        const short sx = tiitg%NL1;
+        const short sy = (tiitg/NL1)/8;
+        const short ly = (tiitg/NL1)%8;
+        const short ib = 4*sx + sy;
+        *(threadgroup half2x4 *)(sb + 64*ib + 8*ly) =
+            half2x4(*((device float2x4 *)y));
+
+        il = (il + 2 < Q4_NL) ? il + 2 : il % 2;
+        x  = (il < 2) ? x + (2 + Q4_NL - 1)/Q4_NL : x;
+        y += NK;
+
+        threadgroup_barrier(mem_flags::mem_threadgroup);
+
+        threadgroup const half *lsma = sa + 4*64*(sgitg%2);
+        threadgroup const half *lsmb = sb + 2*64*(sgitg/2);
+        FOR_UNROLL (short ik = 0; ik < NK/8; ik++) {
+            simdgroup_barrier(mem_flags::mem_none);
+            FOR_UNROLL (short i = 0; i < 4; i++) {
+                simdgroup_load(ma[i], lsma + 64*i, 8, 0, false);
+            }
+            simdgroup_barrier(mem_flags::mem_none);
+            FOR_UNROLL (short i = 0; i < 2; i++) {
+                simdgroup_load(mb[i], lsmb + 64*i, 8, 0, false);
+            }
+            simdgroup_barrier(mem_flags::mem_none);
+            FOR_UNROLL (short i = 0; i < 8; i++) {
+                simdgroup_multiply_accumulate(mc[i], mb[i/4], ma[i%4], mc[i]);
+            }
+            lsma += 8*64;
+            lsmb += 4*64;
+        }
+    }
+
+    threadgroup_barrier(mem_flags::mem_threadgroup);
+    threadgroup float *temp_str =
+        ((threadgroup float *)shmem) + 32*(sgitg&1) +
+        (16*(sgitg >> 1))*NR0;
+    FOR_UNROLL (short i = 0; i < 8; i++) {
+        simdgroup_store(mc[i], temp_str + 8*(i%4) +
+            8*NR0*(i/4), NR0, 0, false);
+    }
+
+    threadgroup_barrier(mem_flags::mem_threadgroup);
+    for (short j = sgitg; j < nr1; j += 4) {
+        device float *D = (device float *)dst + r0 + group*args.ne0 +
+            (uint64_t)(r1 + j)*args.ne1*args.ne0;
+        device float4 *D4 = (device float4 *)D;
+        threadgroup float *C = (threadgroup float *)shmem + j*NR0;
+        threadgroup float4 *C4 = (threadgroup float4 *)C;
+
+        int i = tiisg;
+        for (; i < nr0/4; i += 32) D4[i] = C4[i];
+        i = 4*(nr0/4) + tiisg;
+        for (; i < nr0; i += 32) D[i] = C[i];
+    }
+}
+
 // Address-table variant used by SSD streaming.  The routing ids remain the
 // model's original expert ids, but each expert's resident buffer is found via a
 // GPU-address table instead of a contiguous full-layer tensor.
-template<short NR1, typename S0, typename S0_4x4, typename S0_8x8, typename S1, typename S1_2x4, typename S1_8x8, typename block_q, short nl, void (*dequantize_func)(device const block_q *, short, thread S0_4x4 &), typename T0, typename T0_4x4, typename T1, typename T1_2x4>
+template<short NR1, typename S0, typename S0_4x4, typename S0_8x8, typename S1, typename S1_2x4, typename S1_8x8, typename block_q, short nl, void (*dequantize_func)(device const block_q *, short, thread S0_4x4 &), typename T0, typename T0_4x4, typename T1, typename T1_2x4, bool CULL_TAIL_SIMDGROUPS = false>
 kernel void kernel_mul_mm_id_addr(
         constant ds4_metal_args_mul_mm_id & args,
         device const uint64_t * src0_addrs,
@@ -8166,6 +9207,7 @@ kernel void kernel_mul_mm_id_addr(
         device const char * htpe,
         device const char * hids,
         device       char * dst,
+        device const char * work,
         threadgroup  char * shmem [[threadgroup(0)]],
         uint3  tgpig[[threadgroup_position_in_grid]],
         ushort tiitg[[thread_index_in_threadgroup]],
@@ -8182,14 +9224,31 @@ kernel void kernel_mul_mm_id_addr(
     threadgroup S0 * sa = (threadgroup S0 *)(shmem);
     threadgroup S1 * sb = (threadgroup S1 *)(shmem + SA_BYTES);
 
-    const int im = tgpig.z;
+    // The SSD address table can contain hundreds of experts while a prompt
+    // routes only top-k rows per token.  Consume the compact non-empty tile
+    // list emitted by kernel_mul_mm_id_map0 instead of launching the full
+    // expert x token-tile Cartesian grid.
+    device const uint32_t * work_count =
+        (device const uint32_t *)work;
+    const uint32_t work_index = tgpig.x;
+    if (work_index >= work_count[0]) {
+        return;
+    }
+    device const uint2 * work_items =
+        (device const uint2 *)(work + 8);
+    const uint2 item = work_items[work_index];
+    const int im = (int)item.x;
+    if ((uint)im >= (uint)args.ne02) {
+        return;
+    }
     const int r0 = tgpig.y*NR0;
-    const int r1 = tgpig.x*NR1;
+    const int r1 = (int)item.y;
 
     device const uint32_t * tpe_u32 = (device const uint32_t *) (htpe);
     device const int32_t  * ids_i32 = (device const int32_t  *) (hids);
 
-    const int32_t neh1 = tpe_u32[im];
+    const int32_t neh1 = tpe_u32[2u*(uint)im + 0u];
+    const uint32_t route_base = tpe_u32[2u*(uint)im + 1u];
 
     if (r1 >= neh1) {
         return;
@@ -8203,6 +9262,8 @@ kernel void kernel_mul_mm_id_addr(
 
     const short nr0 = (args.ne0 - r0 < NR0) ? (args.ne0 - r0) : NR0;
     const short nr1 = (    neh1 - r1 < NR1) ? (    neh1 - r1) : NR1;
+    const bool mma_active =
+        !CULL_TAIL_SIMDGROUPS || 16*(short)(sgitg/2) < nr1;
 
     const short lr0 = ((short)tiitg/NL0) < nr0 ? ((short)tiitg/NL0) : nr0 - 1;
     const short lr1 = ((short)tiitg/NL1) < nr1 ? ((short)tiitg/NL1) : nr1 - 1;
@@ -8211,7 +9272,7 @@ kernel void kernel_mul_mm_id_addr(
 
     short il = il0;
 
-    const int id = ids_i32[im*args.ne21 + r1 + lr1];
+    const int id = ids_i32[route_base + r1 + lr1];
 
     const short i11 = (id % args.ne20) % args.ne11;
     const short i12 = (id / args.ne20);
@@ -8234,7 +9295,7 @@ kernel void kernel_mul_mm_id_addr(
 
     simdgroup_float8x8 mc[8];
 
-    for (short i = 0; i < 8; i++){
+    FOR_UNROLL (short i = 0; i < 8; i++) {
         mc[i] = make_filled_simdgroup_matrix<float, 8>(0.f);
     }
 
@@ -8242,7 +9303,7 @@ kernel void kernel_mul_mm_id_addr(
         if (is_same<T0_4x4, block_q>::value && FC_mul_mm_bc_inp) {
             threadgroup_barrier(mem_flags::mem_threadgroup);
 
-            for (short i = 0; i < 16; i++) {
+            FOR_UNROLL (short i = 0; i < 16; i++) {
                 const short sx = 2*il0 + i/8;
                 const short sy = (tiitg/NL0)/8;
 
@@ -8273,7 +9334,7 @@ kernel void kernel_mul_mm_id_addr(
         }
 
         if (FC_mul_mm_bc_inp) {
-            for (short i = 0; i < 8; ++i) {
+            FOR_UNROLL (short i = 0; i < 8; ++i) {
                 const short sx = (tiitg%NL1);
                 const short sy = (tiitg/NL1)/8;
 
@@ -8305,27 +9366,30 @@ kernel void kernel_mul_mm_id_addr(
         threadgroup const S0 * lsma = (sa + 4*64*(sgitg%2));
         threadgroup const S1 * lsmb = (sb + 2*64*(sgitg/2));
 
-        FOR_UNROLL (short ik = 0; ik < NK/8; ik++) {
-            simdgroup_barrier(mem_flags::mem_none);
+        if (mma_active) {
+            FOR_UNROLL (short ik = 0; ik < NK/8; ik++) {
+                simdgroup_barrier(mem_flags::mem_none);
 
-            FOR_UNROLL (short i = 0; i < 4; i++) {
-                simdgroup_load(ma[i], lsma + 64*i, 8, 0, false);
-            }
+                FOR_UNROLL (short i = 0; i < 4; i++) {
+                    simdgroup_load(ma[i], lsma + 64*i, 8, 0, false);
+                }
 
-            simdgroup_barrier(mem_flags::mem_none);
+                simdgroup_barrier(mem_flags::mem_none);
 
-            FOR_UNROLL (short i = 0; i < 2; i++) {
-                simdgroup_load(mb[i], lsmb + 64*i, 8, 0, false);
-            }
+                FOR_UNROLL (short i = 0; i < 2; i++) {
+                    simdgroup_load(mb[i], lsmb + 64*i, 8, 0, false);
+                }
 
-            simdgroup_barrier(mem_flags::mem_none);
+                simdgroup_barrier(mem_flags::mem_none);
 
-            FOR_UNROLL (short i = 0; i < 8; i++){
-                simdgroup_multiply_accumulate(mc[i], mb[i/4], ma[i%4], mc[i]);
-            }
+                FOR_UNROLL (short i = 0; i < 8; i++){
+                    simdgroup_multiply_accumulate(
+                        mc[i], mb[i/4], ma[i%4], mc[i]);
+                }
 
-            lsma += 8*64;
-            lsmb += 4*64;
+                lsma += 8*64;
+                lsmb += 4*64;
+            }
         }
     }
 
@@ -8333,14 +9397,18 @@ kernel void kernel_mul_mm_id_addr(
 
     threadgroup float * temp_str = ((threadgroup float *) shmem) + 32*(sgitg&1) + (16*(sgitg >> 1))*NR0;
 
-    for (short i = 0; i < 8; i++) {
-        simdgroup_store(mc[i], temp_str + 8*(i%4) + 8*NR0*(i/4), NR0, 0, false);
+    if (mma_active) {
+        FOR_UNROLL (short i = 0; i < 8; i++) {
+            simdgroup_store(mc[i],
+                            temp_str + 8*(i%4) + 8*NR0*(i/4),
+                            NR0, 0, false);
+        }
     }
 
     threadgroup_barrier(mem_flags::mem_threadgroup);
 
     for (short j = sgitg; j < nr1; j += 4) {
-        const int idj = ids_i32[im*args.ne21 + r1 + j];
+        const int idj = ids_i32[route_base + r1 + j];
 
         const short ide = idj % args.ne20;
         const short idt = idj / args.ne20;
@@ -8351,13 +9419,11 @@ kernel void kernel_mul_mm_id_addr(
         threadgroup float  * C  = (threadgroup float  *) shmem + j*NR0;
         threadgroup float4 * C4 = (threadgroup float4 *) C;
 
-        int i = tiisg;
-        for (; i < nr0/4; i += 32) {
+        FOR_UNROLL (int i = tiisg; i < nr0/4; i += 32) {
             *(D4 + i) = *(C4 + i);
         }
 
-        i = (4*(nr0/4)) + tiisg;
-        for (; i < nr0; i += 32) {
+        FOR_UNROLL (int i = (4*(nr0/4)) + tiisg; i < nr0; i += 32) {
             *(D + i) = *(C + i);
         }
     }
@@ -8409,7 +9475,8 @@ kernel void kernel_mul_mm_id_pair_swiglu_f16_impl(
     device const uint32_t * tpe_u32 = (device const uint32_t *) (htpe);
     device const int32_t  * ids_i32 = (device const int32_t  *) (hids);
 
-    const int32_t neh1 = tpe_u32[im];
+    const int32_t neh1 = tpe_u32[2u*(uint)im + 0u];
+    const uint32_t route_base = tpe_u32[2u*(uint)im + 1u];
 
     if (r1 >= neh1) {
         return;
@@ -8429,7 +9496,7 @@ kernel void kernel_mul_mm_id_pair_swiglu_f16_impl(
     const short il0 = (tiitg % NL0);
     short il = il0;
 
-    const int id = ids_i32[im*args.ne21 + r1 + lr1];
+    const int id = ids_i32[route_base + r1 + lr1];
 
     const short i11 = (id % args.ne20) % args.ne11;
     const short i12 = (id / args.ne20);
@@ -8458,7 +9525,7 @@ kernel void kernel_mul_mm_id_pair_swiglu_f16_impl(
     simdgroup_float8x8 mc_gate[8];
     simdgroup_float8x8 mc_up[8];
 
-    for (short i = 0; i < 8; i++) {
+    FOR_UNROLL (short i = 0; i < 8; i++) {
         mc_gate[i] = make_filled_simdgroup_matrix<float, 8>(0.f);
         mc_up[i] = make_filled_simdgroup_matrix<float, 8>(0.f);
     }
@@ -8533,15 +9600,15 @@ kernel void kernel_mul_mm_id_pair_swiglu_f16_impl(
 
     threadgroup float * temp_gate = (threadgroup float *) shmem;
     threadgroup float * temp_up = temp_gate + NR0*NR1;
-    threadgroup float * temp_gate_str =
-        temp_gate + 32*(sgitg&1) + (16*(sgitg >> 1))*NR0;
-    threadgroup float * temp_up_str =
-        temp_up + 32*(sgitg&1) + (16*(sgitg >> 1))*NR0;
+    const int str_index = 32*(sgitg&1) + (16*(sgitg >> 1))*NR0;
+    threadgroup float * temp_gate_str = temp_gate + str_index;
+    threadgroup float * temp_up_str = temp_up + str_index;
 
     if (mma_active) {
-        for (short i = 0; i < 8; i++) {
-            simdgroup_store(mc_gate[i], temp_gate_str + 8*(i%4) + 8*NR0*(i/4), NR0, 0, false);
-            simdgroup_store(mc_up[i],   temp_up_str   + 8*(i%4) + 8*NR0*(i/4), NR0, 0, false);
+        FOR_UNROLL (short i = 0; i < 8; i++) {
+            const int si = (8 * (i & 3)) + (8 * NR0 * (i >> 2));
+            simdgroup_store(mc_gate[i], temp_gate_str + si, NR0, 0, false);
+            simdgroup_store(mc_up[i],   temp_up_str   + si, NR0, 0, false);
         }
     }
 
@@ -8549,7 +9616,7 @@ kernel void kernel_mul_mm_id_pair_swiglu_f16_impl(
 
     const float c = act.clamp_value;
     for (short j = sgitg; j < nr1; j += 4) {
-        const int idj = ids_i32[im*args.ne21 + r1 + j];
+        const int idj = ids_i32[route_base + r1 + j];
 
         const short ide = idj % args.ne20;
         const short idt = idj / args.ne20;
@@ -8562,15 +9629,14 @@ kernel void kernel_mul_mm_id_pair_swiglu_f16_impl(
         threadgroup float *Cg = temp_gate + j*NR0;
         threadgroup float *Cu = temp_up   + j*NR0;
 
-        int i = tiisg;
-        for (; i < nr0; i += 32) {
+        for (int i = tiisg; i < nr0; i += 32) {
             float g = Cg[i];
             float u = Cu[i];
             if (c > 1.0e-6f) {
                 g = min(g, c);
                 u = clamp(u, -c, c);
             }
-            const float silu = g / (1.0f + exp(-g));
+            const float silu = ds4_silu(g);
             D[i] = (half)(silu * u * route_weight);
         }
     }
@@ -8624,7 +9690,8 @@ kernel void kernel_mul_mm_id_pair_swiglu_f16_compact_tail_impl(
     device const uint32_t * tpe_u32 = (device const uint32_t *) (htpe);
     device const int32_t  * ids_i32 = (device const int32_t  *) (hids);
 
-    const int32_t neh1 = tpe_u32[im];
+    const int32_t neh1 = tpe_u32[2u*(uint)im + 0u];
+    const uint32_t route_base = tpe_u32[2u*(uint)im + 1u];
     if (r1 >= neh1) {
         return;
     }
@@ -8642,8 +9709,8 @@ kernel void kernel_mul_mm_id_pair_swiglu_f16_compact_tail_impl(
     const short il0 = (tiitg % NL0);
     short il = il0;
 
-    const int id_b0 = ids_i32[im*args.ne21 + r1 + lr1_b0];
-    const int id_b1 = ids_i32[im*args.ne21 + r1 + lr1_b1];
+    const int id_b0 = ids_i32[route_base + r1 + lr1_b0];
+    const int id_b1 = ids_i32[route_base + r1 + lr1_b1];
 
     const short i11_b0 = (id_b0 % args.ne20) % args.ne11;
     const short i12_b0 = (id_b0 / args.ne20);
@@ -8679,7 +9746,7 @@ kernel void kernel_mul_mm_id_pair_swiglu_f16_compact_tail_impl(
     simdgroup_float8x8 mc_gate[8];
     simdgroup_float8x8 mc_up[8];
 
-    for (short i = 0; i < 8; i++) {
+    FOR_UNROLL (short i = 0; i < 8; i++) {
         mc_gate[i] = make_filled_simdgroup_matrix<float, 8>(0.f);
         mc_up[i] = make_filled_simdgroup_matrix<float, 8>(0.f);
     }
@@ -8763,9 +9830,10 @@ kernel void kernel_mul_mm_id_pair_swiglu_f16_compact_tail_impl(
     threadgroup float * temp_up_str = temp_up + 16*sgitg*NR0;
 
     if (mma_active) {
-        for (short i = 0; i < 8; i++) {
-            simdgroup_store(mc_gate[i], temp_gate_str + 8*(i%4) + 8*NR0*(i/4), NR0, 0, false);
-            simdgroup_store(mc_up[i],   temp_up_str   + 8*(i%4) + 8*NR0*(i/4), NR0, 0, false);
+        FOR_UNROLL (short i = 0; i < 8; i++) {
+            const int si = (8 * (i & 3)) + (8 * NR0 * (i >> 2));
+            simdgroup_store(mc_gate[i], temp_gate_str + si, NR0, 0, false);
+            simdgroup_store(mc_up[i],   temp_up_str   + si, NR0, 0, false);
         }
     }
 
@@ -8773,7 +9841,7 @@ kernel void kernel_mul_mm_id_pair_swiglu_f16_compact_tail_impl(
 
     const float c = act.clamp_value;
     for (short j = sgitg; j < nr1; j += 2) {
-        const int idj = ids_i32[im*args.ne21 + r1 + j];
+        const int idj = ids_i32[route_base + r1 + j];
 
         const short ide = idj % args.ne20;
         const short idt = idj / args.ne20;
@@ -8786,21 +9854,21 @@ kernel void kernel_mul_mm_id_pair_swiglu_f16_compact_tail_impl(
         threadgroup float *Cg = temp_gate + j*NR0;
         threadgroup float *Cu = temp_up   + j*NR0;
 
-        int i = tiisg;
-        for (; i < nr0; i += 32) {
+        for (int i = tiisg; i < nr0; i += 32) {
             float g = Cg[i];
             float u = Cu[i];
             if (c > 1.0e-6f) {
                 g = min(g, c);
                 u = clamp(u, -c, c);
             }
-            const float silu = g / (1.0f + exp(-g));
+            const float silu = ds4_silu(g);
             D[i] = (half)(silu * u * route_weight);
         }
     }
 }
 
 typedef decltype(kernel_mul_mm_id_pair_swiglu_f16_impl<block_iq2_xxs, QK_NL, dequantize_iq2_xxs>) mul_mm_id_pair_swiglu_f16_iq2;
+typedef decltype(kernel_mul_mm_id_pair_swiglu_f16_impl<block_iq2_xxs, QK_NL, dequantize_iq2_xxs, true>) mul_mm_id_pair_swiglu_f16_iq2_tail_cull;
 typedef decltype(kernel_mul_mm_id_pair_swiglu_f16_impl<block_q4_K, QK_NL, dequantize_q4_K>) mul_mm_id_pair_swiglu_f16_q4;
 typedef decltype(kernel_mul_mm_id_pair_swiglu_f16_impl<block_mxfp4, 2, dequantize_mxfp4>) mul_mm_id_pair_swiglu_f16_mxfp4;
 typedef decltype(kernel_mul_mm_id_pair_swiglu_f16_impl<block_mxfp4, 2, dequantize_mxfp4, true>) mul_mm_id_pair_swiglu_f16_mxfp4_tail_cull;
@@ -8808,6 +9876,7 @@ typedef decltype(kernel_mul_mm_id_pair_swiglu_f16_compact_tail_impl<block_mxfp4,
 
 // Host-visible fused routed pair matmuls for the DS4 expert quant formats.
 template [[host_name("kernel_mul_mm_id_iq2_xxs_pair_swiglu_f16")]] kernel mul_mm_id_pair_swiglu_f16_iq2 kernel_mul_mm_id_pair_swiglu_f16_impl<block_iq2_xxs, QK_NL, dequantize_iq2_xxs>;
+template [[host_name("kernel_mul_mm_id_iq2_xxs_pair_swiglu_f16_tail_cull")]] kernel mul_mm_id_pair_swiglu_f16_iq2_tail_cull kernel_mul_mm_id_pair_swiglu_f16_impl<block_iq2_xxs, QK_NL, dequantize_iq2_xxs, true>;
 template [[host_name("kernel_mul_mm_id_q4_K_pair_swiglu_f16")]] kernel mul_mm_id_pair_swiglu_f16_q4 kernel_mul_mm_id_pair_swiglu_f16_impl<block_q4_K, QK_NL, dequantize_q4_K>;
 template [[host_name("kernel_mul_mm_id_mxfp4_pair_swiglu_f16")]] kernel mul_mm_id_pair_swiglu_f16_mxfp4 kernel_mul_mm_id_pair_swiglu_f16_impl<block_mxfp4, 2, dequantize_mxfp4>;
 template [[host_name("kernel_mul_mm_id_mxfp4_pair_swiglu_f16_half_scale")]] kernel mul_mm_id_pair_swiglu_f16_mxfp4 kernel_mul_mm_id_pair_swiglu_f16_impl<block_mxfp4, 2, dequantize_mxfp4_half_scale>;
@@ -8822,6 +9891,7 @@ typedef decltype(kernel_mul_mm_id<32, half, half4x4, simdgroup_half8x8, half, ha
 typedef decltype(kernel_mul_mm_id<32, half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_mxfp4, 2, dequantize_mxfp4_half_lut, half, half4x4, half, half2x4>) mul_mm_id_mxfp4_f16_rhs_half_lut;
 typedef decltype(kernel_mul_mm_id<32, half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_mxfp4, 2, dequantize_mxfp4_half_lut, half, half4x4, half, half2x4, true>) mul_mm_id_mxfp4_f16_rhs_half_lut_tail_cull;
 typedef decltype(kernel_mul_mm_id_addr<32, half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_q2_K, QK_NL, dequantize_q2_K, float, float4x4, float, float2x4>) mul_mm_id_addr;
+typedef decltype(kernel_mul_mm_id_addr<32, half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_q2_K, QK_NL, dequantize_q2_K, float, float4x4, float, float2x4, true>) mul_mm_id_addr_tail_cull;
 typedef decltype(kernel_mul_mm_id_addr<32, half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_q2_K, QK_NL, dequantize_q2_K, half, half4x4, half, half2x4>) mul_mm_id_addr_f16_rhs;
 
 // Host-visible batched MoE matmul variants for the DS4 quant formats.
@@ -8844,6 +9914,9 @@ template [[host_name("kernel_mul_mm_id_mxfp4_f16_half_lut")]] kernel mul_mm_id_m
 template [[host_name("kernel_mul_mm_id_mxfp4_f16_half_lut_tail_cull")]] kernel mul_mm_id_mxfp4_f16_rhs_half_lut_tail_cull kernel_mul_mm_id<32, half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_mxfp4, 2, dequantize_mxfp4_half_lut, half, half4x4, half, half2x4, true>;
 
 template [[host_name("kernel_mul_mm_id_addr_q2_K_f32")]]    kernel mul_mm_id_addr kernel_mul_mm_id_addr<32, half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_q2_K, QK_NL, dequantize_q2_K, float, float4x4, float, float2x4>;
+template [[host_name("kernel_mul_mm_id_addr_iq2_xxs_f32")]] kernel mul_mm_id_addr kernel_mul_mm_id_addr<32, half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_iq2_xxs, QK_NL, dequantize_iq2_xxs, float, float4x4, float, float2x4>;
+template [[host_name("kernel_mul_mm_id_addr_q2_K_f32_tail_cull")]] kernel mul_mm_id_addr_tail_cull kernel_mul_mm_id_addr<32, half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_q2_K, QK_NL, dequantize_q2_K, float, float4x4, float, float2x4, true>;
+template [[host_name("kernel_mul_mm_id_addr_iq2_xxs_f32_tail_cull")]] kernel mul_mm_id_addr_tail_cull kernel_mul_mm_id_addr<32, half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_iq2_xxs, QK_NL, dequantize_iq2_xxs, float, float4x4, float, float2x4, true>;
 template [[host_name("kernel_mul_mm_id_addr_q4_K_f32")]]    kernel mul_mm_id_addr kernel_mul_mm_id_addr<32, half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_q4_K, QK_NL, dequantize_q4_K, float, float4x4, float, float2x4>;
 template [[host_name("kernel_mul_mm_id_addr_mxfp4_f32")]]   kernel mul_mm_id_addr kernel_mul_mm_id_addr<32, half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_mxfp4, 2, dequantize_mxfp4, float, float4x4, float, float2x4>;
 template [[host_name("kernel_mul_mm_id_addr_q2_K_f16")]]    kernel mul_mm_id_addr_f16_rhs kernel_mul_mm_id_addr<32, half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_q2_K, QK_NL, dequantize_q2_K, half, half4x4, half, half2x4>;
@@ -8978,9 +10051,12 @@ kernel void kernel_attn_out_low_mpp_direct_rhs(
     }
 }
 
-// Routed-expert grouped matmul on the Metal4 TensorOps/MPP pipeline. The
-// barrier after mm.run prevents the next K iteration from replacing staged
-// tiles while the cooperative matmul still reads them.
+// Routed-expert grouped matmul on the Metal4 TensorOps/MPP pipeline.
+// The 32 routed rows are two independent 16-row cooperative matmul2d
+// ops (one per SIMDgroup pair). Staging is double-buffered so the K
+// loop takes one barrier per step. A short final tile's empty row-half
+// skips MMA, accumulator init, and store; staging and barriers stay
+// unconditional. Threadgroup is sa[2]+sb[2] = 12 KiB.
 template<typename S0, typename S0_4x4, typename S0_8x8, typename S1, typename S1_2x4, typename S1_8x8, typename block_q, short nl, void (*dequantize_func)(device const block_q *, short, thread S0_4x4 &), typename T0, typename T0_4x4, typename T1, typename T1_2x4>
 kernel void kernel_mul_mm_id_mpp(
         constant ds4_metal_args_mul_mm_id & args,
@@ -8996,7 +10072,7 @@ kernel void kernel_mul_mm_id_mpp(
         ushort tiisg[[thread_index_in_simdgroup]],
         ushort sgitg[[simdgroup_index_in_threadgroup]]) {
     threadgroup S0 * sa = (threadgroup S0 *)(shmem);
-    threadgroup S1 * sb = (threadgroup S1 *)(shmem + 4096);
+    threadgroup S1 * sb = (threadgroup S1 *)(shmem + 8192);
     threadgroup float *sc = (threadgroup float *)shmem;
 
     constexpr int NR0 = 64;
@@ -9019,7 +10095,8 @@ kernel void kernel_mul_mm_id_mpp(
     device const uint32_t * tpe_u32 = (device const uint32_t *) (htpe);
     device const int32_t  * ids_i32 = (device const int32_t  *) (hids);
 
-    const int32_t neh1 = tpe_u32[im];
+    const int32_t neh1 = tpe_u32[2u*(uint)im + 0u];
+    const uint32_t route_base = tpe_u32[2u*(uint)im + 1u];
 
     if (r1 >= neh1) {
         return;
@@ -9028,9 +10105,12 @@ kernel void kernel_mul_mm_id_mpp(
     const short nr0 = (args.ne0 - r0 < NR0) ? (args.ne0 - r0) : NR0;
     const short nr1 = (    neh1 - r1 < NR1) ? (    neh1 - r1) : NR1;
 
+    const short h = sgitg/2;
+    const bool mma_active = 16*h < nr1;
+
     if (!ds4_tp_owns_expert(im, args.ne02, args.tp_rank, args.tp_world)) {
         for (short j = sgitg; j < nr1; j += 4) {
-            const int idj = ids_i32[im*args.ne21 + r1 + j];
+            const int idj = ids_i32[route_base + r1 + j];
             const short ide = idj % args.ne20;
             const short idt = idj / args.ne20;
             device float *D = (device float *)dst + r0 + ide*args.ne0 +
@@ -9044,9 +10124,8 @@ kernel void kernel_mul_mm_id_mpp(
     const short lr1 = ((short)tiitg/NL1) < nr1 ? ((short)tiitg/NL1) : nr1 - 1;
 
     const short il0 = (tiitg % NL0);
-    short il = il0;
 
-    const int id = ids_i32[im*args.ne21 + r1 + lr1];
+    const int id = ids_i32[route_base + r1 + lr1];
 
     const short i11 = (id % args.ne20) % args.ne11;
     const short i12 = (id / args.ne20);
@@ -9056,51 +10135,58 @@ kernel void kernel_mul_mm_id_mpp(
         (uint64_t)(im - args.tp_expert_base)*args.nb02 + i13*args.nb03;
     const short    offset1 = il0/nl;
 
-    device const block_q * x = (device const block_q *)(src0 + args.nb01*(r0 + lr0) + offset0) + offset1;
+    device const block_q * x_base = (device const block_q *)(src0 + args.nb01*(r0 + lr0) + offset0) + offset1;
 
     const short iy = 8*(tiitg % NL1);
 
-    device const T1 * y = (device const T1 *)(src1
+    device const T1 * y_base = (device const T1 *)(src1
         + args.nb13*i13
         + args.nb12*i12
         + args.nb11*i11
         + args.nb10*iy);
 
     auto tA = tensor(sa, dextents<int32_t, 2>(NK, NR0));
-    auto tB = tensor(sb, dextents<int32_t, 2>(NR1, NK));
+    auto tA1 = tensor(sa + NR0*NK, dextents<int32_t, 2>(NK, NR0));
+    auto tBh = tensor(sb + h*16*NK, dextents<int32_t, 2>(NK, 16));
+    auto tBh1 = tensor(sb + NR1*NK + h*16*NK, dextents<int32_t, 2>(NK, 16));
 
     matmul2d<
-        matmul2d_descriptor(NR1, NR0, NK, false, true, false,
+        matmul2d_descriptor(16, NR0, NK, false, true, false,
             matmul2d_descriptor::mode::multiply_accumulate),
-        execution_simdgroups<4>> mm;
+        execution_simdgroups<2>> mm;
 
-    auto cT = mm.template get_destination_cooperative_tensor<decltype(tB), decltype(tA), float>();
+    auto cT = mm.template get_destination_cooperative_tensor<decltype(tBh), decltype(tA), float>();
 
-    #pragma unroll
-    for (uint16_t i = 0; i < cT.get_capacity(); ++i) {
-        if (cT.is_valid_element(i)) {
-            cT[i] = 0.0f;
+    if (mma_active) {
+        #pragma unroll
+        for (uint16_t i = 0; i < cT.get_capacity(); ++i) {
+            if (cT.is_valid_element(i)) {
+                cT[i] = 0.0f;
+            }
         }
     }
 
-    for (int loop_k = 0; loop_k < args.ne00; loop_k += NK) {
-        if (is_same<T0_4x4, block_q>::value && FC_mul_mm_bc_inp) {
-            threadgroup_barrier(mem_flags::mem_threadgroup);
+    auto stage_tile = [&](const int loop_k,
+                          threadgroup S0 * sa_buf,
+                          threadgroup S1 * sb_buf) {
+        const int chunk = loop_k / 16 + il0;
+        const short il = (short)(chunk % nl);
 
-            for (short i = 0; i < 16; i++) {
+        device const block_q * xb = x_base + (chunk / nl);
+
+        if (is_same<T0_4x4, block_q>::value && FC_mul_mm_bc_inp) {
+            FOR_UNROLL (short i = 0; i < 16; i++) {
                 const short sx = 2*il0 + i/8;
                 const short sy = (tiitg/NL0)/8;
                 const short lx = i%8;
                 const short ly = (tiitg/NL0)%8;
 
-                *(sa + NK*(8*sy + ly) + 8*sx + lx) =
-                    loop_k + 16*il + i < args.ne00 ? *((device T0 *) x + i) : 0;
+                *(sa_buf + NK*(8*sy + ly) + 8*sx + lx) =
+                    loop_k + 16*il + i < args.ne00 ? *((device T0 *) xb + i) : 0;
             }
         } else {
             S0_4x4 temp_a;
-            dequantize_func(x, il, temp_a);
-
-            threadgroup_barrier(mem_flags::mem_threadgroup);
+            dequantize_func(xb, il, temp_a);
 
             FOR_UNROLL (short i = 0; i < 16; i++) {
                 const short sx = 2*il0 + i/8;
@@ -9108,10 +10194,12 @@ kernel void kernel_mul_mm_id_mpp(
                 const short lx = i%8;
                 const short ly = (tiitg/NL0)%8;
 
-                *(sa + NK*(8*sy + ly) + 8*sx + lx) = temp_a[i/4][i%4];
+                *(sa_buf + NK*(8*sy + ly) + 8*sx + lx) = temp_a[i/4][i%4];
             }
         }
 
+        device const T1 * yb = y_base + loop_k;
+
         if (FC_mul_mm_bc_inp) {
             for (short i = 0; i < 8; ++i) {
                 const short sx = (tiitg%NL1);
@@ -9119,41 +10207,51 @@ kernel void kernel_mul_mm_id_mpp(
                 const short lx = i;
                 const short ly = (tiitg/NL1)%8;
 
-                *(sb + NK*(8*sy + ly) + 8*sx + lx) =
-                    loop_k + iy + i < args.ne00 ? (S1) *((device T1 *) y + i) : 0;
+                *(sb_buf + NK*(8*sy + ly) + 8*sx + lx) =
+                    loop_k + iy + i < args.ne00 ? (S1) *((device T1 *) yb + i) : 0;
             }
         } else {
             const short sx = (tiitg%NL1);
             const short sy = (tiitg/NL1)/8;
             const short ly = (tiitg/NL1)%8;
 
-            *(threadgroup S1_2x4 *)(sb + NK*(8*sy + ly) + 8*sx) =
-                (S1_2x4)(*((device T1_2x4 *) y));
+            *(threadgroup S1_2x4 *)(sb_buf + NK*(8*sy + ly) + 8*sx) =
+                (S1_2x4)(*((device T1_2x4 *) yb));
         }
+    };
 
-        il = (il + 2 < nl) ? il + 2 : il % 2;
-        x  = (il < 2) ? x + (2 + nl - 1)/nl : x;
-
-        y += NK;
-
-        threadgroup_barrier(mem_flags::mem_threadgroup);
+    stage_tile(0, sa, sb);
+    threadgroup_barrier(mem_flags::mem_threadgroup);
 
-        auto sA = tA.slice(0, 0);
-        auto sB = tB.slice(0, 0);
-        mm.run(sB, sA, cT);
+    uint buf_sel = 0;
+    for (int loop_k = 0; loop_k < args.ne00; loop_k += NK) {
+        if (mma_active) {
+            auto sA = (buf_sel ? tA1 : tA).slice(0, 0);
+            auto sB = (buf_sel ? tBh1 : tBh).slice(0, 0);
+            mm.run(sB, sA, cT);
+        }
 
+        const int next_k = loop_k + NK;
+        if (next_k < args.ne00) {
+            buf_sel ^= 1u;
+            stage_tile(next_k,
+                       buf_sel ? (sa + NR0*NK) : sa,
+                       buf_sel ? (sb + NR1*NK) : sb);
+        }
         threadgroup_barrier(mem_flags::mem_threadgroup);
     }
 
     threadgroup_barrier(mem_flags::mem_threadgroup);
 
-    auto tC = tensor(sc, dextents<int32_t, 2>(NR0, NR1));
-    cT.store(tC);
+    if (mma_active) {
+        auto tC = tensor(sc + h*16*NR0, dextents<int32_t, 2>(NR0, 16));
+        cT.store(tC);
+    }
 
     threadgroup_barrier(mem_flags::mem_threadgroup);
 
     for (short j = tiitg/32; j < nr1; j += 4) {
-        const int idj = ids_i32[im*args.ne21 + r1 + j];
+        const int idj = ids_i32[route_base + r1 + j];
 
         const short ide = idj % args.ne20;
         const short idt = idj / args.ne20;
@@ -9176,8 +10274,6 @@ kernel void kernel_mul_mm_id_mpp(
     }
 }
 
-
-
 typedef decltype(kernel_mul_mm_id_mpp<half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_q2_K, QK_NL, dequantize_q2_K, float, float4x4, float, float2x4>) mul_mm_id_mpp_t;
 typedef decltype(kernel_mul_mm_id_mpp<half, half4x4, simdgroup_half8x8, half, half2x4, simdgroup_half8x8, block_q2_K, QK_NL, dequantize_q2_K, half, half4x4, half, half2x4>) mul_mm_id_mpp_f16_rhs_t;
 
diff --git a/metal/sum_rows.metal b/metal/sum_rows.metal
index 8efea9b395..c77a0b509e 100644
--- a/metal/sum_rows.metal
+++ b/metal/sum_rows.metal
@@ -2,7 +2,6 @@
 
 #define FC_SUM_ROWS 1400
 
-#define OP_SUM_ROWS_NUM_SUM_ROWS 10
 #define OP_SUM_ROWS_NUM_MEAN     11
 
 struct ds4_metal_args_sum_rows {
diff --git a/metal/unary.metal b/metal/unary.metal
index 717fb350b1..497b7b9472 100644
--- a/metal/unary.metal
+++ b/metal/unary.metal
@@ -191,7 +191,7 @@ kernel void kernel_unary_impl(
         }
 
         if (FC_OP == OP_UNARY_NUM_SIGMOID) {
-            dst_ptr[i0] = (T) (1 / (1 + exp(-x)));
+            dst_ptr[i0] = (T) ds4_sigmoid_stable(x);
         }
 
         if (FC_OP == OP_UNARY_NUM_GELU) {
@@ -207,7 +207,7 @@ kernel void kernel_unary_impl(
         }
 
         if (FC_OP == OP_UNARY_NUM_SILU) {
-            dst_ptr[i0] = (T) (x / (1 + exp(-x)));
+            dst_ptr[i0] = (T) ds4_silu(x);
         }
 
         if (FC_OP == OP_UNARY_NUM_ELU) {
diff --git a/rocm/ds4_rocm_attention.cuh b/rocm/ds4_rocm_attention.cuh
index 939bb292be..a162fd1802 100644
--- a/rocm/ds4_rocm_attention.cuh
+++ b/rocm/ds4_rocm_attention.cuh
@@ -938,92 +938,6 @@ __global__ static void attention_decode_mixed_kernel(
     }
 }
 
-__global__ static void attention_indexed_mixed_scalar_kernel(
-        float *heads,
-        const float *sinks,
-        const float *q,
-        const float *raw_kv,
-        const float *comp_kv,
-        const int32_t *topk,
-        uint32_t n_tokens,
-        uint32_t pos0,
-        uint32_t n_raw,
-        uint32_t raw_cap,
-        uint32_t raw_start,
-        uint32_t n_comp,
-        uint32_t top_k,
-        uint32_t window,
-        uint32_t ratio,
-        uint32_t n_head,
-        uint32_t head_dim) {
-    const uint64_t idx = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x;
-    const uint64_t total = (uint64_t)n_tokens * n_head * head_dim;
-    if (idx >= total) return;
-    const uint32_t d = (uint32_t)(idx % head_dim);
-    const uint64_t th = idx / head_dim;
-    const uint32_t h = (uint32_t)(th % n_head);
-    const uint32_t t = (uint32_t)(th / n_head);
-    const uint32_t qpos = pos0 + t;
-    const uint32_t last_pos = pos0 + n_tokens - 1u;
-    const uint32_t first_raw_pos = last_pos + 1u - n_raw;
-    const float *qh = q + ((uint64_t)t * n_head + h) * head_dim;
-    const float scale = rsqrtf((float)head_dim);
-    const uint32_t visible = ratio ? (qpos + 1u) / ratio : n_comp;
-    float max_score = sinks[h];
-
-    for (uint32_t r = 0; r < n_raw; r++) {
-        const uint32_t kpos = first_raw_pos + r;
-        if (kpos > qpos) continue;
-        if (window != 0 && qpos - kpos >= window) continue;
-        const uint32_t row = raw_cap ? ((raw_start + r) % raw_cap) : r;
-        const float *kv = raw_kv + (uint64_t)row * head_dim;
-        float s = 0.0f;
-        for (uint32_t i = 0; i < head_dim; i++) s += qh[i] * kv[i];
-        s *= scale;
-        if (s > max_score) max_score = s;
-    }
-    for (uint32_t u = 0; u < top_k; u++) {
-        const int32_t ci = topk[(uint64_t)t * top_k + u];
-        if (ci < 0) continue;
-        const uint32_t c = (uint32_t)ci;
-        if (c >= n_comp || c >= visible) continue;
-        const float *kv = comp_kv + (uint64_t)c * head_dim;
-        float s = 0.0f;
-        for (uint32_t i = 0; i < head_dim; i++) s += qh[i] * kv[i];
-        s *= scale;
-        if (s > max_score) max_score = s;
-    }
-
-    float denom = expf(sinks[h] - max_score);
-    float acc = 0.0f;
-    for (uint32_t r = 0; r < n_raw; r++) {
-        const uint32_t kpos = first_raw_pos + r;
-        if (kpos > qpos) continue;
-        if (window != 0 && qpos - kpos >= window) continue;
-        const uint32_t row = raw_cap ? ((raw_start + r) % raw_cap) : r;
-        const float *kv = raw_kv + (uint64_t)row * head_dim;
-        float s = 0.0f;
-        for (uint32_t i = 0; i < head_dim; i++) s += qh[i] * kv[i];
-        const float w = expf(s * scale - max_score);
-        denom += w;
-        acc += w * kv[d];
-    }
-    for (uint32_t u = 0; u < top_k; u++) {
-        const int32_t ci = topk[(uint64_t)t * top_k + u];
-        if (ci < 0) continue;
-        const uint32_t c = (uint32_t)ci;
-        if (c >= n_comp || c >= visible) continue;
-        const float *kv = comp_kv + (uint64_t)c * head_dim;
-        float s = 0.0f;
-        for (uint32_t i = 0; i < head_dim; i++) s += qh[i] * kv[i];
-        s *= scale;
-        const float w = expf(s - max_score);
-        denom += w;
-        acc += w * kv[d];
-    }
-    heads[idx] = acc / denom;
-}
-
 __global__ static void attention_indexed_mixed_kernel(
         float *heads,
         const float *sinks,
@@ -1344,6 +1258,11 @@ __global__ static void attention_indexed_mixed_heads8_online_kernel(
     }
 }
 
+#if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__)
+/* gfx1151/wave32 mixed-attention core. MODE 0 handles contiguous prefill,
+ * MODE 1 indexed mixed attention, and MODE 2 the causal raw ring. The tuned
+ * MODE 1/2 vector path remains controlled by the capability-checked launch
+ * gates in ds4_rocm_attention_launch.cuh. */
 template <int MODE, uint32_t HEADS, bool F32_VEC2 = false>
 __global__ static void attention_mixed_heads16_wmma_kernel(
         float *heads,
@@ -1363,6 +1282,10 @@ __global__ static void attention_mixed_heads16_wmma_kernel(
         uint32_t ratio,
         uint32_t n_head,
         uint32_t head_dim) {
+    static_assert(MODE >= 0 && MODE <= 2,
+                  "gfx1151 WMMA attention mode is invalid");
+    static_assert(HEADS == 16u || HEADS == 32u,
+                  "gfx1151 WMMA attention requires 16 or 32 heads/workgroup");
     constexpr uint32_t KEYS = 80u;
     constexpr uint32_t DIMS = 64u;
     constexpr uint32_t TILE = 16u;
@@ -1370,6 +1293,19 @@ __global__ static void attention_mixed_heads16_wmma_kernel(
     constexpr uint32_t HEAD_TILES = HEADS / TILE;
     constexpr uint32_t KEY_TILES = KEYS / TILE;
     constexpr uint32_t WORKGROUP = HEADS * 16u;
+    constexpr size_t SHARED_BYTES =
+        (size_t)HEADS * STRIDE * sizeof(half) +
+        (size_t)KEYS * STRIDE * sizeof(half) +
+        (size_t)HEADS * STRIDE * sizeof(float) +
+        (size_t)HEADS * 3u * sizeof(float) +
+        256u * sizeof(uint32_t) +
+        DS4_ROCM_ATTENTION_INDEXED_TOPK_CAP * sizeof(uint32_t) +
+        3u * sizeof(uint32_t);
+    static_assert(KEYS % TILE == 0u && DIMS % TILE == 0u &&
+                      HEADS % TILE == 0u && WORKGROUP <= 512u,
+                  "gfx1151 WMMA attention tile geometry changed");
+    static_assert(SHARED_BYTES <= 64u * 1024u,
+                  "gfx1151 WMMA attention exceeds the validated LDS budget");
     const uint32_t t = (uint32_t)blockIdx.x;
     const uint32_t head_base = (uint32_t)blockIdx.y * HEADS;
     if (t >= n_tokens || head_dim != 512u) return;
@@ -1734,6 +1670,7 @@ __global__ static void attention_mixed_heads16_wmma_kernel(
         }
     }
 }
+#endif
 
 __global__ static void attention_static_mixed_heads8_online_kernel(
         float *heads,
diff --git a/rocm/ds4_rocm_attention_launch.cuh b/rocm/ds4_rocm_attention_launch.cuh
index 3190f8bb49..54bd54b7c6 100644
--- a/rocm/ds4_rocm_attention_launch.cuh
+++ b/rocm/ds4_rocm_attention_launch.cuh
@@ -1,4 +1,192 @@
 extern "C" int ds4_gpu_store_raw_kv_tensor(ds4_gpu_tensor *raw_cache, const ds4_gpu_tensor *kv, uint32_t raw_cap, uint32_t row, uint32_t head_dim);
+
+/* The gfx1151 WMMA attention kernels deliberately stay behind per-path
+ * opt-ins until their approximate-F16 arithmetic has a wider prompt oracle.
+ * The common rollback wins over both opt-ins.  Environment values are read on
+ * every call so an in-process A/B harness can switch arms without rebuilding
+ * or reinitializing the backend. */
+static const char *const DS4_ROCM_GFX1151_WMMA_RING_ENABLE_ENV =
+    "DS4_ROCM_ENABLE_GFX1151_PREFILL_WMMA_RING";
+static const char *const DS4_ROCM_GFX1151_WMMA_INDEXED_ENABLE_ENV =
+    "DS4_ROCM_ENABLE_GFX1151_PREFILL_WMMA_INDEXED";
+static const char *const DS4_ROCM_GFX1151_WMMA_DISABLE_ENV =
+    "DS4_ROCM_DISABLE_GFX1151_PREFILL_WMMA_ATTN";
+
+enum {
+    DS4_ROCM_ATTN_ENV_UNSET = -1,
+    DS4_ROCM_ATTN_ENV_INVALID = -2,
+};
+
+static pthread_mutex_t g_rocm_attention_wmma_notice_mu =
+    PTHREAD_MUTEX_INITIALIZER;
+static uint32_t g_rocm_attention_wmma_notice_mask;
+
+static int rocm_attention_wmma_notice_once(uint32_t bit) {
+    pthread_mutex_lock(&g_rocm_attention_wmma_notice_mu);
+    const int report = (g_rocm_attention_wmma_notice_mask & bit) == 0u;
+    g_rocm_attention_wmma_notice_mask |= bit;
+    pthread_mutex_unlock(&g_rocm_attention_wmma_notice_mu);
+    return report;
+}
+
+static int rocm_attention_env_value_eq(
+        const char *value, size_t value_len, const char *literal) {
+    const size_t literal_len = strlen(literal);
+    if (value_len != literal_len) return 0;
+    for (size_t i = 0; i < value_len; i++) {
+        if (tolower((unsigned char)value[i]) !=
+            tolower((unsigned char)literal[i])) {
+            return 0;
+        }
+    }
+    return 1;
+}
+
+/* Return -1 for unset, 0/1 for recognized booleans, and -2 for invalid.
+ * Empty is an explicit presence opt-in; conventional false spellings remain
+ * safe and invalid values fail closed. */
+static int rocm_attention_env_bool_value(
+        const char *name, uint32_t invalid_notice_bit) {
+    const char *value = name ? getenv(name) : NULL;
+    if (!value) return DS4_ROCM_ATTN_ENV_UNSET;
+    while (isspace((unsigned char)*value)) value++;
+    size_t value_len = strlen(value);
+    while (value_len != 0u &&
+           isspace((unsigned char)value[value_len - 1u])) {
+        value_len--;
+    }
+    if (value_len == 0u) return 1;
+    if (rocm_attention_env_value_eq(value, value_len, "1") ||
+        rocm_attention_env_value_eq(value, value_len, "true") ||
+        rocm_attention_env_value_eq(value, value_len, "yes") ||
+        rocm_attention_env_value_eq(value, value_len, "on")) {
+        return 1;
+    }
+    if (rocm_attention_env_value_eq(value, value_len, "0") ||
+        rocm_attention_env_value_eq(value, value_len, "false") ||
+        rocm_attention_env_value_eq(value, value_len, "no") ||
+        rocm_attention_env_value_eq(value, value_len, "off")) {
+        return 0;
+    }
+    if (rocm_attention_wmma_notice_once(invalid_notice_bit)) {
+        const size_t shown_len = value_len < 96u ? value_len : 96u;
+        fprintf(stderr,
+                DS4_GPU_LOG_PREFIX "invalid boolean environment value "
+                "%s=%.*s%s; gfx1151 WMMA attention remains disabled\n",
+                name,
+                (int)shown_len,
+                value,
+                shown_len == value_len ? "" : "...");
+    }
+    return DS4_ROCM_ATTN_ENV_INVALID;
+}
+
+/* Cache a successful device-property query per host thread and active device.
+ * This is intentionally not a process-global, unkeyed architecture flag:
+ * HIP's active device is thread-local and callers may switch devices. */
+static int rocm_attention_runtime_is_gfx1151_wave32(
+        int report_unavailable) {
+#if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__)
+    int device = -1;
+    const cudaError_t device_err = cudaGetDevice(&device);
+    if (device_err != cudaSuccess || device < 0) {
+        if (report_unavailable &&
+            rocm_attention_wmma_notice_once(1u << 3)) {
+            fprintf(stderr,
+                    DS4_GPU_LOG_PREFIX "cannot query active device for "
+                    "gfx1151 WMMA attention; using fallback: %s\n",
+                    cudaGetErrorString(device_err));
+        }
+        (void)cudaGetLastError();
+        return 0;
+    }
+
+    static thread_local int cached_device = -1;
+    static thread_local int cached_eligible = -1;
+    if (cached_device == device && cached_eligible >= 0) {
+        if (!cached_eligible && report_unavailable &&
+            rocm_attention_wmma_notice_once(1u << 5)) {
+            fprintf(stderr,
+                    DS4_GPU_LOG_PREFIX "gfx1151 WMMA attention requested on "
+                    "a cached unsupported device; using fallback\n");
+        }
+        return cached_eligible;
+    }
+
+    cudaDeviceProp prop = {};
+    const cudaError_t prop_err = cudaGetDeviceProperties(&prop, device);
+    if (prop_err != cudaSuccess) {
+        if (report_unavailable &&
+            rocm_attention_wmma_notice_once(1u << 4)) {
+            fprintf(stderr,
+                    DS4_GPU_LOG_PREFIX "cannot query device properties for "
+                    "gfx1151 WMMA attention; using fallback: %s\n",
+                    cudaGetErrorString(prop_err));
+        }
+        (void)cudaGetLastError();
+        return 0;
+    }
+
+    static const char kGfx1151[] = "gfx1151";
+    const int arch_matches =
+        strncmp(prop.gcnArchName, kGfx1151, sizeof(kGfx1151) - 1u) == 0 &&
+        (prop.gcnArchName[sizeof(kGfx1151) - 1u] == '\0' ||
+         prop.gcnArchName[sizeof(kGfx1151) - 1u] == ':');
+    cached_device = device;
+    cached_eligible = arch_matches && prop.warpSize == 32 ? 1 : 0;
+    if (!cached_eligible && report_unavailable &&
+        rocm_attention_wmma_notice_once(1u << 5)) {
+        fprintf(stderr,
+                DS4_GPU_LOG_PREFIX "gfx1151 WMMA attention requested on "
+                "unsupported device arch=%s warpSize=%d; using fallback\n",
+                prop.gcnArchName,
+                prop.warpSize);
+    }
+    return cached_eligible;
+#else
+    (void)report_unavailable;
+    return 0;
+#endif
+}
+
+static int rocm_attention_gfx1151_wmma_enabled(
+        const char *enable_env,
+        uint32_t enable_invalid_notice_bit,
+        uint32_t enabled_notice_bit) {
+    const int enable = rocm_attention_env_bool_value(
+        enable_env, enable_invalid_notice_bit);
+    if (enable == DS4_ROCM_ATTN_ENV_INVALID || enable != 1) return 0;
+    const int disable = rocm_attention_env_bool_value(
+        DS4_ROCM_GFX1151_WMMA_DISABLE_ENV, 1u << 2);
+    if (disable == DS4_ROCM_ATTN_ENV_INVALID) return 0;
+    if (disable == 1) {
+        if (enable == 1 && rocm_attention_wmma_notice_once(1u << 6)) {
+            fprintf(stderr,
+                    DS4_GPU_LOG_PREFIX "gfx1151 WMMA attention disabled by "
+                    "%s (overrides %s)\n",
+                    DS4_ROCM_GFX1151_WMMA_DISABLE_ENV,
+                    enable_env);
+        }
+        return 0;
+    }
+    if (g_quality_mode) {
+        if (rocm_attention_wmma_notice_once(1u << 7)) {
+            fprintf(stderr,
+                    DS4_GPU_LOG_PREFIX "gfx1151 WMMA attention is unavailable "
+                    "in quality mode; using fallback\n");
+        }
+        return 0;
+    }
+    if (!rocm_attention_runtime_is_gfx1151_wave32(1)) return 0;
+    if (rocm_attention_wmma_notice_once(enabled_notice_bit)) {
+        fprintf(stderr,
+                DS4_GPU_LOG_PREFIX "gfx1151 wave32 WMMA attention enabled "
+                "by %s (32 heads/workgroup, 80 keys/tile, float2 KV staging)\n",
+                enable_env);
+    }
+    return 1;
+}
+
 extern "C" int ds4_gpu_kv_fp8_store_raw_tensor(
         ds4_gpu_tensor *kv,
         ds4_gpu_tensor *raw_cache,
@@ -37,20 +225,28 @@ extern "C" int ds4_gpu_attention_noncausal_raw_batch_heads_tensor(
         uint32_t              raw_start,
         uint32_t              n_head,
         uint32_t              head_dim) {
+    uint64_t sink_bytes = 0;
+    uint64_t head_elems = 0;
+    uint64_t head_bytes = 0;
+    uint64_t raw_bytes = 0;
+    const bool sizes_ok =
+        cuda_u64_mul_checked(n_head, sizeof(float), &sink_bytes) &&
+        cuda_u64_mul3_checked(n_tokens, n_head, head_dim, &head_elems) &&
+        cuda_u64_mul_checked(head_elems, sizeof(float), &head_bytes) &&
+        cuda_u64_mul3_checked(raw_cap, head_dim, sizeof(float), &raw_bytes);
     if (!heads || !q || !raw_kv || !model_map ||
+        !sizes_ok ||
         n_tokens == 0 || n_raw == 0 || raw_cap < n_raw ||
         raw_start >= raw_cap || n_head == 0 || head_dim == 0 ||
-        sinks_offset > model_size ||
-        (uint64_t)n_head * sizeof(float) > model_size - sinks_offset ||
-        heads->bytes < (uint64_t)n_tokens * n_head * head_dim * sizeof(float) ||
-        q->bytes < (uint64_t)n_tokens * n_head * head_dim * sizeof(float) ||
-        raw_kv->bytes < (uint64_t)raw_cap * head_dim * sizeof(float)) {
+        sinks_offset > model_size || sink_bytes > model_size - sinks_offset ||
+        heads->bytes < head_bytes || q->bytes < head_bytes ||
+        raw_kv->bytes < raw_bytes) {
         return 0;
     }
     const float *sinks = (const float *)cuda_model_range_ptr(
             model_map,
             sinks_offset,
-            (uint64_t)n_head * sizeof(float),
+            sink_bytes,
             "dspark_attn_sinks");
     if (!sinks) return 0;
 
@@ -80,8 +276,8 @@ extern "C" int ds4_gpu_attention_noncausal_raw_batch_heads_tensor(
     if (verify_left > 0) {
         verify_left--;
         (void)cudaDeviceSynchronize();
-        const uint64_t qn = (uint64_t)n_tokens * n_head * head_dim;
-        const uint64_t kn = (uint64_t)raw_cap * head_dim;
+        const uint64_t qn = head_elems;
+        const uint64_t kn = raw_bytes / sizeof(float);
         std::vector<float> hq(qn), hkv(kn), hout(qn), hsink(n_head);
         (void)cudaMemcpy(hq.data(), q->ptr, qn * sizeof(float),
                          cudaMemcpyDeviceToHost);
@@ -89,7 +285,7 @@ extern "C" int ds4_gpu_attention_noncausal_raw_batch_heads_tensor(
                          cudaMemcpyDeviceToHost);
         (void)cudaMemcpy(hout.data(), heads->ptr, qn * sizeof(float),
                          cudaMemcpyDeviceToHost);
-        (void)cudaMemcpy(hsink.data(), sinks, (uint64_t)n_head * sizeof(float),
+        (void)cudaMemcpy(hsink.data(), sinks, sink_bytes,
                          cudaMemcpyDeviceToHost);
         double max_abs = 0.0;
         double max_rel = 0.0;
@@ -367,30 +563,37 @@ static int attention_decode_batch_launch(
             model_map, sinks_offset, (uint64_t)n_head * sizeof(float), "attn_sinks");
     if (!sinks) return 0;
     const int fast_window_attention = !g_quality_mode;
-    const bool use_wmma_ring =
-            ds4_rocm_gfx1151_flag("DS4_ROCM_ATTN_WMMA32_RING");
-    if (use_wmma_ring && !use_comp_mask && n_tokens > 1u &&
-        head_dim == 512u && fast_window_attention) {
-        dim3 grid(n_tokens, (n_head + 31u) / 32u, 1);
-        attention_mixed_heads16_wmma_kernel<2, 32><<<grid, 512>>>((float *)heads->ptr,
-                                                                  sinks,
-                                                                  (const float *)q->ptr,
-                                                                  (const float *)raw_kv->ptr,
-                                                                  n_comp ? (const float *)comp_kv->ptr : (const float *)raw_kv->ptr,
-                                                                  NULL,
-                                                                  n_tokens,
-                                                                  pos0,
-                                                                  n_raw,
-                                                                  raw_cap,
-                                                                  raw_start,
-                                                                  n_comp,
-                                                                  0,
-                                                                  window,
-                                                                  ratio,
-                                                                  n_head,
-                                                                  head_dim);
-        return cuda_ok(cudaGetLastError(), "attention ring wmma32 launch");
+#if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__)
+    if (!use_comp_mask && n_tokens > 1u && head_dim == 512u &&
+        n_head == 64u && (uint64_t)n_raw <= (uint64_t)pos0 + n_tokens &&
+        rocm_attention_gfx1151_wmma_enabled(
+            DS4_ROCM_GFX1151_WMMA_RING_ENABLE_ENV,
+            1u << 0,
+            1u << 8)) {
+        dim3 grid(n_tokens, 2u, 1u);
+        attention_mixed_heads16_wmma_kernel<2, 32, true>
+            <<<grid, 512>>>((float *)heads->ptr,
+                            sinks,
+                            (const float *)q->ptr,
+                            (const float *)raw_kv->ptr,
+                            n_comp ? (const float *)comp_kv->ptr
+                                   : (const float *)raw_kv->ptr,
+                            NULL,
+                            n_tokens,
+                            pos0,
+                            n_raw,
+                            raw_cap,
+                            raw_start,
+                            n_comp,
+                            0u,
+                            window,
+                            ratio,
+                            n_head,
+                            head_dim);
+        return cuda_ok(cudaGetLastError(),
+                       "attention raw-ring gfx1151 wmma32 tile80 launch");
     }
+#endif
     if (!cuda_attention_score_buffer_fits(n_comp)) {
         if (!use_comp_mask && head_dim == 512u) {
             dim3 online_grid(n_tokens, (n_head + 7u) / 8u, 1);
@@ -569,75 +772,35 @@ extern "C" int ds4_gpu_attention_indexed_mixed_batch_heads_tensor(
         head_dim == 512 &&
         top_k <= DS4_ROCM_ATTENTION_INDEXED_TOPK_CAP) {
 #if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__)
+        if (n_head == 64u && top_k == 512u &&
+            (uint64_t)n_raw <= (uint64_t)pos0 + n_tokens &&
+            rocm_attention_gfx1151_wmma_enabled(
+                DS4_ROCM_GFX1151_WMMA_INDEXED_ENABLE_ENV,
+                1u << 1,
+                1u << 9)) {
+            dim3 grid(n_tokens, 2u, 1u);
+            attention_mixed_heads16_wmma_kernel<1, 32, true>
+                <<<grid, 512>>>((float *)heads->ptr,
+                                sinks,
+                                (const float *)q->ptr,
+                                (const float *)raw_kv->ptr,
+                                (const float *)comp_kv->ptr,
+                                topk_ptr,
+                                n_tokens,
+                                pos0,
+                                n_raw,
+                                raw_cap,
+                                raw_start,
+                                n_comp,
+                                top_k,
+                                window,
+                                ratio,
+                                n_head,
+                                head_dim);
+            return cuda_ok(cudaGetLastError(),
+                           "attention indexed gfx1151 wmma32 tile80 launch");
+        }
         if (!g_quality_mode && n_head <= 64u) {
-            const bool use_wmma32 =
-                    ds4_rocm_gfx1151_flag("DS4_ROCM_ATTN_WMMA32_INDEXED");
-            if (use_wmma32) {
-                dim3 grid(n_tokens, (n_head + 31u) / 32u, 1);
-                const bool use_vec2 =
-                        ds4_rocm_gfx1151_flag("DS4_ROCM_ATTN_F32_VEC2");
-                if (use_vec2) {
-                    attention_mixed_heads16_wmma_kernel<1, 32, true><<<grid, 512>>>((float *)heads->ptr,
-                                                                            sinks,
-                                                                            (const float *)q->ptr,
-                                                                            (const float *)raw_kv->ptr,
-                                                                            (const float *)comp_kv->ptr,
-                                                                            topk_ptr,
-                                                                            n_tokens,
-                                                                            pos0,
-                                                                            n_raw,
-                                                                            raw_cap,
-                                                                            raw_start,
-                                                                            n_comp,
-                                                                            top_k,
-                                                                            window,
-                                                                            ratio,
-                                                                            n_head,
-                                                                            head_dim);
-                    return cuda_ok(cudaGetLastError(), "attention indexed wmma32 f32 vec2 launch");
-                }
-                attention_mixed_heads16_wmma_kernel<1, 32><<<grid, 512>>>((float *)heads->ptr,
-                                                                            sinks,
-                                                                            (const float *)q->ptr,
-                                                                            (const float *)raw_kv->ptr,
-                                                                            (const float *)comp_kv->ptr,
-                                                                            topk_ptr,
-                                                                            n_tokens,
-                                                                            pos0,
-                                                                            n_raw,
-                                                                            raw_cap,
-                                                                            raw_start,
-                                                                            n_comp,
-                                                                            top_k,
-                                                                            window,
-                                                                            ratio,
-                                                                            n_head,
-                                                                            head_dim);
-                return cuda_ok(cudaGetLastError(), "attention indexed wmma32 launch");
-            }
-            const char *wmma_env = getenv("DS4_ROCM_ATTN_WMMA16_INDEXED");
-            const bool use_wmma = wmma_env && wmma_env[0] != '\0' && wmma_env[0] != '0';
-            if (use_wmma) {
-                dim3 grid(n_tokens, (n_head + 15u) / 16u, 1);
-                attention_mixed_heads16_wmma_kernel<1, 16><<<grid, 256>>>((float *)heads->ptr,
-                                                                        sinks,
-                                                                        (const float *)q->ptr,
-                                                                        (const float *)raw_kv->ptr,
-                                                                        (const float *)comp_kv->ptr,
-                                                                        topk_ptr,
-                                                                        n_tokens,
-                                                                        pos0,
-                                                                        n_raw,
-                                                                        raw_cap,
-                                                                        raw_start,
-                                                                        n_comp,
-                                                                        top_k,
-                                                                        window,
-                                                                        ratio,
-                                                                        n_head,
-                                                                        head_dim);
-                return cuda_ok(cudaGetLastError(), "attention indexed wmma16 launch");
-            }
             dim3 grid(n_tokens, (n_head + 31u) / 32u, 1);
             attention_indexed_mixed_heads8_online_kernel<8, 32><<<grid, 1024>>>((float *)heads->ptr,
                                                                                 sinks,
diff --git a/rocm/ds4_rocm_current_api_compat.cuh b/rocm/ds4_rocm_current_api_compat.cuh
index d24668ce7e..5cf63a986e 100644
--- a/rocm/ds4_rocm_current_api_compat.cuh
+++ b/rocm/ds4_rocm_current_api_compat.cuh
@@ -117,6 +117,9 @@ extern "C" int ds4_gpu_preload_q4_expert_tables(
 }
 
 extern "C" void ds4_gpu_set_ssd_streaming(bool enabled) {
+    if (enabled && !g_ssd_streaming_mode) {
+        (void)ds4_gpu_release_q4_attn_q_b_f16_sidecars();
+    }
     g_ssd_streaming_mode = enabled ? 1 : 0;
     cuda_model_range_release_all();
     cuda_q8_f16_cache_release_all();
diff --git a/rocm/ds4_rocm_deepseek4_vision.cuh b/rocm/ds4_rocm_deepseek4_vision.cuh
index aeb521774b..641d48d9cb 100644
--- a/rocm/ds4_rocm_deepseek4_vision.cuh
+++ b/rocm/ds4_rocm_deepseek4_vision.cuh
@@ -236,7 +236,7 @@ extern "C" int ds4_gpu_attention_visual_mixed_batch_heads_tensor(
     if (!cuda_ok(cudaGetLastError(), "visual attention KV pack launch"))
         return 0;
 
-    const float alpha = rsqrtf((float)head_dim);
+    const float alpha = 1.0f / sqrtf((float)head_dim);
     const float beta = 0.0f;
     cublasStatus_t status = cublasSgemmStridedBatched(
             g_cublas, CUBLAS_OP_T, CUBLAS_OP_N,
diff --git a/rocm/ds4_rocm_glm.cuh b/rocm/ds4_rocm_glm.cuh
index e0bface992..86ec32b7d7 100644
--- a/rocm/ds4_rocm_glm.cuh
+++ b/rocm/ds4_rocm_glm.cuh
@@ -1142,31 +1142,6 @@ __global__ static void glm_store_indexer_k_kernel(
     }
 }
 
-__global__ static void glm_k_b_project_q8_0_kernel(
-        float *out,
-        const unsigned char *weight,
-        const float *kv_norm,
-        uint32_t n_tokens,
-        uint32_t kv_lora_dim,
-        uint32_t qk_nope,
-        uint32_t n_head,
-        uint32_t row_bytes) {
-    const uint32_t token = blockIdx.x;
-    const uint32_t head = blockIdx.y;
-    const uint32_t q = threadIdx.x + blockIdx.z * blockDim.x;
-    if (token >= n_tokens || head >= n_head || q >= qk_nope) return;
-    const float *kv = kv_norm + (uint64_t)token * kv_lora_dim;
-    float acc = 0.0f;
-    const uint32_t b = q >> 5u;
-    const uint32_t j_in_block = q & 31u;
-    for (uint32_t j = 0; j < kv_lora_dim; j++) {
-        const unsigned char *row = weight + ((uint64_t)head * kv_lora_dim + j) * row_bytes;
-        const unsigned char *blk = row + (uint64_t)b * 34u;
-        acc += q8_0_scale_scalar(blk) * (float)((const int8_t *)(blk + 2u))[j_in_block] * kv[j];
-    }
-    out[((uint64_t)token * n_head + head) * qk_nope + q] = acc;
-}
-
 __global__ static void glm_k_b_project_q8_0_head_kernel(
         float *out,
         const unsigned char *weight,
@@ -1201,46 +1176,6 @@ __global__ static void glm_k_b_project_q8_0_head_kernel(
     }
 }
 
-__global__ static void glm_q8_project_rows_kernel(
-        float *out,
-        const unsigned char *weight,
-        const float *x,
-        uint32_t n_tokens,
-        uint32_t n_head,
-        uint32_t in_dim,
-        uint32_t out_dim,
-        uint32_t x_stride,
-        uint32_t x_head_stride,
-        uint32_t row_bytes) {
-    const uint32_t out_row = blockIdx.x;
-    const uint32_t token = blockIdx.y;
-    if (token >= n_tokens || out_row >= n_head * out_dim) return;
-    const uint32_t head = out_row / out_dim;
-    const uint32_t d = out_row - head * out_dim;
-    const float *xr = x + (uint64_t)token * x_stride + (uint64_t)head * x_head_stride;
-    const unsigned char *row = weight + ((uint64_t)head * out_dim + d) * row_bytes;
-    float acc = 0.0f;
-    const uint32_t blocks = (in_dim + 31u) >> 5u;
-    for (uint32_t b = threadIdx.x; b < blocks; b += blockDim.x) {
-        const uint32_t base = b << 5u;
-        const uint32_t count = min(32u, in_dim - base);
-        const unsigned char *blk = row + (uint64_t)b * 34u;
-        const float scale = q8_0_scale_scalar(blk);
-        const int8_t *qs = (const int8_t *)(blk + 2u);
-        for (uint32_t i = 0; i < count; i++) acc += scale * (float)qs[i] * xr[base + i];
-    }
-    __shared__ float partial[256];
-    partial[threadIdx.x] = acc;
-    __syncthreads();
-    for (uint32_t stride = blockDim.x >> 1u; stride > 0u; stride >>= 1u) {
-        if (threadIdx.x < stride) partial[threadIdx.x] += partial[threadIdx.x + stride];
-        __syncthreads();
-    }
-    if (threadIdx.x == 0u) {
-        out[((uint64_t)token * n_head + head) * out_dim + d] = partial[0];
-    }
-}
-
 __global__ static void glm_q8_project_head_kernel(
         float *out,
         const unsigned char *weight,
diff --git a/rocm/ds4_rocm_hipblaslt.cuh b/rocm/ds4_rocm_hipblaslt.cuh
deleted file mode 100644
index ce5c15a864..0000000000
--- a/rocm/ds4_rocm_hipblaslt.cuh
+++ /dev/null
@@ -1,137 +0,0 @@
-/* HIP-only hipBLASLt state and helpers.
- * Included from ds4_cuda.cu under __HIP_PLATFORM_AMD__ to keep ROCm
- * planning/cache code out of the CUDA host runtime body. */
-
-static hipblasLtHandle_t g_hipblaslt;
-static int g_hipblaslt_ready;
-struct cuda_hipblaslt_gemm_plan {
-    uint32_t out_dim;
-    uint32_t n_tok;
-    uint32_t in_dim;
-    hipblasLtMatmulDesc_t desc;
-    hipblasLtMatrixLayout_t a_desc;
-    hipblasLtMatrixLayout_t b_desc;
-    hipblasLtMatrixLayout_t c_desc;
-    hipblasLtMatrixLayout_t d_desc;
-    hipblasLtMatmulAlgo_t algo;
-};
-static std::vector<cuda_hipblaslt_gemm_plan> g_hipblaslt_gemm_plans;
-
-static void hipblaslt_gemm_plan_clear(void) {
-    for (size_t i = 0; i < g_hipblaslt_gemm_plans.size(); i++) {
-        cuda_hipblaslt_gemm_plan &p = g_hipblaslt_gemm_plans[i];
-        if (p.d_desc) (void)hipblasLtMatrixLayoutDestroy(p.d_desc);
-        if (p.c_desc) (void)hipblasLtMatrixLayoutDestroy(p.c_desc);
-        if (p.b_desc) (void)hipblasLtMatrixLayoutDestroy(p.b_desc);
-        if (p.a_desc) (void)hipblasLtMatrixLayoutDestroy(p.a_desc);
-        if (p.desc) (void)hipblasLtMatmulDescDestroy(p.desc);
-    }
-    g_hipblaslt_gemm_plans.clear();
-}
-
-static int hipblaslt_ok(hipblasStatus_t st, const char *what) {
-    if (st == HIPBLAS_STATUS_SUCCESS) return 1;
-    fprintf(stderr, "ds4: hipBLASLt %s failed: status %d\n", what, (int)st);
-    return 0;
-}
-
-static cuda_hipblaslt_gemm_plan *hipblaslt_gemm_plan_get(
-        uint32_t out_dim,
-        uint32_t n_tok,
-        uint32_t in_dim,
-        const char *label) {
-    for (size_t i = 0; i < g_hipblaslt_gemm_plans.size(); i++) {
-        cuda_hipblaslt_gemm_plan &p = g_hipblaslt_gemm_plans[i];
-        if (p.out_dim == out_dim && p.n_tok == n_tok && p.in_dim == in_dim) return &p;
-    }
-
-    hipblasLtMatmulDesc_t desc = NULL;
-    hipblasLtMatrixLayout_t a_desc = NULL, b_desc = NULL, c_desc = NULL, d_desc = NULL;
-    hipblasLtMatmulPreference_t pref = NULL;
-    hipblasLtMatmulHeuristicResult_t heur[8];
-    int returned = 0;
-    int ok = 0;
-    do {
-        if (!hipblaslt_ok(hipblasLtMatmulDescCreate(&desc, HIPBLAS_COMPUTE_32F, HIP_R_32F),
-                          "matmul desc create")) break;
-        hipblasOperation_t op_a = HIPBLAS_OP_T;
-        hipblasOperation_t op_b = HIPBLAS_OP_N;
-        if (!hipblaslt_ok(hipblasLtMatmulDescSetAttribute(desc, HIPBLASLT_MATMUL_DESC_TRANSA,
-                                                          &op_a, sizeof(op_a)),
-                          "set transA")) break;
-        if (!hipblaslt_ok(hipblasLtMatmulDescSetAttribute(desc, HIPBLASLT_MATMUL_DESC_TRANSB,
-                                                          &op_b, sizeof(op_b)),
-                          "set transB")) break;
-        if (!hipblaslt_ok(hipblasLtMatrixLayoutCreate(&a_desc, HIP_R_16F, in_dim, out_dim, in_dim),
-                          "A layout create")) break;
-        if (!hipblaslt_ok(hipblasLtMatrixLayoutCreate(&b_desc, HIP_R_16F, in_dim, n_tok, in_dim),
-                          "B layout create")) break;
-        if (!hipblaslt_ok(hipblasLtMatrixLayoutCreate(&c_desc, HIP_R_16F, out_dim, n_tok, out_dim),
-                          "C layout create")) break;
-        if (!hipblaslt_ok(hipblasLtMatrixLayoutCreate(&d_desc, HIP_R_16F, out_dim, n_tok, out_dim),
-                          "D layout create")) break;
-        if (!hipblaslt_ok(hipblasLtMatmulPreferenceCreate(&pref), "preference create")) break;
-        const size_t max_workspace = 0;
-        if (!hipblaslt_ok(hipblasLtMatmulPreferenceSetAttribute(
-                                  pref, HIPBLASLT_MATMUL_PREF_MAX_WORKSPACE_BYTES,
-                                  &max_workspace, sizeof(max_workspace)),
-                          "set max workspace")) break;
-        if (!hipblaslt_ok(hipblasLtMatmulAlgoGetHeuristic(g_hipblaslt, desc,
-                                                          a_desc, b_desc, c_desc, d_desc,
-                                                          pref, 8, heur, &returned),
-                          "algo heuristic")) break;
-        if (returned <= 0 || heur[0].state != HIPBLAS_STATUS_SUCCESS) {
-            fprintf(stderr, "ds4: hipBLASLt no algo for %s m=%u n=%u k=%u\n",
-                    label ? label : "gemm", out_dim, n_tok, in_dim);
-            break;
-        }
-        ok = 1;
-    } while (0);
-    if (pref) (void)hipblasLtMatmulPreferenceDestroy(pref);
-    if (!ok) {
-        if (d_desc) (void)hipblasLtMatrixLayoutDestroy(d_desc);
-        if (c_desc) (void)hipblasLtMatrixLayoutDestroy(c_desc);
-        if (b_desc) (void)hipblasLtMatrixLayoutDestroy(b_desc);
-        if (a_desc) (void)hipblasLtMatrixLayoutDestroy(a_desc);
-        if (desc) (void)hipblasLtMatmulDescDestroy(desc);
-        return NULL;
-    }
-
-    cuda_hipblaslt_gemm_plan p;
-    p.out_dim = out_dim;
-    p.n_tok = n_tok;
-    p.in_dim = in_dim;
-    p.desc = desc;
-    p.a_desc = a_desc;
-    p.b_desc = b_desc;
-    p.c_desc = c_desc;
-    p.d_desc = d_desc;
-    p.algo = heur[0].algo;
-    g_hipblaslt_gemm_plans.push_back(p);
-    return &g_hipblaslt_gemm_plans.back();
-}
-
-static int hipblaslt_gemm_tn_f16_out_f16(
-        __half *out,
-        const __half *w_rowmajor_out_in,
-        const __half *x_rowmajor_tok_in,
-        uint32_t out_dim,
-        uint32_t n_tok,
-        uint32_t in_dim,
-        const char *label) {
-    if (!g_hipblaslt_ready || !out || !w_rowmajor_out_in || !x_rowmajor_tok_in ||
-        out_dim == 0 || n_tok == 0 || in_dim == 0) return 0;
-    cuda_hipblaslt_gemm_plan *p = hipblaslt_gemm_plan_get(out_dim, n_tok, in_dim, label);
-    if (!p) return 0;
-    const float alpha = 1.0f;
-    const float beta = 0.0f;
-    return hipblaslt_ok(hipblasLtMatmul(g_hipblaslt, p->desc, &alpha,
-                                        w_rowmajor_out_in, p->a_desc,
-                                        x_rowmajor_tok_in, p->b_desc,
-                                        &beta,
-                                        out, p->c_desc,
-                                        out, p->d_desc,
-                                        &p->algo,
-                                        NULL, 0, 0),
-                        label ? label : "gemm");
-}
diff --git a/rocm/ds4_rocm_indexer.cuh b/rocm/ds4_rocm_indexer.cuh
index 173ee99956..7e1e0d940c 100644
--- a/rocm/ds4_rocm_indexer.cuh
+++ b/rocm/ds4_rocm_indexer.cuh
@@ -336,7 +336,8 @@ __global__ static void indexer_top1_rows_kernel(
     indices[tid] = best_index;
     __syncthreads();
 
-    for (uint32_t stride = THREADS / 2u; stride > 0u; stride >>= 1u) {
+#pragma unroll
+    for (uint32_t stride = THREADS / 2; stride > 0u; stride >>= 1u) {
         if (tid < stride &&
             topk_score_better(values[tid + stride], indices[tid + stride],
                               values[tid], indices[tid])) {
diff --git a/rocm/ds4_rocm_matmul.cuh b/rocm/ds4_rocm_matmul.cuh
index 19c0a26598..c90b1b5971 100644
--- a/rocm/ds4_rocm_matmul.cuh
+++ b/rocm/ds4_rocm_matmul.cuh
@@ -51,6 +51,7 @@ __global__ static void matmul_f16_smallm_wmma_kernel(
     rocwmma::fill_fragment(c, 0.0f);
 
     for (uint32_t k0 = 0; k0 < k; k0 += 16u) {
+#pragma unroll
         for (uint32_t p = tid; p < BM * 8u; p += WAVES * 32u) {
             const uint32_t j = p * 2u;
             const uint32_t kk = j & 15u;
@@ -59,6 +60,7 @@ __global__ static void matmul_f16_smallm_wmma_kernel(
                     *reinterpret_cast<const uint32_t *>(
                             w + (k0 + kk) + (uint64_t)(m0 + row) * k);
         }
+#pragma unroll
         for (uint32_t p = tid; p < 8u * BN; p += WAVES * 32u) {
             const uint32_t j = p * 2u;
             const uint32_t kk = j & 15u;
@@ -80,20 +82,48 @@ __global__ static void matmul_f16_smallm_wmma_kernel(
 
 __global__ static void matmul_f16_tinym24_wmma_kernel(
         float *out, const half *w, const half *x) {
-    constexpr uint32_t M=24u, N=2048u, K=16384u, BM=32u, BN=64u, WAVES=8u;
-    __shared__ half sa[BM*16u]; __shared__ half sb[16u*BN]; __shared__ float sc[BM*BN];
-    const uint32_t tid=threadIdx.x, wave=tid>>5u, wm=wave&1u, wn=wave>>1u, n0=blockIdx.x*BN;
-    using fa_t=rocwmma::fragment<rocwmma::matrix_a,16,16,16,half,rocwmma::col_major>;
-    using fb_t=rocwmma::fragment<rocwmma::matrix_b,16,16,16,half,rocwmma::col_major>;
-    using fc_t=rocwmma::fragment<rocwmma::accumulator,16,16,16,float>;
-    fa_t a; fb_t b; fc_t c; rocwmma::fill_fragment(c,0.0f);
-    for (uint32_t k0=0;k0<K;k0+=16u) {
-        for (uint32_t j=tid;j<BM*16u;j+=WAVES*32u) { const uint32_t r=j&31u, k=j>>5u; sa[j]=r<M?w[(uint64_t)r*K+k0+k]:__float2half(0.0f); }
-        for (uint32_t p=tid;p<8u*BN;p+=WAVES*32u) { const uint32_t j=p*2u,k=j&15u,n=j>>4u; *reinterpret_cast<uint32_t*>(sb+j)=*reinterpret_cast<const uint32_t*>(x+k0+k+(uint64_t)(n0+n)*K); }
-        __syncthreads(); rocwmma::load_matrix_sync(a,sa+wm*16u,BM); rocwmma::load_matrix_sync(b,sb+wn*256u,16u); rocwmma::mma_sync(c,a,b,c); __syncthreads();
+    constexpr uint32_t M = 24u, N = 2048u, K = 16384u;
+    constexpr uint32_t BM = 32u, BN = 64u, WAVES = 8u;
+    __shared__ half sa[BM * 16u];
+    __shared__ half sb[16u * BN];
+    __shared__ float sc[BM * BN];
+    const uint32_t tid = threadIdx.x, wave = tid >> 5u;
+    const uint32_t wm = wave & 1u, wn = wave >> 1u;
+    const uint32_t n0 = blockIdx.x * BN;
+    using fa_t = rocwmma::fragment<rocwmma::matrix_a, 16, 16, 16, half, rocwmma::col_major>;
+    using fb_t = rocwmma::fragment<rocwmma::matrix_b, 16, 16, 16, half, rocwmma::col_major>;
+    using fc_t = rocwmma::fragment<rocwmma::accumulator, 16, 16, 16, float>;
+    fa_t a;
+    fb_t b;
+    fc_t c;
+    rocwmma::fill_fragment(c, 0.0f);
+#pragma unroll
+    for (uint32_t k0 = 0; k0 < K; k0 += 16u) {
+#pragma unroll
+        for (uint32_t j = tid; j < BM * 16u; j += WAVES * 32u) {
+            const uint32_t r = j & 31u;
+            const uint32_t k = j >> 5u;
+            sa[j] = r < M ? w[(uint64_t)r * K + k0 + k] : __float2half(0.0f);
+        }
+#pragma unroll
+        for (uint32_t p = tid; p < 8u * BN; p += WAVES * 32u) {
+            const uint32_t j = p * 2u, k = j & 15u, n = j >> 4u;
+            *reinterpret_cast<uint32_t*>(sb + j) = *reinterpret_cast<const uint32_t*>(x + k0 + k + (uint64_t)(n0 + n) * K);
+        }
+        __syncthreads();
+        rocwmma::load_matrix_sync(a, sa + wm * 16u, BM);
+        rocwmma::load_matrix_sync(b, sb + wn * 256u, 16u);
+        rocwmma::mma_sync(c, a, b, c);
+        __syncthreads();
+    }
+    rocwmma::store_matrix_sync(sc + wm * 16u + wn * 16u * BM, c,
+    BM, rocwmma::mem_col_major);
+    __syncthreads();
+#pragma unroll
+    for (uint32_t i = tid; i < M * BN; i += WAVES * 32u) {
+        const uint32_t n = i / M, r = i - n * M;
+        out[r + (uint64_t)(n0 + n) * M] = sc[r + n * BM];
     }
-    rocwmma::store_matrix_sync(sc+wm*16u+wn*16u*BM,c,BM,rocwmma::mem_col_major); __syncthreads();
-    for (uint32_t i=tid;i<M*BN;i+=WAVES*32u) { const uint32_t n=i/M,r=i-n*M; out[r+(uint64_t)(n0+n)*M]=sc[r+n*BM]; }
 }
 
 template <uint32_t BT>
diff --git a/rocm/ds4_rocm_moe.cuh b/rocm/ds4_rocm_moe.cuh
index 57f20218f8..ce86b0e15c 100644
--- a/rocm/ds4_rocm_moe.cuh
+++ b/rocm/ds4_rocm_moe.cuh
@@ -228,45 +228,6 @@ __device__ static void dev_dot_iq2_xxs_q8_K_block4(
     for (uint32_t p = 0; p < n; p++) acc[p] += 0.125f * xd * ys[p]->d * (float)bsum[p];
 }
 
-__device__ static DS4_ROCM_UNUSED void dev_dot_iq2_xxs_q8_K_block8(
-        const cuda_block_iq2_xxs *x,
-        const cuda_block_q8_K *y0,
-        const cuda_block_q8_K *y1,
-        const cuda_block_q8_K *y2,
-        const cuda_block_q8_K *y3,
-        const cuda_block_q8_K *y4,
-        const cuda_block_q8_K *y5,
-        const cuda_block_q8_K *y6,
-        const cuda_block_q8_K *y7,
-        uint32_t n,
-        float acc[8]) {
-    const float xd = dev_f16_to_f32(x->d);
-    const uint16_t *q2 = x->qs;
-    int32_t bsum[8] = {0, 0, 0, 0, 0, 0, 0, 0};
-    const int8_t *q8[8] = {
-        y0 ? y0->qs : NULL, y1 ? y1->qs : NULL, y2 ? y2->qs : NULL, y3 ? y3->qs : NULL,
-        y4 ? y4->qs : NULL, y5 ? y5->qs : NULL, y6 ? y6->qs : NULL, y7 ? y7->qs : NULL,
-    };
-    for (int ib32 = 0; ib32 < CUDA_QK_K / 32; ib32++) {
-        const uint32_t aux0 = (uint32_t)q2[0] | ((uint32_t)q2[1] << 16);
-        const uint32_t aux1 = (uint32_t)q2[2] | ((uint32_t)q2[3] << 16);
-        q2 += 4;
-        const uint32_t ls = 2u * (aux1 >> 28) + 1u;
-        const uint8_t a0 = (uint8_t)(aux0 & 0xffu);
-        const uint8_t a1 = (uint8_t)((aux0 >> 8) & 0xffu);
-        const uint8_t a2 = (uint8_t)((aux0 >> 16) & 0xffu);
-        const uint8_t a3 = (uint8_t)((aux0 >> 24) & 0xffu);
-        for (uint32_t p = 0; p < n; p++) {
-            int32_t sumi = 0;
-            sumi += dev_dot_iq2_pair_16(a0, (aux1 >> 0) & 127u, a1, (aux1 >> 7) & 127u, q8[p] + ib32 * 32);
-            sumi += dev_dot_iq2_pair_16(a2, (aux1 >> 14) & 127u, a3, (aux1 >> 21) & 127u, q8[p] + ib32 * 32 + 16);
-            bsum[p] += sumi * (int32_t)ls;
-        }
-    }
-    const cuda_block_q8_K *ys[8] = { y0, y1, y2, y3, y4, y5, y6, y7 };
-    for (uint32_t p = 0; p < n; p++) acc[p] += 0.125f * xd * ys[p]->d * (float)bsum[p];
-}
-
 __device__ static void dev_q4_K_get_scale_min(
         uint32_t j,
         const uint8_t *scales,
@@ -354,30 +315,6 @@ __device__ __forceinline__ static void dev_mxfp4_unpack2x4(
     *high = (int32_t)dev_mxfp4_unpack4(packed >> 4u);
 }
 
-/* One q8_K chunk covers eight consecutive 32-value MXFP4 blocks.  MXFP4
- * stores the first 16 values in the low nibbles and the second 16 in the
- * high nibbles, rather than interleaving them. */
-__device__ static float dev_dot_mxfp4_q8_K_block(
-        const cuda_block_mxfp4 *x8,
-        const cuda_block_q8_K *y) {
-    float chunk = 0.0f;
-    #pragma unroll
-    for (uint32_t sb = 0; sb < 8u; sb++) {
-        const cuda_block_mxfp4 *x = x8 + sb;
-        const int8_t *q8 = y->qs + sb * 32u;
-        int32_t bsum = 0;
-        #pragma unroll
-        for (uint32_t j = 0; j < 16u; j += 4u) {
-            int32_t wlo, whi;
-            dev_mxfp4_unpack2x4(x->qs + j, &wlo, &whi);
-            bsum = __dp4a(wlo, *(const int32_t *)(q8 + j), bsum);
-            bsum = __dp4a(whi, *(const int32_t *)(q8 + 16u + j), bsum);
-        }
-        chunk += dev_e8m0_to_f32(x->e) * (float)bsum;
-    }
-    return 0.5f * y->d * chunk;
-}
-
 /* Split one 32-value MXFP4 block across a pair of lanes.  A wave therefore
  * reads 16 consecutive 17-byte blocks instead of having each quarter-wave
  * lane jump by a full 136-byte Q8_K chunk.  This mirrors the coalesced Metal
@@ -750,18 +687,20 @@ __device__ static void dev_dot_q2_K_q8_K_block8(
 }
 
 __device__ static float half_warp_sum_f32(float v, uint32_t lane16) {
-    uint32_t mask = 0xffffu << (threadIdx.x & 16u);
+    const uint32_t wave_lane = threadIdx.x & (warpSize - 1u);
+    const MASK_T mask = static_cast<MASK_T>(0xffffu) << (wave_lane & ~15u);
     for (int offset = 8; offset > 0; offset >>= 1) {
-        v += __shfl_down_sync(static_cast<MASK_T>(mask), v, offset, 16);
+        v += __shfl_down_sync(mask, v, offset, 16);
     }
     (void)lane16;
     return v;
 }
 
 __device__ static float quarter_warp_sum_f32(float v, uint32_t lane8) {
-    uint32_t mask = 0xffu << (threadIdx.x & 24u);
+    const uint32_t wave_lane = threadIdx.x & (warpSize - 1u);
+    const MASK_T mask = static_cast<MASK_T>(0xffu) << (wave_lane & ~7u);
     for (int offset = 4; offset > 0; offset >>= 1) {
-        v += __shfl_down_sync(static_cast<MASK_T>(mask), v, offset, 8);
+        v += __shfl_down_sync(mask, v, offset, 8);
     }
     (void)lane8;
     return v;
@@ -816,160 +755,138 @@ __global__ static void q8_K_quantize_kernel(cuda_block_q8_K *out, const float *x
     if (tid == 0) yb->d = 1.0f / iscale_s;
 }
 
-__global__ static DS4_ROCM_UNUSED void moe_gate_up_mid_kernel(
-        float *gate_out,
-        float *up_out,
-        float *mid_out,
-        const char *gate_base,
-        const char *up_base,
-        const cuda_block_q8_K *xq,
-        const int32_t *selected,
-        const float *weights,
-        uint64_t gate_expert_bytes,
-        uint64_t gate_row_bytes,
-        uint32_t xq_blocks,
-        uint32_t expert_mid_dim,
-        uint32_t n_expert,
-        float clamp) {
-    uint32_t row = blockIdx.x;
-    uint32_t pair = blockIdx.y;
-    if (row >= expert_mid_dim) return;
-    uint32_t tok = pair / n_expert;
-    uint32_t slot = pair - tok * n_expert;
-    int32_t expert_i = selected[(uint64_t)tok * n_expert + slot];
-    if (expert_i < 0) expert_i = 0;
-    uint32_t expert = (uint32_t)expert_i;
-    const cuda_block_iq2_xxs *gr = (const cuda_block_iq2_xxs *)(gate_base + (uint64_t)expert * gate_expert_bytes + (uint64_t)row * gate_row_bytes);
-    const cuda_block_iq2_xxs *ur = (const cuda_block_iq2_xxs *)(up_base + (uint64_t)expert * gate_expert_bytes + (uint64_t)row * gate_row_bytes);
-    const cuda_block_q8_K *xqb = xq + (uint64_t)tok * xq_blocks;
-    float gate = 0.0f;
-    float up = 0.0f;
-    for (uint32_t b = threadIdx.x; b < xq_blocks; b += blockDim.x) {
-        gate += dev_dot_iq2_xxs_q8_K_block(gr + b, xqb + b);
-        up += dev_dot_iq2_xxs_q8_K_block(ur + b, xqb + b);
-    }
-    __shared__ float partial_gate[256];
-    __shared__ float partial_up[256];
-    partial_gate[threadIdx.x] = gate;
-    partial_up[threadIdx.x] = up;
-    __syncthreads();
-    for (uint32_t stride = blockDim.x >> 1; stride > 0; stride >>= 1) {
-        if (threadIdx.x < stride) {
-            partial_gate[threadIdx.x] += partial_gate[threadIdx.x + stride];
-            partial_up[threadIdx.x] += partial_up[threadIdx.x + stride];
-        }
-        __syncthreads();
-    }
-    if (threadIdx.x == 0) {
-        gate = partial_gate[0];
-        up = partial_up[0];
-        if (clamp > 1.0e-6f) {
-            if (gate > clamp) gate = clamp;
-            if (up > clamp) up = clamp;
-            if (up < -clamp) up = -clamp;
+/* Wave32 Q8_K activation quantizer for the Q4 prefill path.
+ *
+ * A 256-thread workgroup quantizes up to eight independent 256-value blocks,
+ * one per wave.  Each lane owns the eight values lane + 32*k.  Visiting those
+ * values in three-bit reversed order, then reducing lanes with strict-greater
+ * shuffle comparisons, reproduces q8_K_quantize_kernel's complete 256-thread
+ * reduction tree including its tie behavior.  The scale, lrintf conversion,
+ * clamp and 16-value bsums are otherwise the canonical operations above.  No
+ * cross-wave communication or LDS is used.
+ *
+ * Dispatch is deliberately restricted to a runtime-proven wave32 target in
+ * ds4_rocm_q4.cuh.  Keep the device guard as a second fail-closed boundary in
+ * case a future caller bypasses that host policy. */
+enum {
+    ROCM_Q8_K_WAVE32_WIDTH = 32u,
+    ROCM_Q8_K_WAVE32_BLOCK_THREADS = 256u,
+    ROCM_Q8_K_WAVE32_WAVES_PER_BLOCK =
+        ROCM_Q8_K_WAVE32_BLOCK_THREADS / ROCM_Q8_K_WAVE32_WIDTH,
+};
+
+__global__ static void q8_K_quantize_wave32_kernel(
+        cuda_block_q8_K *out,
+        const float *x,
+        uint32_t in_dim,
+        uint32_t n_rows) {
+    if (warpSize != ROCM_Q8_K_WAVE32_WIDTH) return;
+
+    const uint32_t lane = threadIdx.x & (ROCM_Q8_K_WAVE32_WIDTH - 1u);
+    const uint32_t wave = threadIdx.x / ROCM_Q8_K_WAVE32_WIDTH;
+    const uint32_t blocks_per_row = in_dim / CUDA_QK_K;
+    const uint32_t b = blockIdx.x * ROCM_Q8_K_WAVE32_WAVES_PER_BLOCK + wave;
+    const uint32_t row = blockIdx.y;
+    if (row >= n_rows || b >= blocks_per_row) return;
+
+    const float *xr = x + (uint64_t)row * in_dim +
+                      (uint64_t)b * CUDA_QK_K;
+    cuda_block_q8_K *yb = out + (uint64_t)row * blocks_per_row + b;
+
+    float max_value = xr[lane];
+    float max_abs = fabsf(max_value);
+    #pragma unroll
+    for (uint32_t order = 1u;
+         order < CUDA_QK_K / ROCM_Q8_K_WAVE32_WIDTH; order++) {
+        /* q8_K_quantize_kernel compares strides 128, 64 and 32 before
+         * crossing lanes.  Bit-reversing the local k bits gives the same
+         * priority order: 0,4,2,6,1,5,3,7. */
+        const uint32_t k = ((order & 1u) << 2u) |
+                           (order & 2u) |
+                          ((order & 4u) >> 2u);
+        const uint32_t index = lane + k * ROCM_Q8_K_WAVE32_WIDTH;
+        const float value = xr[index];
+        const float value_abs = fabsf(value);
+        if (value_abs > max_abs) {
+            max_abs = value_abs;
+            max_value = value;
+        }
+    }
+
+    for (uint32_t offset = ROCM_Q8_K_WAVE32_WIDTH / 2u;
+         offset != 0u; offset >>= 1u) {
+#if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__)
+        const float other_abs = __shfl_down(max_abs, offset, 32);
+        const float other_value = __shfl_down(max_value, offset, 32);
+#else
+        const float other_abs = __shfl_down_sync(
+            FULL_WARP_MASK, max_abs, offset, 32);
+        const float other_value = __shfl_down_sync(
+            FULL_WARP_MASK, max_value, offset, 32);
+#endif
+        if (lane + offset < ROCM_Q8_K_WAVE32_WIDTH &&
+            other_abs > max_abs) {
+            max_abs = other_abs;
+            max_value = other_value;
         }
-        const uint64_t off = (uint64_t)pair * expert_mid_dim + row;
-        gate_out[off] = gate;
-        up_out[off] = up;
-        mid_out[off] = (gate / (1.0f + expf(-gate))) * up * weights[(uint64_t)tok * n_expert + slot];
     }
-}
 
-__global__ static DS4_ROCM_UNUSED void moe_gate_up_mid_warp8_kernel(
-        float *gate_out,
-        float *up_out,
-        float *mid_out,
-        const char *gate_base,
-        const char *up_base,
-        const cuda_block_q8_K *xq,
-        const int32_t *selected,
-        const float *weights,
-        uint64_t gate_expert_bytes,
-        uint64_t gate_row_bytes,
-        uint32_t xq_blocks,
-        uint32_t expert_mid_dim,
-        uint32_t n_expert,
-        float clamp) {
-    uint32_t lane = threadIdx.x & 31u;
-    uint32_t warp = threadIdx.x >> 5u;
-    uint32_t row = blockIdx.x * 8u + warp;
-    uint32_t pair = blockIdx.y;
-    if (row >= expert_mid_dim) return;
-    uint32_t tok = pair / n_expert;
-    uint32_t slot = pair - tok * n_expert;
-    int32_t expert_i = selected[(uint64_t)tok * n_expert + slot];
-    if (expert_i < 0) expert_i = 0;
-    uint32_t expert = (uint32_t)expert_i;
-    const cuda_block_iq2_xxs *gr = (const cuda_block_iq2_xxs *)(gate_base + (uint64_t)expert * gate_expert_bytes + (uint64_t)row * gate_row_bytes);
-    const cuda_block_iq2_xxs *ur = (const cuda_block_iq2_xxs *)(up_base + (uint64_t)expert * gate_expert_bytes + (uint64_t)row * gate_row_bytes);
-    const cuda_block_q8_K *xqb = xq + (uint64_t)tok * xq_blocks;
-    float gate = 0.0f;
-    float up = 0.0f;
-    for (uint32_t b = lane; b < xq_blocks; b += 32u) {
-        gate += dev_dot_iq2_xxs_q8_K_block(gr + b, xqb + b);
-        up += dev_dot_iq2_xxs_q8_K_block(ur + b, xqb + b);
-    }
-    gate = warp_sum_f32(gate);
-    up = warp_sum_f32(up);
-    if (lane == 0) {
-        if (clamp > 1.0e-6f) {
-            if (gate > clamp) gate = clamp;
-            if (up > clamp) up = clamp;
-            if (up < -clamp) up = -clamp;
+#if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__)
+    const float amax = __shfl(max_abs, 0, 32);
+    const float signed_max = __shfl(max_value, 0, 32);
+#else
+    const float amax = __shfl_sync(FULL_WARP_MASK, max_abs, 0, 32);
+    const float signed_max = __shfl_sync(
+        FULL_WARP_MASK, max_value, 0, 32);
+#endif
+    if (amax == 0.0f) {
+        #pragma unroll
+        for (uint32_t k = 0u;
+             k < CUDA_QK_K / ROCM_Q8_K_WAVE32_WIDTH; k++) {
+            yb->qs[lane + k * ROCM_Q8_K_WAVE32_WIDTH] = 0;
         }
-        const uint64_t off = (uint64_t)pair * expert_mid_dim + row;
-        gate_out[off] = gate;
-        up_out[off] = up;
-        mid_out[off] = (gate / (1.0f + expf(-gate))) * up * weights[(uint64_t)tok * n_expert + slot];
+        if (lane < CUDA_QK_K / 16u) yb->bsums[lane] = 0;
+        if (lane == 0u) yb->d = 0.0f;
+        return;
     }
-}
 
-__global__ static DS4_ROCM_UNUSED void moe_gate_up_mid_hwarp16_kernel(
-        float *gate_out,
-        float *up_out,
-        float *mid_out,
-        const char *gate_base,
-        const char *up_base,
-        const cuda_block_q8_K *xq,
-        const int32_t *selected,
-        const float *weights,
-        uint64_t gate_expert_bytes,
-        uint64_t gate_row_bytes,
-        uint32_t xq_blocks,
-        uint32_t expert_mid_dim,
-        uint32_t n_expert,
-        float clamp) {
-    uint32_t lane = threadIdx.x & 15u;
-    uint32_t row = blockIdx.x * 16u + (threadIdx.x >> 4u);
-    uint32_t pair = blockIdx.y;
-    if (row >= expert_mid_dim) return;
-    uint32_t tok = pair / n_expert;
-    uint32_t slot = pair - tok * n_expert;
-    int32_t expert_i = selected[(uint64_t)tok * n_expert + slot];
-    if (expert_i < 0) expert_i = 0;
-    uint32_t expert = (uint32_t)expert_i;
-    const cuda_block_iq2_xxs *gr = (const cuda_block_iq2_xxs *)(gate_base + (uint64_t)expert * gate_expert_bytes + (uint64_t)row * gate_row_bytes);
-    const cuda_block_iq2_xxs *ur = (const cuda_block_iq2_xxs *)(up_base + (uint64_t)expert * gate_expert_bytes + (uint64_t)row * gate_row_bytes);
-    const cuda_block_q8_K *xqb = xq + (uint64_t)tok * xq_blocks;
-    float gate = 0.0f;
-    float up = 0.0f;
-    for (uint32_t b = lane; b < xq_blocks; b += 16u) {
-        gate += dev_dot_iq2_xxs_q8_K_block(gr + b, xqb + b);
-        up += dev_dot_iq2_xxs_q8_K_block(ur + b, xqb + b);
+    const float iscale = -127.0f / signed_max;
+    int qv[CUDA_QK_K / ROCM_Q8_K_WAVE32_WIDTH];
+    #pragma unroll
+    for (uint32_t k = 0u;
+         k < CUDA_QK_K / ROCM_Q8_K_WAVE32_WIDTH; k++) {
+        const uint32_t index = lane + k * ROCM_Q8_K_WAVE32_WIDTH;
+        int q = (int)lrintf(iscale * xr[index]);
+        if (q > 127) q = 127;
+        if (q < -128) q = -128;
+        qv[k] = q;
+        yb->qs[index] = (int8_t)q;
+    }
+
+    /* A width-16 reduction simultaneously produces the low- and high-half
+     * bsum for each contiguous 32-value slice.  Only lanes 0 and 16 store. */
+    #pragma unroll
+    for (uint32_t offset = 8u; offset != 0u; offset >>= 1u) {
+        #pragma unroll
+        for (uint32_t k = 0u;
+             k < CUDA_QK_K / ROCM_Q8_K_WAVE32_WIDTH; k++) {
+#if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__)
+            qv[k] += __shfl_down(qv[k], offset, 16);
+#else
+            qv[k] += __shfl_down_sync(
+                FULL_WARP_MASK, qv[k], offset, 16);
+#endif
+        }
     }
-    gate = half_warp_sum_f32(gate, lane);
-    up = half_warp_sum_f32(up, lane);
-    if (lane == 0) {
-        if (clamp > 1.0e-6f) {
-            if (gate > clamp) gate = clamp;
-            if (up > clamp) up = clamp;
-            if (up < -clamp) up = -clamp;
+    if (lane == 0u || lane == 16u) {
+        const uint32_t half = lane >> 4u;
+        #pragma unroll
+        for (uint32_t k = 0u;
+             k < CUDA_QK_K / ROCM_Q8_K_WAVE32_WIDTH; k++) {
+            yb->bsums[2u * k + half] = (int16_t)qv[k];
         }
-        const uint64_t off = (uint64_t)pair * expert_mid_dim + row;
-        gate_out[off] = gate;
-        up_out[off] = up;
-        mid_out[off] = (gate / (1.0f + expf(-gate))) * up * weights[(uint64_t)tok * n_expert + slot];
     }
+    if (lane == 0u) yb->d = 1.0f / iscale;
 }
 
 __global__ static void moe_gate_up_mid_qwarp32_kernel(
@@ -1301,21 +1218,6 @@ __global__ static void moe_prefix_sorted_pairs_kernel(
     }
 }
 
-__global__ static void moe_scatter_sorted_pairs_kernel(
-        uint32_t *sorted_pairs,
-        uint32_t *cursors,
-        const int32_t *selected,
-        uint32_t pair_count,
-        uint32_t n_total_expert) {
-    uint32_t pair = (uint32_t)((uint64_t)blockIdx.x * blockDim.x + threadIdx.x);
-    if (pair >= pair_count) return;
-    int32_t expert_i = selected[pair];
-    if (expert_i < 0) expert_i = 0;
-    if ((uint32_t)expert_i >= n_total_expert) return;
-    uint32_t pos = atomicAdd(cursors + (uint32_t)expert_i, 1u);
-    sorted_pairs[pos] = pair;
-}
-
 /* Keep pair order stable inside each expert bucket.  The MoE WMMA kernels are
  * row-position sensitive enough that atomic append order changes logits. */
 __global__ static void moe_scatter_sorted_pairs_deterministic_kernel(
@@ -1430,68 +1332,6 @@ __global__ static void moe_gate_up_mid_sorted_qwarp32_kernel(
     }
 }
 
-__global__ static DS4_ROCM_UNUSED void moe_gate_up_mid_expert_tile8_kernel(
-        float *gate_out,
-        float *up_out,
-        float *mid_out,
-        const char *gate_base,
-        const char *up_base,
-        const cuda_block_q8_K *xq,
-        const uint32_t *sorted_pairs,
-        const uint32_t *offsets,
-        const uint32_t *counts,
-        const uint32_t *tile_total,
-        const uint32_t *tile_experts,
-        const uint32_t *tile_starts,
-        const float *weights,
-        uint64_t gate_expert_bytes,
-        uint64_t gate_row_bytes,
-        uint32_t xq_blocks,
-        uint32_t expert_mid_dim,
-        uint32_t n_expert,
-        float clamp) {
-    uint32_t tile = blockIdx.y;
-    if (tile >= *tile_total) return;
-    uint32_t group = threadIdx.x >> 3u;
-    uint32_t lane = threadIdx.x & 7u;
-    uint32_t pair_slot = group & 7u;
-    uint32_t row_lane = group >> 3u;
-    uint32_t expert = tile_experts[tile];
-    uint32_t local_pair = tile_starts[tile] + pair_slot;
-    if (local_pair >= counts[expert]) return;
-    uint32_t sorted_idx = offsets[expert] + local_pair;
-    uint32_t pair = sorted_pairs[sorted_idx];
-    uint32_t tok = pair / n_expert;
-    uint32_t slot = pair - tok * n_expert;
-    const cuda_block_q8_K *xqb = xq + (uint64_t)tok * xq_blocks;
-
-    for (uint32_t rr = 0; rr < 2u; rr++) {
-        uint32_t row = blockIdx.x * 8u + row_lane + rr * 4u;
-        if (row >= expert_mid_dim) continue;
-        const cuda_block_iq2_xxs *gr = (const cuda_block_iq2_xxs *)(gate_base + (uint64_t)expert * gate_expert_bytes + (uint64_t)row * gate_row_bytes);
-        const cuda_block_iq2_xxs *ur = (const cuda_block_iq2_xxs *)(up_base + (uint64_t)expert * gate_expert_bytes + (uint64_t)row * gate_row_bytes);
-        float gate = 0.0f;
-        float up = 0.0f;
-        for (uint32_t b = lane; b < xq_blocks; b += 8u) {
-            gate += dev_dot_iq2_xxs_q8_K_block(gr + b, xqb + b);
-            up += dev_dot_iq2_xxs_q8_K_block(ur + b, xqb + b);
-        }
-        gate = quarter_warp_sum_f32(gate, lane);
-        up = quarter_warp_sum_f32(up, lane);
-        if (lane == 0) {
-            if (clamp > 1.0e-6f) {
-                if (gate > clamp) gate = clamp;
-                if (up > clamp) up = clamp;
-                if (up < -clamp) up = -clamp;
-            }
-            const uint64_t off = (uint64_t)pair * expert_mid_dim + row;
-            gate_out[off] = gate;
-            up_out[off] = up;
-            mid_out[off] = (gate / (1.0f + expf(-gate))) * up * weights[(uint64_t)tok * n_expert + slot];
-        }
-    }
-}
-
 __global__ static void moe_gate_up_mid_expert_tile4_row32_kernel(
         float *gate_out,
         float *up_out,
@@ -2633,90 +2473,6 @@ __global__ static void moe_gate_up_mid_mxfp4_expert_tile32_row32_kernel(
     }
 }
 
-__global__ static DS4_ROCM_UNUSED void moe_down_kernel(
-        float *down_out,
-        const char *down_base,
-        const cuda_block_q8_K *midq,
-        const int32_t *selected,
-        uint64_t down_expert_bytes,
-        uint64_t down_row_bytes,
-        uint32_t midq_blocks,
-        uint32_t out_dim,
-        uint32_t n_expert) {
-    uint32_t row = blockIdx.x;
-    uint32_t pair = blockIdx.y;
-    if (row >= out_dim) return;
-    uint32_t tok = pair / n_expert;
-    uint32_t slot = pair - tok * n_expert;
-    int32_t expert_i = selected[(uint64_t)tok * n_expert + slot];
-    if (expert_i < 0) expert_i = 0;
-    const cuda_block_q2_K *wr = (const cuda_block_q2_K *)(down_base + (uint64_t)(uint32_t)expert_i * down_expert_bytes + (uint64_t)row * down_row_bytes);
-    const cuda_block_q8_K *xq = midq + (uint64_t)pair * midq_blocks;
-    float acc = 0.0f;
-    for (uint32_t b = threadIdx.x; b < midq_blocks; b += blockDim.x) acc += dev_dot_q2_K_q8_K_block(wr + b, xq + b);
-    __shared__ float partial[256];
-    partial[threadIdx.x] = acc;
-    __syncthreads();
-    for (uint32_t stride = blockDim.x >> 1; stride > 0; stride >>= 1) {
-        if (threadIdx.x < stride) partial[threadIdx.x] += partial[threadIdx.x + stride];
-        __syncthreads();
-    }
-    if (threadIdx.x == 0) down_out[(uint64_t)pair * out_dim + row] = partial[0];
-}
-
-__global__ static DS4_ROCM_UNUSED void moe_down_warp8_kernel(
-        float *down_out,
-        const char *down_base,
-        const cuda_block_q8_K *midq,
-        const int32_t *selected,
-        uint64_t down_expert_bytes,
-        uint64_t down_row_bytes,
-        uint32_t midq_blocks,
-        uint32_t out_dim,
-        uint32_t n_expert) {
-    uint32_t lane = threadIdx.x & 31u;
-    uint32_t warp = threadIdx.x >> 5u;
-    uint32_t row = blockIdx.x * 8u + warp;
-    uint32_t pair = blockIdx.y;
-    if (row >= out_dim) return;
-    uint32_t tok = pair / n_expert;
-    uint32_t slot = pair - tok * n_expert;
-    int32_t expert_i = selected[(uint64_t)tok * n_expert + slot];
-    if (expert_i < 0) expert_i = 0;
-    const cuda_block_q2_K *wr = (const cuda_block_q2_K *)(down_base + (uint64_t)(uint32_t)expert_i * down_expert_bytes + (uint64_t)row * down_row_bytes);
-    const cuda_block_q8_K *xq = midq + (uint64_t)pair * midq_blocks;
-    float acc = 0.0f;
-    for (uint32_t b = lane; b < midq_blocks; b += 32u) acc += dev_dot_q2_K_q8_K_block(wr + b, xq + b);
-    acc = warp_sum_f32(acc);
-    if (lane == 0) down_out[(uint64_t)pair * out_dim + row] = acc;
-}
-
-__global__ static DS4_ROCM_UNUSED void moe_down_hwarp16_kernel(
-        float *down_out,
-        const char *down_base,
-        const cuda_block_q8_K *midq,
-        const int32_t *selected,
-        uint64_t down_expert_bytes,
-        uint64_t down_row_bytes,
-        uint32_t midq_blocks,
-        uint32_t out_dim,
-        uint32_t n_expert) {
-    uint32_t lane = threadIdx.x & 15u;
-    uint32_t row = blockIdx.x * 16u + (threadIdx.x >> 4u);
-    uint32_t pair = blockIdx.y;
-    if (row >= out_dim) return;
-    uint32_t tok = pair / n_expert;
-    uint32_t slot = pair - tok * n_expert;
-    int32_t expert_i = selected[(uint64_t)tok * n_expert + slot];
-    if (expert_i < 0) expert_i = 0;
-    const cuda_block_q2_K *wr = (const cuda_block_q2_K *)(down_base + (uint64_t)(uint32_t)expert_i * down_expert_bytes + (uint64_t)row * down_row_bytes);
-    const cuda_block_q8_K *xq = midq + (uint64_t)pair * midq_blocks;
-    float acc = 0.0f;
-    for (uint32_t b = lane; b < midq_blocks; b += 16u) acc += dev_dot_q2_K_q8_K_block(wr + b, xq + b);
-    acc = half_warp_sum_f32(acc, lane);
-    if (lane == 0) down_out[(uint64_t)pair * out_dim + row] = acc;
-}
-
 __global__ static void moe_down_qwarp32_kernel(
         float *down_out,
         const char *down_base,
@@ -3497,45 +3253,6 @@ __global__ static void moe_down_sorted_qwarp32_kernel(
     if (lane == 0) down_out[(uint64_t)pair * out_dim + row] = acc;
 }
 
-__global__ static DS4_ROCM_UNUSED void moe_down_expert_tile8_kernel(
-        float *down_out,
-        const char *down_base,
-        const cuda_block_q8_K *midq,
-        const uint32_t *sorted_pairs,
-        const uint32_t *offsets,
-        const uint32_t *counts,
-        const uint32_t *tile_total,
-        const uint32_t *tile_experts,
-        const uint32_t *tile_starts,
-        uint64_t down_expert_bytes,
-        uint64_t down_row_bytes,
-        uint32_t midq_blocks,
-        uint32_t out_dim,
-        uint32_t n_expert) {
-    uint32_t tile = blockIdx.y;
-    if (tile >= *tile_total) return;
-    uint32_t group = threadIdx.x >> 3u;
-    uint32_t lane = threadIdx.x & 7u;
-    uint32_t pair_slot = group & 7u;
-    uint32_t row_lane = group >> 3u;
-    uint32_t expert = tile_experts[tile];
-    uint32_t local_pair = tile_starts[tile] + pair_slot;
-    if (local_pair >= counts[expert]) return;
-    uint32_t sorted_idx = offsets[expert] + local_pair;
-    uint32_t pair = sorted_pairs[sorted_idx];
-    const cuda_block_q8_K *xq = midq + (uint64_t)pair * midq_blocks;
-
-    for (uint32_t rr = 0; rr < 2u; rr++) {
-        uint32_t row = blockIdx.x * 8u + row_lane + rr * 4u;
-        if (row >= out_dim) continue;
-        const cuda_block_q2_K *wr = (const cuda_block_q2_K *)(down_base + (uint64_t)expert * down_expert_bytes + (uint64_t)row * down_row_bytes);
-        float acc = 0.0f;
-        for (uint32_t b = lane; b < midq_blocks; b += 8u) acc += dev_dot_q2_K_q8_K_block(wr + b, xq + b);
-        acc = quarter_warp_sum_f32(acc, lane);
-        if (lane == 0) down_out[(uint64_t)pair * out_dim + row] = acc;
-    }
-}
-
 __global__ static void moe_down_expert_tile4_row32_kernel(
         float *down_out,
         const char *down_base,
@@ -3986,21 +3703,6 @@ __device__ __forceinline__ static float q2_K_dequant_256_scaled_w32(
 }
 
 
-__device__ __forceinline__ static float q2_K_dequant_256_direct(const unsigned char *blk, uint32_t i) {
-    const uint16_t d_bits = (uint16_t)blk[80] | ((uint16_t)blk[81] << 8);
-    const uint16_t dmin_bits = (uint16_t)blk[82] | ((uint16_t)blk[83] << 8);
-    const unsigned char *sc = blk;
-    const unsigned char *qs = blk + 16u;
-    const uint32_t g = i >> 4u;
-    const uint32_t within = g & 7u;
-    const uint32_t qi = (g >> 3u) * 32u + (within & 1u) * 16u + (i & 15u);
-    const uint32_t shift = (within >> 1u) * 2u;
-    const float q = (float)((qs[qi] >> shift) & 3u);
-    const float scale = (float)(sc[g] & 0x0fu);
-    const float mn = (float)(sc[g] >> 4u);
-    return dev_f16_to_f32(d_bits) * scale * q - dev_f16_to_f32(dmin_bits) * mn;
-}
-
 template <int BN, int BK>
 __device__ __forceinline__ static void q2_K_dequant_tile_half_rowwise(
         half *shB,
@@ -4041,107 +3743,6 @@ __device__ __forceinline__ static void q2_K_dequant_tile_half_rowwise(
     }
 }
 
-template <int BN, int BK>
-__device__ __forceinline__ static void q2_K_dequant_dual_tile_half_rowwise(
-        half *shB0,
-        half *shB1,
-        const unsigned char *base0,
-        const unsigned char *base1,
-        uint64_t row_bytes,
-        uint32_t n0,
-        uint32_t k0,
-        uint32_t out_dim,
-        uint32_t tid) {
-    const uint32_t g = (k0 & 255u) >> 4u;
-    const uint32_t within = g & 7u;
-    const uint32_t qbase = (g >> 3u) * 32u + (within & 1u) * 16u;
-    const uint32_t shift = (within >> 1u) * 2u;
-    constexpr uint32_t KG = 2u;
-    for (uint32_t j = tid; j < (uint32_t)(BN * (BK / KG)); j += blockDim.x) {
-        const uint32_t nn = j / (uint32_t)(BK / KG);
-        const uint32_t kk0 = (j - nn * (uint32_t)(BK / KG)) * KG;
-        const uint32_t row = n0 + nn;
-        if (row < out_dim) {
-            const unsigned char *blk0 = base0 + (uint64_t)row * row_bytes + (uint64_t)(k0 >> 8u) * 84u;
-            const unsigned char *blk1 = base1 + (uint64_t)row * row_bytes + (uint64_t)(k0 >> 8u) * 84u;
-            const float d0 = dev_f16_to_f32((uint16_t)blk0[80] | ((uint16_t)blk0[81] << 8));
-            const float dm0 = dev_f16_to_f32((uint16_t)blk0[82] | ((uint16_t)blk0[83] << 8));
-            const float d1 = dev_f16_to_f32((uint16_t)blk1[80] | ((uint16_t)blk1[81] << 8));
-            const float dm1 = dev_f16_to_f32((uint16_t)blk1[82] | ((uint16_t)blk1[83] << 8));
-            const float s0 = (float)(blk0[g] & 0x0fu);
-            const float m0 = (float)(blk0[g] >> 4u);
-            const float s1 = (float)(blk1[g] & 0x0fu);
-            const float m1 = (float)(blk1[g] >> 4u);
-#pragma unroll
-            for (uint32_t u = 0; u < KG; u++) {
-                const uint32_t kk = kk0 + u;
-                const float q0 = (float)((blk0[16u + qbase + kk] >> shift) & 3u);
-                const float q1 = (float)((blk1[16u + qbase + kk] >> shift) & 3u);
-                const uint32_t sj = kk * (uint32_t)BN + nn;
-                shB0[sj] = __float2half(d0 * s0 * q0 - dm0 * m0);
-                shB1[sj] = __float2half(d1 * s1 * q1 - dm1 * m1);
-            }
-        } else {
-#pragma unroll
-            for (uint32_t u = 0; u < KG; u++) {
-                const uint32_t kk = kk0 + u;
-                const uint32_t sj = kk * (uint32_t)BN + nn;
-                shB0[sj] = __float2half(0.0f);
-                shB1[sj] = __float2half(0.0f);
-            }
-        }
-    }
-}
-
-template <int BN, int BK>
-__device__ __forceinline__ static void q2_K_dequant_pair_tile_half_rowwise(
-        half *shB0,
-        half *shB1,
-        const unsigned char *base,
-        uint64_t row_bytes,
-        uint32_t n0,
-        uint32_t k0,
-        uint32_t out_dim,
-        uint32_t tid) {
-    const uint32_t g = (k0 & 255u) >> 4u;
-    const uint32_t within = g & 7u;
-    const uint32_t qbase = (g >> 3u) * 32u + (within & 1u) * 16u;
-    const uint32_t shift = (within >> 1u) * 2u;
-    constexpr uint32_t KG = 4u;
-    constexpr uint32_t UNITS_PER_TILE = (uint32_t)(BN * (BK / KG));
-    for (uint32_t j = tid; j < 2u * UNITS_PER_TILE; j += blockDim.x) {
-        const uint32_t tile = j / UNITS_PER_TILE;
-        const uint32_t rem = j - tile * UNITS_PER_TILE;
-        const uint32_t nn = rem / (uint32_t)(BK / KG);
-        const uint32_t kk0 = (rem - nn * (uint32_t)(BK / KG)) * KG;
-        const uint32_t row = n0 + tile * (uint32_t)BN + nn;
-        half *shB = tile == 0u ? shB0 : shB1;
-        uint32_t v0 = 0u;
-        uint32_t v1 = 0u;
-        if (row < out_dim) {
-            const unsigned char *blk = base + (uint64_t)row * row_bytes + (uint64_t)(k0 >> 8u) * 84u;
-            const float d = dev_f16_to_f32((uint16_t)blk[80] | ((uint16_t)blk[81] << 8));
-            const float dm = dev_f16_to_f32((uint16_t)blk[82] | ((uint16_t)blk[83] << 8));
-            const float s = (float)(blk[g] & 0x0fu);
-            const float m = (float)(blk[g] >> 4u);
-            const uint32_t qbits = *reinterpret_cast<const uint32_t *>(blk + 16u + qbase + kk0);
-            const uint32_t q0 = (qbits >> shift) & 3u;
-            const uint32_t q1 = (qbits >> (8u + shift)) & 3u;
-            const uint32_t q2 = (qbits >> (16u + shift)) & 3u;
-            const uint32_t q3 = (qbits >> (24u + shift)) & 3u;
-            const float ds = d * s;
-            const float dmm = dm * m;
-            v0 = dev_pack_half2_bits(ds * (float)q0 - dmm,
-                                     ds * (float)q1 - dmm);
-            v1 = dev_pack_half2_bits(ds * (float)q2 - dmm,
-                                     ds * (float)q3 - dmm);
-        }
-        half *dst = shB + nn * (uint32_t)BK + kk0;
-        *reinterpret_cast<uint32_t *>(dst) = v0;
-        *reinterpret_cast<uint32_t *>(dst + 2u) = v1;
-    }
-}
-
 template <int BN, int BK>
 __device__ __forceinline__ static void q2_K_dequant_pair_tile_half_rowwise_staged(
         half *shB0,
@@ -4838,7 +4439,7 @@ __device__ __forceinline__ static void iq2_xxs_dequant_dual_pair_tile_half_rowwi
     }
 }
 
-template <int MTILES=8, int BM=16, int BN=16, int BK=16, bool OUT_F16=false, bool X_F16=false>
+template <int MTILES=8, int BM=16, int BN=16, int BK=16, bool OUT_F16=false, bool X_F16=false, bool TAIL_WAVE_CULL=false>
 __global__ static void moe_gate_up_mid_iq2_hotlist_wmma_n2_kernel(
         float *mid_out,
         half *mid_out_h,
@@ -4857,6 +4458,10 @@ __global__ static void moe_gate_up_mid_iq2_hotlist_wmma_n2_kernel(
         uint64_t gate_expert_bytes,
         uint64_t gate_row_bytes,
         float clamp) {
+    /* rocWMMA is collective over the hardware wave.  The host selector keeps
+     * this kernel off wave64 devices so the scalar launch remains the complete
+     * fallback; retain a uniform device-side guard as a final safety net. */
+    if (warpSize != 32) return;
     extern __shared__ unsigned char raw_sh[];
     half *shA = reinterpret_cast<half *>(raw_sh);
     half *shBg0 = shA + MTILES * BM * BK;
@@ -4890,7 +4495,10 @@ __global__ static void moe_gate_up_mid_iq2_hotlist_wmma_n2_kernel(
     frag_a a;
     frag_b bg0, bu0, bg1, bu1;
     frag_c accg0, accu0, accg1, accu1;
-    if (wave < MTILES) {
+    const bool wmma_active =
+        wave < MTILES &&
+        (!TAIL_WAVE_CULL || wave * (uint32_t)BM < count - m_group0);
+    if (wmma_active) {
         rocwmma::fill_fragment(accg0, 0.0f);
         rocwmma::fill_fragment(accu0, 0.0f);
         rocwmma::fill_fragment(accg1, 0.0f);
@@ -4926,7 +4534,7 @@ __global__ static void moe_gate_up_mid_iq2_hotlist_wmma_n2_kernel(
         iq2_xxs_dequant_dual_pair_tile_half_rowwise<BN, BK>(
                 shBg0, shBu0, shBg1, shBu1, gew, uew, gate_row_bytes, n0, k0, expert_mid_dim, tid);
         __syncthreads();
-        if (wave < MTILES) {
+        if (wmma_active) {
             rocwmma::load_matrix_sync(a, shA + wave * BM * BK, BK);
             rocwmma::load_matrix_sync(bg0, shBg0, BN);
             rocwmma::load_matrix_sync(bu0, shBu0, BN);
@@ -4940,7 +4548,7 @@ __global__ static void moe_gate_up_mid_iq2_hotlist_wmma_n2_kernel(
         __syncthreads();
     }
 
-    if (wave < MTILES) {
+    if (wmma_active) {
         rocwmma::store_matrix_sync(shCg0 + wave * BM * BN, accg0, BN, rocwmma::mem_row_major);
         rocwmma::store_matrix_sync(shCu0 + wave * BM * BN, accu0, BN, rocwmma::mem_row_major);
         rocwmma::store_matrix_sync(shCg1 + wave * BM * BN, accg1, BN, rocwmma::mem_row_major);
@@ -5223,7 +4831,7 @@ __global__ static void moe_down_q2K_hotlist_wmma_kernel(
     }
 }
 
-template <int MTILES=8, int BM=16, int BN=16, int BK=16, bool MID_F16=false, bool OUT_F16=false, bool SLOT_MAJOR=false>
+template <int MTILES=8, int BM=16, int BN=16, int BK=16, bool MID_F16=false, bool OUT_F16=false, bool SLOT_MAJOR=false, bool TAIL_WAVE_CULL=false>
 __global__ static void moe_down_q2K_hotlist_wmma_n2_kernel(
         float *down_out,
         half *down_out_h,
@@ -5241,6 +4849,9 @@ __global__ static void moe_down_q2K_hotlist_wmma_n2_kernel(
         uint64_t down_row_bytes,
         uint32_t n_expert,
         uint32_t n_tokens = 0u) {
+    /* See the IQ2 gate/up hot-list kernel above.  Correct wave64 fallback is
+     * selected on the host before scalar_max excludes any routed rows. */
+    if (warpSize != 32) return;
     extern __shared__ unsigned char raw_sh[];
     half *shA = reinterpret_cast<half *>(raw_sh);
     half *shB0 = shA + MTILES * BM * BK;
@@ -5274,7 +4885,10 @@ __global__ static void moe_down_q2K_hotlist_wmma_n2_kernel(
     frag_b b1;
     frag_c acc0;
     frag_c acc1;
-    if (wave < MTILES) {
+    const bool wmma_active =
+        wave < MTILES &&
+        (!TAIL_WAVE_CULL || wave * (uint32_t)BM < count - m_group0);
+    if (wmma_active) {
         rocwmma::fill_fragment(acc0, 0.0f);
         rocwmma::fill_fragment(acc1, 0.0f);
     }
@@ -5326,7 +4940,7 @@ __global__ static void moe_down_q2K_hotlist_wmma_n2_kernel(
             q2_K_dequant_pair_tile_half_rowwise_staged<BN, BK>(
                     shB0, shB1, shW, krel, tid);
             __syncthreads();
-            if (wave < MTILES) {
+            if (wmma_active) {
                 rocwmma::load_matrix_sync(a, shA + wave * BM * BK, BK);
                 rocwmma::load_matrix_sync(b0, shB0, BN);
                 rocwmma::load_matrix_sync(b1, shB1, BN);
@@ -5337,7 +4951,7 @@ __global__ static void moe_down_q2K_hotlist_wmma_n2_kernel(
         }
     }
 
-    if (wave < MTILES) {
+    if (wmma_active) {
         rocwmma::store_matrix_sync(shC + wave * BM * BN, acc0, BN, rocwmma::mem_row_major);
     }
     __syncthreads();
@@ -5366,7 +4980,7 @@ __global__ static void moe_down_q2K_hotlist_wmma_n2_kernel(
     }
     __syncthreads();
 
-    if (wave < MTILES) {
+    if (wmma_active) {
         rocwmma::store_matrix_sync(shC + wave * BM * BN, acc1, BN, rocwmma::mem_row_major);
     }
     __syncthreads();
diff --git a/rocm/ds4_rocm_moe_launch.cuh b/rocm/ds4_rocm_moe_launch.cuh
index 16ad7bcb4b..a7f4df50dc 100644
--- a/rocm/ds4_rocm_moe_launch.cuh
+++ b/rocm/ds4_rocm_moe_launch.cuh
@@ -10,6 +10,350 @@ static int routed_moe_align256_checked(uint64_t v, uint64_t *out) {
     return 1;
 }
 
+enum {
+    DS4_ROCM_MOE_ENV_INVALID = -2
+};
+
+static const char *const DS4_ROCM_IQ2_Q2_TAIL_CULL_ENABLE_ENV =
+    "DS4_ROCM_ENABLE_IQ2_MOE_WMMA_TAIL_CULL";
+static const char *const DS4_ROCM_IQ2_Q2_TAIL_CULL_DISABLE_ENV =
+    "DS4_ROCM_DISABLE_IQ2_MOE_WMMA_TAIL_CULL";
+static const char *const DS4_ROCM_IQ2_Q2_WMMA_PROFILE_ENV =
+    "DS4_ROCM_IQ2_MOE_WMMA_PROFILE";
+
+static pthread_mutex_t g_routed_moe_hotlist_policy_mu =
+    PTHREAD_MUTEX_INITIALIZER;
+static uint32_t g_routed_moe_hotlist_notice_mask;
+
+static int routed_moe_hotlist_notice_once(uint32_t bit) {
+    pthread_mutex_lock(&g_routed_moe_hotlist_policy_mu);
+    const int report = (g_routed_moe_hotlist_notice_mask & bit) == 0u;
+    g_routed_moe_hotlist_notice_mask |= bit;
+    pthread_mutex_unlock(&g_routed_moe_hotlist_policy_mu);
+    return report;
+}
+
+static int routed_moe_env_value_eq(const char *value,
+                                   size_t value_len,
+                                   const char *literal) {
+    const size_t literal_len = strlen(literal);
+    if (value_len != literal_len) return 0;
+    for (size_t i = 0; i < value_len; i++) {
+        if (tolower((unsigned char)value[i]) !=
+            tolower((unsigned char)literal[i])) {
+            return 0;
+        }
+    }
+    return 1;
+}
+
+/* Return -1 for unset, 0/1 for a recognized boolean, and -2 for invalid.
+ * Do not cache values: benchmark A/B arms may change the environment between
+ * launches in the same process. */
+static int routed_moe_env_bool_value(const char *name, uint32_t notice_bit) {
+    const char *value = name ? getenv(name) : NULL;
+    if (!value) return -1;
+    while (isspace((unsigned char)*value)) value++;
+    size_t value_len = strlen(value);
+    while (value_len != 0u &&
+           isspace((unsigned char)value[value_len - 1u])) {
+        value_len--;
+    }
+    if (value_len == 0u) return 1;
+    if (routed_moe_env_value_eq(value, value_len, "1") ||
+        routed_moe_env_value_eq(value, value_len, "true") ||
+        routed_moe_env_value_eq(value, value_len, "yes") ||
+        routed_moe_env_value_eq(value, value_len, "on")) {
+        return 1;
+    }
+    if (routed_moe_env_value_eq(value, value_len, "0") ||
+        routed_moe_env_value_eq(value, value_len, "false") ||
+        routed_moe_env_value_eq(value, value_len, "no") ||
+        routed_moe_env_value_eq(value, value_len, "off")) {
+        return 0;
+    }
+    if (routed_moe_hotlist_notice_once(notice_bit)) {
+        const size_t shown_len = value_len < 96u ? value_len : 96u;
+        fprintf(stderr,
+                DS4_GPU_LOG_PREFIX "invalid boolean environment value "
+                "%s=%.*s%s; treating the opt-in as disabled\n",
+                name,
+                (int)shown_len,
+                value,
+                shown_len == value_len ? "" : "...");
+    }
+    return DS4_ROCM_MOE_ENV_INVALID;
+}
+
+/* HIP exposes the hardware wave width through cudaDeviceProp::warpSize.  The
+ * active device is thread-local in the runtime, so keep the successful property
+ * query thread-local too.  This avoids a process-wide mutex on every prefill
+ * layer while still noticing cudaSetDevice changes.  Failures stay fail-closed
+ * and may be retried on a later launch. */
+#if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__)
+static int routed_moe_runtime_warp_size_value(void) {
+    int device = -1;
+    const cudaError_t device_err = cudaGetDevice(&device);
+    if (device_err != cudaSuccess || device < 0) {
+        if (routed_moe_hotlist_notice_once(1u << 4)) {
+            fprintf(stderr,
+                    DS4_GPU_LOG_PREFIX "cannot query the active device for "
+                    "routed-MoE rocWMMA; using scalar fallback: %s\n",
+                    cudaGetErrorString(device_err));
+        }
+        (void)cudaGetLastError();
+        return 0;
+    }
+    static thread_local int cached_device = -1;
+    static thread_local int cached_warp_size = 0;
+    if (cached_device == device && cached_warp_size != 0) {
+        return cached_warp_size;
+    }
+
+    cudaDeviceProp prop = {};
+    const cudaError_t prop_err = cudaGetDeviceProperties(&prop, device);
+    if (prop_err != cudaSuccess || prop.warpSize <= 0) {
+        if (routed_moe_hotlist_notice_once(1u << 5)) {
+            fprintf(stderr,
+                    DS4_GPU_LOG_PREFIX "cannot query runtime warpSize for "
+                    "routed-MoE rocWMMA; using scalar fallback: %s\n",
+                    cudaGetErrorString(prop_err));
+        }
+        (void)cudaGetLastError();
+        return 0;
+    }
+    cached_device = device;
+    cached_warp_size = prop.warpSize;
+    return cached_warp_size;
+}
+#endif
+
+static int routed_moe_iq2_q2_tail_wave_cull_resolve(
+        int runtime_warp_size,
+        int *enable_value,
+        int *disable_value) {
+    const int enable = routed_moe_env_bool_value(
+        DS4_ROCM_IQ2_Q2_TAIL_CULL_ENABLE_ENV, 1u << 0);
+    const int disable = routed_moe_env_bool_value(
+        DS4_ROCM_IQ2_Q2_TAIL_CULL_DISABLE_ENV, 1u << 1);
+    if (enable_value) *enable_value = enable;
+    if (disable_value) *disable_value = disable;
+    if (enable == DS4_ROCM_MOE_ENV_INVALID ||
+        disable == DS4_ROCM_MOE_ENV_INVALID) {
+        return 0;
+    }
+    if (disable == 1) {
+        if (enable == 1 && routed_moe_hotlist_notice_once(1u << 2)) {
+            fprintf(stderr,
+                    DS4_GPU_LOG_PREFIX "IQ2/Q2 hot-list tail-wave cull "
+                    "disabled by %s (overrides %s)\n",
+                    DS4_ROCM_IQ2_Q2_TAIL_CULL_DISABLE_ENV,
+                    DS4_ROCM_IQ2_Q2_TAIL_CULL_ENABLE_ENV);
+        }
+        return 0;
+    }
+    if (enable != 1) return 0; /* Explicit opt-in: default is off. */
+    if (runtime_warp_size != 32) {
+        if (routed_moe_hotlist_notice_once(1u << 3)) {
+            fprintf(stderr,
+                    DS4_GPU_LOG_PREFIX "IQ2/Q2 hot-list tail-wave cull "
+                    "requested with runtime warpSize=%d; using scalar "
+                    "fallback\n",
+                    runtime_warp_size);
+        }
+        return 0;
+    }
+    if (routed_moe_hotlist_notice_once(1u << 6)) {
+        fprintf(stderr,
+                DS4_GPU_LOG_PREFIX "IQ2/Q2 hot-list tail-wave cull policy enabled "
+                "(opt-in, runtime warpSize=32)\n");
+    }
+    return 1;
+}
+
+#if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__)
+typedef struct {
+    cudaEvent_t start;
+    cudaEvent_t end;
+} routed_moe_wmma_profile_timer;
+
+static int routed_moe_wmma_profile_timer_destroy(
+        routed_moe_wmma_profile_timer *timer,
+        const char *what) {
+    if (!timer) return 0;
+    int ok = 1;
+    if (timer->start) {
+        const cudaError_t err = cudaEventDestroy(timer->start);
+        timer->start = NULL;
+        if (!cuda_ok(err, what)) ok = 0;
+    }
+    if (timer->end) {
+        const cudaError_t err = cudaEventDestroy(timer->end);
+        timer->end = NULL;
+        if (!cuda_ok(err, what)) ok = 0;
+    }
+    return ok;
+}
+
+static int routed_moe_wmma_profile_timer_begin(
+        routed_moe_wmma_profile_timer *timer,
+        const char *what) {
+    if (!timer) return 0;
+    timer->start = NULL;
+    timer->end = NULL;
+    cudaError_t err = cudaEventCreate(&timer->start);
+    if (err != cudaSuccess) return cuda_ok(err, what);
+    err = cudaEventCreate(&timer->end);
+    if (err != cudaSuccess) {
+        (void)cuda_ok(err, what);
+        (void)routed_moe_wmma_profile_timer_destroy(timer, what);
+        return 0;
+    }
+    err = cudaEventRecord(timer->start, 0);
+    if (err != cudaSuccess) {
+        (void)cuda_ok(err, what);
+        (void)routed_moe_wmma_profile_timer_destroy(timer, what);
+        return 0;
+    }
+    return 1;
+}
+
+static int routed_moe_wmma_profile_timer_finish(
+        routed_moe_wmma_profile_timer *timer,
+        float *elapsed_ms,
+        const char *what) {
+    if (!timer || !timer->start || !timer->end || !elapsed_ms) {
+        if (timer) {
+            (void)routed_moe_wmma_profile_timer_destroy(timer, what);
+        }
+        return 0;
+    }
+    int ok = cuda_ok(cudaEventRecord(timer->end, 0), what);
+    if (ok) ok = cuda_ok(cudaEventSynchronize(timer->end), what);
+    if (ok) {
+        float ms = 0.0f;
+        ok = cuda_ok(cudaEventElapsedTime(&ms, timer->start, timer->end),
+                     what);
+        if (ok) *elapsed_ms = ms;
+    }
+    if (!routed_moe_wmma_profile_timer_destroy(timer, what)) ok = 0;
+    return ok;
+}
+#endif
+
+static int routed_moe_wmma_profile_emit(
+        int enabled,
+        int tail_wave_cull,
+        uint32_t n_tokens,
+        uint64_t assignments,
+        int gate_up_seen,
+        float gate_up_ms,
+        int down_seen,
+        float down_ms,
+        int call_ok) {
+    if (!enabled) return call_ok;
+    const int complete = call_ok && gate_up_seen && down_seen;
+    const float shown_gate_up_ms = gate_up_seen ? gate_up_ms : -1.0f;
+    const float shown_down_ms = down_seen ? down_ms : -1.0f;
+    const float total_ms = gate_up_seen && down_seen
+        ? gate_up_ms + down_ms
+        : -1.0f;
+    const int printed = fprintf(stderr,
+            DS4_GPU_LOG_PREFIX "DS4_ROCM_IQ2_MOE_WMMA_PROFILE "
+            "tail_cull=%d tokens=%u assignments=%llu "
+            "gate_up_gpu_ms=%.6f down_gpu_ms=%.6f "
+            "wmma_total_gpu_ms=%.6f timing=cudaEvent result=%s\n",
+            tail_wave_cull,
+            n_tokens,
+            (unsigned long long)assignments,
+            shown_gate_up_ms,
+            shown_down_ms,
+            total_ms,
+            complete ? "PASS" : "FAIL");
+    if (printed < 0 || fflush(stderr) != 0) return 0;
+    return complete;
+}
+
+#if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__)
+template <int MTILES, bool OUT_F16, bool X_F16>
+static void routed_moe_launch_iq2_hotlist_wmma_n2(
+        dim3 grid,
+        dim3 block,
+        size_t shmem,
+        int tail_wave_cull,
+        float *mid_out,
+        half *mid_out_h,
+        const char *gate_base,
+        const char *up_base,
+        const float *x,
+        const half *x_h,
+        const float *weights,
+        const uint32_t *counts,
+        const uint32_t *offsets,
+        const uint32_t *pairs,
+        const uint32_t *hot_experts,
+        uint32_t hot_count,
+        uint32_t expert_in_dim,
+        uint32_t expert_mid_dim,
+        uint64_t gate_expert_bytes,
+        uint64_t gate_row_bytes,
+        float clamp) {
+    if (tail_wave_cull) {
+        moe_gate_up_mid_iq2_hotlist_wmma_n2_kernel<
+            MTILES, 16, 16, 16, OUT_F16, X_F16, true>
+            <<<grid, block, shmem>>>(
+                mid_out, mid_out_h, gate_base, up_base, x, x_h, weights,
+                counts, offsets, pairs, hot_experts, hot_count, expert_in_dim,
+                expert_mid_dim, gate_expert_bytes, gate_row_bytes, clamp);
+    } else {
+        moe_gate_up_mid_iq2_hotlist_wmma_n2_kernel<
+            MTILES, 16, 16, 16, OUT_F16, X_F16, false>
+            <<<grid, block, shmem>>>(
+                mid_out, mid_out_h, gate_base, up_base, x, x_h, weights,
+                counts, offsets, pairs, hot_experts, hot_count, expert_in_dim,
+                expert_mid_dim, gate_expert_bytes, gate_row_bytes, clamp);
+    }
+}
+
+template <int MTILES, bool MID_F16, bool OUT_F16>
+static void routed_moe_launch_q2_down_hotlist_wmma_n2(
+        dim3 grid,
+        dim3 block,
+        size_t shmem,
+        int tail_wave_cull,
+        float *down_out,
+        half *down_out_h,
+        const char *down_base,
+        const float *mid,
+        const half *mid_h,
+        const uint32_t *counts,
+        const uint32_t *offsets,
+        const uint32_t *pairs,
+        const uint32_t *hot_experts,
+        uint32_t hot_count,
+        uint32_t expert_mid_dim,
+        uint32_t out_dim,
+        uint64_t down_expert_bytes,
+        uint64_t down_row_bytes,
+        uint32_t n_expert) {
+    if (tail_wave_cull) {
+        moe_down_q2K_hotlist_wmma_n2_kernel<
+            MTILES, 16, 16, 16, MID_F16, OUT_F16, false, true>
+            <<<grid, block, shmem>>>(
+                down_out, down_out_h, down_base, mid, mid_h, counts, offsets,
+                pairs, hot_experts, hot_count, expert_mid_dim, out_dim,
+                down_expert_bytes, down_row_bytes, n_expert);
+    } else {
+        moe_down_q2K_hotlist_wmma_n2_kernel<
+            MTILES, 16, 16, 16, MID_F16, OUT_F16, false, false>
+            <<<grid, block, shmem>>>(
+                down_out, down_out_h, down_base, mid, mid_h, counts, offsets,
+                pairs, hot_experts, hot_count, expert_mid_dim, out_dim,
+                down_expert_bytes, down_row_bytes, n_expert);
+    }
+}
+#endif
+
 enum {
     DS4_ROCM_MOE_DECODE_PROFILE_GATE_RESIDENT_START = 0,
     DS4_ROCM_MOE_DECODE_PROFILE_GATE_RESIDENT_END,
@@ -204,6 +548,11 @@ static int routed_moe_q2_float_down_launch(
         const uint32_t *offsets,
         const uint32_t *sorted_pairs,
         uint32_t *hot_experts_dev,
+        int hotlist_wmma_wave32,
+        int tail_wave_cull,
+        int wmma_profile,
+        int *wmma_profile_seen,
+        float *wmma_profile_ms,
         uint32_t n_tokens,
         uint32_t n_total_expert,
         uint32_t n_expert,
@@ -215,6 +564,7 @@ static int routed_moe_q2_float_down_launch(
         n_tokens == 0u || n_total_expert == 0u || n_total_expert > DS4_ROCM_MAX_N_EXPERT ||
         n_expert == 0u || n_expert > DS4_ROCM_N_EXPERT_USED ||
         (expert_mid_dim % CUDA_QK_K) != 0u || expert_mid_dim == 0u || out_dim == 0u ||
+        (wmma_profile && (!wmma_profile_seen || !wmma_profile_ms)) ||
         !cuda_tensor_has_elems3(mid, n_tokens, n_expert, expert_mid_dim, sizeof(float)) ||
         !cuda_tensor_has_elems3(down, n_tokens, n_expert, out_dim, sizeof(float)) ||
         !cuda_tensor_has_elems2(out, n_tokens, out_dim, sizeof(float))) {
@@ -240,6 +590,7 @@ static int routed_moe_q2_float_down_launch(
     const uint32_t hot_threshold = 8u;
 #if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__)
     const int use_wmma_hot = n_tokens >= hot_threshold && hot_experts_dev &&
+        hotlist_wmma_wave32 &&
         !g_quality_mode &&
         (expert_mid_dim % 16u) == 0u && (out_dim % 16u) == 0u;
 #else
@@ -331,6 +682,13 @@ static int routed_moe_q2_float_down_launch(
 
 #if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__)
     if (use_wmma_hot && hot_count != 0u) {
+        routed_moe_wmma_profile_timer profile_timer = {};
+        if (wmma_profile &&
+            !routed_moe_wmma_profile_timer_begin(
+                &profile_timer,
+                "routed_moe q2 wmma down profile begin")) {
+            return 0;
+        }
         constexpr uint32_t bm = 16u, bn = 16u, bk = 16u;
         const int no_n2 = 0;
         const uint32_t wmma_mtiles = 4u;
@@ -343,22 +701,26 @@ static int routed_moe_q2_float_down_launch(
                 const size_t shmem_n2 = (mt * bm * bk + 2u * bk * bn) * sizeof(half) +
                                         (mt * bm * bn) * sizeof(float) + 2u * bn * 84u;
                 if (use_f16_down && hot_mid_f16 && mid_h_hot) {
-                    moe_down_q2K_hotlist_wmma_n2_kernel<4,16,16,16,true,true><<<grid, block, shmem_n2>>>(
+                    routed_moe_launch_q2_down_hotlist_wmma_n2<4,true,true>(
+                            grid, block, shmem_n2, tail_wave_cull,
                             NULL, down_h, down_w, NULL, mid_h_hot,
                             counts, offsets, sorted_pairs, hot_experts_dev, hot_count,
                             expert_mid_dim, out_dim, down_expert_bytes, down_row_bytes, n_expert);
                 } else if (use_f16_down) {
-                    moe_down_q2K_hotlist_wmma_n2_kernel<4,16,16,16,false,true><<<grid, block, shmem_n2>>>(
+                    routed_moe_launch_q2_down_hotlist_wmma_n2<4,false,true>(
+                            grid, block, shmem_n2, tail_wave_cull,
                             NULL, down_h, down_w, (const float *)mid->ptr, NULL,
                             counts, offsets, sorted_pairs, hot_experts_dev, hot_count,
                             expert_mid_dim, out_dim, down_expert_bytes, down_row_bytes, n_expert);
                 } else if (hot_mid_f16 && mid_h_hot) {
-                    moe_down_q2K_hotlist_wmma_n2_kernel<4,16,16,16,true,false><<<grid, block, shmem_n2>>>(
+                    routed_moe_launch_q2_down_hotlist_wmma_n2<4,true,false>(
+                            grid, block, shmem_n2, tail_wave_cull,
                             (float *)down->ptr, NULL, down_w, NULL, mid_h_hot,
                             counts, offsets, sorted_pairs, hot_experts_dev, hot_count,
                             expert_mid_dim, out_dim, down_expert_bytes, down_row_bytes, n_expert);
                 } else {
-                    moe_down_q2K_hotlist_wmma_n2_kernel<4,16,16,16><<<grid, block, shmem_n2>>>(
+                    routed_moe_launch_q2_down_hotlist_wmma_n2<4,false,false>(
+                            grid, block, shmem_n2, tail_wave_cull,
                             (float *)down->ptr, NULL, down_w, (const float *)mid->ptr, NULL,
                             counts, offsets, sorted_pairs, hot_experts_dev, hot_count,
                             expert_mid_dim, out_dim, down_expert_bytes, down_row_bytes, n_expert);
@@ -371,22 +733,26 @@ static int routed_moe_q2_float_down_launch(
                 const size_t shmem_n2 = (mt * bm * bk + 2u * bk * bn) * sizeof(half) +
                                         (mt * bm * bn) * sizeof(float) + 2u * bn * 84u;
                 if (use_f16_down && hot_mid_f16 && mid_h_hot) {
-                    moe_down_q2K_hotlist_wmma_n2_kernel<16,16,16,16,true,true><<<grid, block, shmem_n2>>>(
+                    routed_moe_launch_q2_down_hotlist_wmma_n2<16,true,true>(
+                            grid, block, shmem_n2, tail_wave_cull,
                             NULL, down_h, down_w, NULL, mid_h_hot,
                             counts, offsets, sorted_pairs, hot_experts_dev, hot_count,
                             expert_mid_dim, out_dim, down_expert_bytes, down_row_bytes, n_expert);
                 } else if (use_f16_down) {
-                    moe_down_q2K_hotlist_wmma_n2_kernel<16,16,16,16,false,true><<<grid, block, shmem_n2>>>(
+                    routed_moe_launch_q2_down_hotlist_wmma_n2<16,false,true>(
+                            grid, block, shmem_n2, tail_wave_cull,
                             NULL, down_h, down_w, (const float *)mid->ptr, NULL,
                             counts, offsets, sorted_pairs, hot_experts_dev, hot_count,
                             expert_mid_dim, out_dim, down_expert_bytes, down_row_bytes, n_expert);
                 } else if (hot_mid_f16 && mid_h_hot) {
-                    moe_down_q2K_hotlist_wmma_n2_kernel<16,16,16,16,true,false><<<grid, block, shmem_n2>>>(
+                    routed_moe_launch_q2_down_hotlist_wmma_n2<16,true,false>(
+                            grid, block, shmem_n2, tail_wave_cull,
                             (float *)down->ptr, NULL, down_w, NULL, mid_h_hot,
                             counts, offsets, sorted_pairs, hot_experts_dev, hot_count,
                             expert_mid_dim, out_dim, down_expert_bytes, down_row_bytes, n_expert);
                 } else {
-                    moe_down_q2K_hotlist_wmma_n2_kernel<16,16,16,16><<<grid, block, shmem_n2>>>(
+                    routed_moe_launch_q2_down_hotlist_wmma_n2<16,false,false>(
+                            grid, block, shmem_n2, tail_wave_cull,
                             (float *)down->ptr, NULL, down_w, (const float *)mid->ptr, NULL,
                             counts, offsets, sorted_pairs, hot_experts_dev, hot_count,
                             expert_mid_dim, out_dim, down_expert_bytes, down_row_bytes, n_expert);
@@ -399,22 +765,26 @@ static int routed_moe_q2_float_down_launch(
                 const size_t shmem_n2 = (mt * bm * bk + 2u * bk * bn) * sizeof(half) +
                                         (mt * bm * bn) * sizeof(float) + 2u * bn * 84u;
                 if (use_f16_down && hot_mid_f16 && mid_h_hot) {
-                    moe_down_q2K_hotlist_wmma_n2_kernel<8,16,16,16,true,true><<<grid, block, shmem_n2>>>(
+                    routed_moe_launch_q2_down_hotlist_wmma_n2<8,true,true>(
+                            grid, block, shmem_n2, tail_wave_cull,
                             NULL, down_h, down_w, NULL, mid_h_hot,
                             counts, offsets, sorted_pairs, hot_experts_dev, hot_count,
                             expert_mid_dim, out_dim, down_expert_bytes, down_row_bytes, n_expert);
                 } else if (use_f16_down) {
-                    moe_down_q2K_hotlist_wmma_n2_kernel<8,16,16,16,false,true><<<grid, block, shmem_n2>>>(
+                    routed_moe_launch_q2_down_hotlist_wmma_n2<8,false,true>(
+                            grid, block, shmem_n2, tail_wave_cull,
                             NULL, down_h, down_w, (const float *)mid->ptr, NULL,
                             counts, offsets, sorted_pairs, hot_experts_dev, hot_count,
                             expert_mid_dim, out_dim, down_expert_bytes, down_row_bytes, n_expert);
                 } else if (hot_mid_f16 && mid_h_hot) {
-                    moe_down_q2K_hotlist_wmma_n2_kernel<8,16,16,16,true,false><<<grid, block, shmem_n2>>>(
+                    routed_moe_launch_q2_down_hotlist_wmma_n2<8,true,false>(
+                            grid, block, shmem_n2, tail_wave_cull,
                             (float *)down->ptr, NULL, down_w, NULL, mid_h_hot,
                             counts, offsets, sorted_pairs, hot_experts_dev, hot_count,
                             expert_mid_dim, out_dim, down_expert_bytes, down_row_bytes, n_expert);
                 } else {
-                    moe_down_q2K_hotlist_wmma_n2_kernel<8,16,16,16><<<grid, block, shmem_n2>>>(
+                    routed_moe_launch_q2_down_hotlist_wmma_n2<8,false,false>(
+                            grid, block, shmem_n2, tail_wave_cull,
                             (float *)down->ptr, NULL, down_w, (const float *)mid->ptr, NULL,
                             counts, offsets, sorted_pairs, hot_experts_dev, hot_count,
                             expert_mid_dim, out_dim, down_expert_bytes, down_row_bytes, n_expert);
@@ -451,7 +821,26 @@ static int routed_moe_q2_float_down_launch(
                     counts, offsets, sorted_pairs, hot_experts_dev, hot_count,
                     expert_mid_dim, out_dim, down_expert_bytes, down_row_bytes);
         }
-        if (!cuda_ok(cudaGetLastError(), "routed_moe iq2/q2 float-down wmma launch")) return 0;
+        const cudaError_t launch_err = cudaGetLastError();
+        if (!cuda_ok(
+                launch_err,
+                "routed_moe iq2/q2 float-down wmma launch")) {
+            if (wmma_profile) {
+                (void)routed_moe_wmma_profile_timer_destroy(
+                    &profile_timer,
+                    "routed_moe q2 wmma down profile cleanup");
+            }
+            return 0;
+        }
+        if (wmma_profile) {
+            if (!routed_moe_wmma_profile_timer_finish(
+                    &profile_timer,
+                    wmma_profile_ms,
+                    "routed_moe q2 wmma down profile finish")) {
+                return 0;
+            }
+            *wmma_profile_seen = 1;
+        }
     }
 #endif
 
@@ -597,12 +986,42 @@ static int routed_moe_launch(
     const int mxfp4_path = plan.mxfp4_path;
     const uint64_t gate_bytes = plan.gate_bytes;
     const uint64_t down_bytes = plan.down_bytes;
+    int runtime_warp_size = 0;
+#if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__)
+    if ((iq2_gate_path && n_tokens > 1u) ||
+        (q2k_path && n_tokens >= 32u)) {
+        runtime_warp_size = routed_moe_runtime_warp_size_value();
+    }
+#endif
+    const int hotlist_wmma_wave32 = runtime_warp_size == 32;
+    int iq2_q2_tail_cull_enable_value = -1;
+    int iq2_q2_tail_cull_disable_value = -1;
+    const int iq2_q2_tail_wave_cull = iq2_path && n_tokens > 1u
+        ? routed_moe_iq2_q2_tail_wave_cull_resolve(
+              runtime_warp_size,
+              &iq2_q2_tail_cull_enable_value,
+              &iq2_q2_tail_cull_disable_value)
+        : 0;
+    const int iq2_q2_wmma_profile_value = iq2_path && n_tokens > 1u
+        ? routed_moe_env_bool_value(
+              DS4_ROCM_IQ2_Q2_WMMA_PROFILE_ENV, 1u << 7)
+        : -1;
+    const int iq2_q2_wmma_profile = iq2_q2_wmma_profile_value == 1;
+    int iq2_q2_profile_gate_up_seen = 0;
+    int iq2_q2_profile_down_seen = 0;
+    float iq2_q2_profile_gate_up_ms = 0.0f;
+    float iq2_q2_profile_down_ms = 0.0f;
     uint64_t pair_count64 = 0;
     if (!cuda_u64_mul_checked(n_tokens, n_expert, &pair_count64) ||
         pair_count64 > UINT32_MAX) {
         return 0;
     }
     const uint32_t pair_count = (uint32_t)pair_count64;
+    if (iq2_q2_wmma_profile && !hotlist_wmma_wave32) {
+        return routed_moe_wmma_profile_emit(
+            1, iq2_q2_tail_wave_cull, n_tokens, pair_count64,
+            0, 0.0f, 0, 0.0f, 0);
+    }
     const ds4_gpu_tensor *selected_exec = selected;
     const char *gate_w = NULL;
     const char *up_w = NULL;
@@ -760,6 +1179,12 @@ static int routed_moe_launch(
         !cuda_u64_mul_checked(midq_count, sizeof(cuda_block_q8_K), &midq_bytes)) {
         return 0;
     }
+    if (iq2_q2_wmma_profile &&
+        (q2k_path || down->bytes < xq_bytes || gate->bytes < midq_bytes)) {
+        return routed_moe_wmma_profile_emit(
+            1, iq2_q2_tail_wave_cull, n_tokens, pair_count64,
+            0, 0.0f, 0, 0.0f, 0);
+    }
     if (!q2k_path && down->bytes >= xq_bytes && gate->bytes >= midq_bytes) {
         cuda_block_q8_K *xq = (cuda_block_q8_K *)down->ptr;
         cuda_block_q8_K *midq = (cuda_block_q8_K *)gate->ptr;
@@ -997,7 +1422,16 @@ static int routed_moe_launch(
                 }
             }
             if (ok) ok = cuda_stream_batch_selected_mark_inflight();
-            return ok;
+            return routed_moe_wmma_profile_emit(
+                iq2_q2_wmma_profile,
+                iq2_q2_tail_wave_cull,
+                n_tokens,
+                pair_count64,
+                iq2_q2_profile_gate_up_seen,
+                iq2_q2_profile_gate_up_ms,
+                iq2_q2_profile_down_seen,
+                iq2_q2_profile_down_ms,
+                ok);
         }
         if (ok && use_sorted_pairs) {
             const uint32_t bucket_count = n_total_expert;
@@ -1164,7 +1598,18 @@ static int routed_moe_launch(
             n_tokens >= iq2_gate_hot_threshold && n_expert == 6u && !write_gate_up &&
             sorted_pairs && sorted_offsets && sorted_counts && tile_experts && iq2_gate_hot_dev && use_expert_tiles &&
             (expert_in_dim % 16u) == 0u && (expert_mid_dim % 16u) == 0u &&
-            !g_quality_mode;
+            hotlist_wmma_wave32 && !g_quality_mode;
+        if (iq2_gate_path && n_tokens > 1u &&
+            getenv("DS4_ROCM_MOE_PATH_DEBUG") != NULL) {
+            fprintf(stderr,
+                    DS4_GPU_LOG_PREFIX "IQ2 hot-list selector runtime_warp=%d "
+                    "wmma_eligible=%u tail_policy=%d enable=%d disable=%d\n",
+                    runtime_warp_size,
+                    use_iq2_gate_wmma,
+                    iq2_q2_tail_wave_cull,
+                    iq2_q2_tail_cull_enable_value,
+                    iq2_q2_tail_cull_disable_value);
+        }
         if (use_iq2_gate_wmma) {
             uint32_t h_counts[DS4_ROCM_MAX_N_EXPERT] = {0};
             if (!cuda_ok(cudaMemcpy(h_counts, sorted_counts, n_total_expert * sizeof(uint32_t), cudaMemcpyDeviceToHost),
@@ -1661,6 +2106,15 @@ static int routed_moe_launch(
             ok = cuda_ok(cudaGetLastError(), "routed_moe gate/up launch");
 #if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__)
             if (ok && use_iq2_gate_wmma && iq2_gate_hot_count != 0u) {
+                routed_moe_wmma_profile_timer profile_timer = {};
+                if (iq2_q2_wmma_profile &&
+                    !routed_moe_wmma_profile_timer_begin(
+                        &profile_timer,
+                        "routed_moe iq2 wmma gate/up profile begin")) {
+                    return routed_moe_wmma_profile_emit(
+                        1, iq2_q2_tail_wave_cull, n_tokens, pair_count64,
+                        0, 0.0f, 0, 0.0f, 0);
+                }
                 constexpr uint32_t bm = 16u, bn = 16u, bk = 16u;
                 const uint32_t wmma_mtiles = 4u;
                 if (wmma_mtiles == 4u) {
@@ -1672,25 +2126,29 @@ static int routed_moe_launch(
                     const size_t shmem_n2 = (mt * bm * bk + 4u * bk * bn) * sizeof(half) +
                                             (4u * mt * bm * bn) * sizeof(float);
                     if (use_iq2_hot_f16_mid && use_iq2_x_f16) {
-                        moe_gate_up_mid_iq2_hotlist_wmma_n2_kernel<4,16,16,16,true,true><<<grid, block, shmem_n2>>>(
+                        routed_moe_launch_iq2_hotlist_wmma_n2<4,true,true>(
+                                grid, block, shmem_n2, iq2_q2_tail_wave_cull,
                                 NULL, iq2_hot_mid_h, gate_w, up_w, (const float *)x->ptr, iq2_x_h,
                                 (const float *)weights->ptr, sorted_counts, sorted_offsets, sorted_pairs,
                                 iq2_gate_hot_dev, iq2_gate_hot_count, expert_in_dim, expert_mid_dim,
                                 gate_expert_bytes, gate_row_bytes, clamp);
                     } else if (use_iq2_hot_f16_mid) {
-                        moe_gate_up_mid_iq2_hotlist_wmma_n2_kernel<4,16,16,16,true><<<grid, block, shmem_n2>>>(
+                        routed_moe_launch_iq2_hotlist_wmma_n2<4,true,false>(
+                                grid, block, shmem_n2, iq2_q2_tail_wave_cull,
                                 NULL, iq2_hot_mid_h, gate_w, up_w, (const float *)x->ptr, NULL,
                                 (const float *)weights->ptr, sorted_counts, sorted_offsets, sorted_pairs,
                                 iq2_gate_hot_dev, iq2_gate_hot_count, expert_in_dim, expert_mid_dim,
                                 gate_expert_bytes, gate_row_bytes, clamp);
                     } else if (use_iq2_x_f16) {
-                        moe_gate_up_mid_iq2_hotlist_wmma_n2_kernel<4,16,16,16,false,true><<<grid, block, shmem_n2>>>(
+                        routed_moe_launch_iq2_hotlist_wmma_n2<4,false,true>(
+                                grid, block, shmem_n2, iq2_q2_tail_wave_cull,
                                 (float *)mid->ptr, NULL, gate_w, up_w, (const float *)x->ptr, iq2_x_h,
                                 (const float *)weights->ptr, sorted_counts, sorted_offsets, sorted_pairs,
                                 iq2_gate_hot_dev, iq2_gate_hot_count, expert_in_dim, expert_mid_dim,
                                 gate_expert_bytes, gate_row_bytes, clamp);
                     } else {
-                        moe_gate_up_mid_iq2_hotlist_wmma_n2_kernel<4,16,16,16><<<grid, block, shmem_n2>>>(
+                        routed_moe_launch_iq2_hotlist_wmma_n2<4,false,false>(
+                                grid, block, shmem_n2, iq2_q2_tail_wave_cull,
                                 (float *)mid->ptr, NULL, gate_w, up_w, (const float *)x->ptr, NULL,
                                 (const float *)weights->ptr, sorted_counts, sorted_offsets, sorted_pairs,
                                 iq2_gate_hot_dev, iq2_gate_hot_count, expert_in_dim, expert_mid_dim,
@@ -1705,32 +2163,50 @@ static int routed_moe_launch(
                     const size_t shmem_n2 = (mt * bm * bk + 4u * bk * bn) * sizeof(half) +
                                             (4u * mt * bm * bn) * sizeof(float);
                     if (use_iq2_hot_f16_mid && use_iq2_x_f16) {
-                        moe_gate_up_mid_iq2_hotlist_wmma_n2_kernel<8,16,16,16,true,true><<<grid, block, shmem_n2>>>(
+                        routed_moe_launch_iq2_hotlist_wmma_n2<8,true,true>(
+                                grid, block, shmem_n2, iq2_q2_tail_wave_cull,
                                 NULL, iq2_hot_mid_h, gate_w, up_w, (const float *)x->ptr, iq2_x_h,
                                 (const float *)weights->ptr, sorted_counts, sorted_offsets, sorted_pairs,
                                 iq2_gate_hot_dev, iq2_gate_hot_count, expert_in_dim, expert_mid_dim,
                                 gate_expert_bytes, gate_row_bytes, clamp);
                     } else if (use_iq2_hot_f16_mid) {
-                        moe_gate_up_mid_iq2_hotlist_wmma_n2_kernel<8,16,16,16,true><<<grid, block, shmem_n2>>>(
+                        routed_moe_launch_iq2_hotlist_wmma_n2<8,true,false>(
+                                grid, block, shmem_n2, iq2_q2_tail_wave_cull,
                                 NULL, iq2_hot_mid_h, gate_w, up_w, (const float *)x->ptr, NULL,
                                 (const float *)weights->ptr, sorted_counts, sorted_offsets, sorted_pairs,
                                 iq2_gate_hot_dev, iq2_gate_hot_count, expert_in_dim, expert_mid_dim,
                                 gate_expert_bytes, gate_row_bytes, clamp);
                     } else if (use_iq2_x_f16) {
-                        moe_gate_up_mid_iq2_hotlist_wmma_n2_kernel<8,16,16,16,false,true><<<grid, block, shmem_n2>>>(
+                        routed_moe_launch_iq2_hotlist_wmma_n2<8,false,true>(
+                                grid, block, shmem_n2, iq2_q2_tail_wave_cull,
                                 (float *)mid->ptr, NULL, gate_w, up_w, (const float *)x->ptr, iq2_x_h,
                                 (const float *)weights->ptr, sorted_counts, sorted_offsets, sorted_pairs,
                                 iq2_gate_hot_dev, iq2_gate_hot_count, expert_in_dim, expert_mid_dim,
                                 gate_expert_bytes, gate_row_bytes, clamp);
                     } else {
-                        moe_gate_up_mid_iq2_hotlist_wmma_n2_kernel<8,16,16,16><<<grid, block, shmem_n2>>>(
+                        routed_moe_launch_iq2_hotlist_wmma_n2<8,false,false>(
+                                grid, block, shmem_n2, iq2_q2_tail_wave_cull,
                                 (float *)mid->ptr, NULL, gate_w, up_w, (const float *)x->ptr, NULL,
                                 (const float *)weights->ptr, sorted_counts, sorted_offsets, sorted_pairs,
                                 iq2_gate_hot_dev, iq2_gate_hot_count, expert_in_dim, expert_mid_dim,
                                 gate_expert_bytes, gate_row_bytes, clamp);
                     }
                 }
-                ok = cuda_ok(cudaGetLastError(), "routed_moe iq2 wmma hot gate/up launch");
+                const cudaError_t launch_err = cudaGetLastError();
+                ok = cuda_ok(
+                        launch_err,
+                        "routed_moe iq2 wmma hot gate/up launch");
+                if (!ok && iq2_q2_wmma_profile) {
+                    (void)routed_moe_wmma_profile_timer_destroy(
+                            &profile_timer,
+                            "routed_moe iq2 wmma gate/up profile cleanup");
+                } else if (iq2_q2_wmma_profile) {
+                    ok = routed_moe_wmma_profile_timer_finish(
+                            &profile_timer,
+                            &iq2_q2_profile_gate_up_ms,
+                            "routed_moe iq2 wmma gate/up profile finish");
+                    if (ok) iq2_q2_profile_gate_up_seen = 1;
+                }
             }
 #endif
         }
@@ -1853,6 +2329,10 @@ static int routed_moe_launch(
                     ok = routed_moe_q2_float_down_launch(
                             out, down, mid, iq2_hot_mid_h, use_iq2_hot_f16_mid, down_w,
                             sorted_counts, sorted_offsets, sorted_pairs, tile_experts,
+                            hotlist_wmma_wave32, iq2_q2_tail_wave_cull,
+                            iq2_q2_wmma_profile,
+                            &iq2_q2_profile_down_seen,
+                            &iq2_q2_profile_down_ms,
                             n_tokens, n_total_expert, n_expert, expert_mid_dim, out_dim,
                             down_expert_bytes, down_row_bytes);
                 }
@@ -2071,7 +2551,16 @@ static int routed_moe_launch(
             ok = cuda_ok(cudaGetLastError(), "routed_moe sum launch");
         }
         if (ok && compact_selected) ok = cuda_stream_selected_mark_inflight();
-        return ok;
+        return routed_moe_wmma_profile_emit(
+            iq2_q2_wmma_profile,
+            iq2_q2_tail_wave_cull,
+            n_tokens,
+            pair_count64,
+            iq2_q2_profile_gate_up_seen,
+            iq2_q2_profile_gate_up_ms,
+            iq2_q2_profile_down_seen,
+            iq2_q2_profile_down_ms,
+            ok);
     }
 
     const ds4_rocm_runtime_config *cfg = cuda_runtime_config();
@@ -2190,7 +2679,7 @@ static int routed_moe_launch(
         const uint64_t f16_low_gate_bytes = (uint64_t)bucket_count * sizeof(uint32_t);
         const uint64_t f16_low_down_bytes = (uint64_t)bucket_count * sizeof(uint32_t);
 #if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__)
-        const int moe_wmma_hot = !g_quality_mode &&
+        const int moe_wmma_hot = hotlist_wmma_wave32 && !g_quality_mode &&
                                  expert_in_dim % 16u == 0u &&
                                  expert_mid_dim % 16u == 0u &&
                                  out_dim % 16u == 0u;
diff --git a/rocm/ds4_rocm_norm_rope.cuh b/rocm/ds4_rocm_norm_rope.cuh
index b3f543b2fb..a7cb07e070 100644
--- a/rocm/ds4_rocm_norm_rope.cuh
+++ b/rocm/ds4_rocm_norm_rope.cuh
@@ -357,12 +357,6 @@ __device__ static float dsv4_e2m1fn_dequant_dev(float x) {
     return sign * dsv4_e2m1fn_value_dev(best);
 }
 
-__device__ static float model_scalar_dev(const void *base, uint64_t offset, uint32_t type, uint64_t idx) {
-    const char *p = (const char *)base + offset;
-    if (type == 1u) return __half2float(((const __half *)p)[idx]);
-    return ((const float *)p)[idx];
-}
-
 __device__ static float model_ape_value_dev(const void *base, uint64_t offset, uint32_t type,
                                             uint32_t width, uint32_t row, uint32_t col) {
     const char *p = (const char *)base + offset;
@@ -377,38 +371,6 @@ __device__ static float model_ape_value_dev(const void *base, uint64_t offset, u
     return ((const float *)p)[(uint64_t)row * width + col];
 }
 
-__device__ static float rope_yarn_ramp_cpu_equiv_dev(float low, float high, int i0) {
-    float y = ((float)(i0 / 2) - low) / fmaxf(0.001f, high - low);
-    return 1.0f - fminf(1.0f, fmaxf(0.0f, y));
-}
-
-__device__ static DS4_ROCM_UNUSED void rope_tail_one_dev(float *x, uint32_t head_dim, uint32_t n_rot, uint32_t pos, uint32_t n_ctx_orig, float freq_base, float freq_scale, float ext_factor, float attn_factor, float beta_fast, float beta_slow) {
-    uint32_t n_nope = head_dim - n_rot;
-    float corr0 = 0.0f, corr1 = 0.0f;
-    if (ext_factor != 0.0f) {
-        float denom = 2.0f * logf(freq_base);
-        corr0 = fmaxf(0.0f, floorf((float)n_rot * logf((float)n_ctx_orig / (beta_fast * 2.0f * (float)M_PI)) / denom));
-        corr1 = fminf((float)(n_rot - 1), ceilf((float)n_rot * logf((float)n_ctx_orig / (beta_slow * 2.0f * (float)M_PI)) / denom));
-    }
-    for (uint32_t i = 0; i < n_rot; i += 2) {
-        float theta_extrap = (float)pos * powf(freq_base, -((float)i) / (float)n_rot);
-        float theta_interp = freq_scale * theta_extrap;
-        float theta = theta_interp;
-        float mscale = attn_factor;
-        if (ext_factor != 0.0f) {
-            float mix = rope_yarn_ramp_cpu_equiv_dev(corr0, corr1, (int)i) * ext_factor;
-            theta = theta_interp * (1.0f - mix) + theta_extrap * mix;
-            mscale *= 1.0f + 0.1f * logf(1.0f / freq_scale);
-        }
-        float c = cosf(theta) * mscale;
-        float s = sinf(theta) * mscale;
-        float x0 = x[n_nope + i];
-        float x1 = x[n_nope + i + 1];
-        x[n_nope + i] = x0 * c - x1 * s;
-        x[n_nope + i + 1] = x0 * s + x1 * c;
-    }
-}
-
 extern "C" int ds4_gpu_rms_norm_plain_tensor(ds4_gpu_tensor *out, const ds4_gpu_tensor *x, uint32_t n, float eps) {
     if (!cuda_tensor_has_f32(out, n) || !cuda_tensor_has_f32(x, n)) return 0;
     if (n == 0u) return 1;
@@ -529,12 +491,439 @@ extern "C" int ds4_gpu_head_rms_norm_rope_tail_tensor(ds4_gpu_tensor *x, uint32_
     head_rms_norm_rope_tail_kernel<<<(uint32_t)rows64, 256>>>((float *)x->ptr, n_tok, n_head, head_dim, n_rot, pos0, n_ctx_orig, inverse ? 1 : 0, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow, eps);
     return cuda_ok(cudaGetLastError(), "head_rms_norm_rope_tail launch");
 }
+
+static int rocm_q4_attn_q_b_prefixes_overlap(
+        const void *a, uint64_t a_bytes,
+        const void *b, uint64_t b_bytes) {
+    const uintptr_t ap = reinterpret_cast<uintptr_t>(a);
+    const uintptr_t bp = reinterpret_cast<uintptr_t>(b);
+    return ap <= bp ? (uint64_t)(bp - ap) < a_bytes
+                    : (uint64_t)(ap - bp) < b_bytes;
+}
+
+static int rocm_q4_attn_q_b_f16_head_rms_rope_tail_tensor(
+        ds4_gpu_tensor       *out,
+        ds4_gpu_tensor       *q_half,
+        const void           *model_map,
+        uint64_t              model_size,
+        uint64_t              weight_offset,
+        uint64_t              in_dim,
+        uint64_t              out_dim,
+        const ds4_gpu_tensor *x,
+        uint32_t              n_tok,
+        uint32_t              n_head,
+        uint32_t              head_dim,
+        uint32_t              n_rot,
+        uint32_t              pos0,
+        uint32_t              n_ctx_orig,
+        bool                  inverse,
+        float                 freq_base,
+        float                 freq_scale,
+        float                 ext_factor,
+        float                 attn_factor,
+        float                 beta_fast,
+        float                 beta_slow,
+        float                 eps) {
+    /* Decode and tiny/final chunks retain the native Q4_K path without even
+     * taking the cache mutex. REQUIRE applies only to configured candidates. */
+    if (n_tok < 32u) return 0;
+    const int required = rocm_q4_attn_q_b_f16_required();
+    if (!rocm_q4_attn_q_b_f16_enabled() && !required) return 0;
+    if ((uint64_t)n_tok < rocm_q4_attn_q_b_f16_min_tokens()) return 0;
+    rocm_q4_attn_q_b_f16_note_candidate();
+    const int f16_output = rocm_q4_attn_q_b_f16_output_enabled();
+
+    uint64_t x_elems = 0;
+    uint64_t out_elems = 0;
+    uint64_t x_bytes = 0;
+    uint64_t out_bytes = 0;
+    uint64_t out_f16_bytes = 0;
+    uint64_t head_rows = 0;
+    if (!rocm_q4_attn_q_b_f16_policy_allowed() ||
+        rocm_q4_attn_q_b_f16_circuit_open() ||
+        !g_cublas_ready || !out || !out->ptr ||
+        !x || !x->ptr || !model_map ||
+        model_map != g_model_host_base ||
+        model_size != g_model_registered_size ||
+        in_dim != DS4_ROCM_Q4_ATTN_Q_B_IN_DIM ||
+        out_dim != DS4_ROCM_Q4_ATTN_Q_B_OUT_DIM ||
+        n_head == 0u || head_dim == 0u ||
+        out_dim != (uint64_t)n_head * head_dim ||
+        n_rot > head_dim || (n_rot & 1u) != 0u ||
+        n_tok > (uint32_t)INT_MAX ||
+        pos0 > (uint32_t)INT_MAX - n_tok ||
+        !cuda_u64_mul_checked(n_tok, in_dim, &x_elems) ||
+        !cuda_u64_mul_checked(n_tok, out_dim, &out_elems) ||
+        !cuda_u64_mul_checked(n_tok, n_head, &head_rows) ||
+        head_rows > UINT32_MAX ||
+        (x_elems + 255u) / 256u > UINT32_MAX ||
+        !cuda_u64_mul_checked(x_elems, sizeof(float), &x_bytes) ||
+        !cuda_u64_mul_checked(out_elems, sizeof(float), &out_bytes) ||
+        !cuda_u64_mul_checked(out_elems, sizeof(__half), &out_f16_bytes) ||
+        x->bytes < x_bytes || out->bytes < out_bytes) {
+        return rocm_q4_attn_q_b_f16_fallback(required, 1, 0);
+    }
+
+    const uint64_t blocks_per_row = in_dim / CUDA_QK_K;
+    uint64_t row_bytes = 0;
+    uint64_t weight_bytes = 0;
+    if (!cuda_u64_mul_checked(blocks_per_row,
+                              sizeof(cuda_block_q4_K), &row_bytes) ||
+        !cuda_u64_mul_checked(out_dim, row_bytes, &weight_bytes) ||
+        !cuda_model_range_fits(model_size, weight_offset, weight_bytes)) {
+        return rocm_q4_attn_q_b_f16_fallback(required, 1, 0);
+    }
+
+    __half *unused_weight_f16 = NULL;
+    __half *xh = NULL;
+    __half *q_scratch = NULL;
+    if (!rocm_q4_attn_q_b_transient_f16_acquire(
+            n_tok, f16_output,
+            &unused_weight_f16, &xh, &q_scratch)) {
+        return rocm_q4_attn_q_b_f16_fallback(required, 0, 0);
+    }
+    (void)unused_weight_f16;
+    const int use_graph_q_half =
+        f16_output && q_half && q_half->ptr &&
+        q_half->bytes >= out_f16_bytes &&
+        !rocm_q4_attn_q_b_prefixes_overlap(
+            q_half->ptr, out_f16_bytes, out->ptr, out_bytes) &&
+        !rocm_q4_attn_q_b_prefixes_overlap(
+            q_half->ptr, out_f16_bytes, x->ptr, x_bytes);
+    __half *const qh = use_graph_q_half
+                     ? (__half *)q_half->ptr : q_scratch;
+
+    const __half *w_f16 = rocm_q4_attn_q_b_f16_acquire(
+        model_map, model_size, weight_offset, weight_bytes, in_dim, out_dim,
+        DS4_ROCM_Q4_K_TYPE);
+    if (!w_f16) {
+        rocm_q4_attn_q_b_transient_f16_release_acquired();
+        return rocm_q4_attn_q_b_f16_fallback(required, 0, 0);
+    }
+
+    /* Keep both the persistent weight pin and the dedicated X/Q staging lock
+     * through the complete enqueue sequence. Lifecycle release takes the same
+     * locks before synchronizing, so it cannot miss the final epilogue. */
+    f32_to_f16_kernel<<<(x_elems + 255u) / 256u, 256>>>(
+        xh, (const float *)x->ptr, x_elems);
+    cudaError_t launch_err = cudaGetLastError();
+    if (launch_err != cudaSuccess) {
+        fprintf(stderr,
+                DS4_GPU_LOG_PREFIX
+                "Q4 attn q_b F16 activation conversion failed: %s\n",
+                cudaGetErrorString(launch_err));
+        (void)cudaGetLastError();
+        rocm_q4_attn_q_b_f16_release_acquired();
+        rocm_q4_attn_q_b_transient_f16_release_acquired();
+        return rocm_q4_attn_q_b_f16_fallback(required, 0, 1);
+    }
+
+    const float alpha = 1.0f;
+    const float beta = 0.0f;
+    /* The release path remains F32 by default. The explicit F16-output arm
+     * matches Q8's boundary and consumes either caller staging or ROCm-owned
+     * Q_F16 without materializing the large F32 Q. */
+    const cublasStatus_t st = cublasGemmEx(
+        g_cublas,
+        CUBLAS_OP_T,
+        CUBLAS_OP_N,
+        (int)out_dim,
+        (int)n_tok,
+        (int)in_dim,
+        &alpha,
+        w_f16,
+        CUDA_R_16F,
+        (int)in_dim,
+        xh,
+        CUDA_R_16F,
+        (int)in_dim,
+        &beta,
+        f16_output ? (void *)qh : out->ptr,
+        f16_output ? CUDA_R_16F : CUDA_R_32F,
+        (int)out_dim,
+        CUBLAS_COMPUTE_32F,
+        CUBLAS_GEMM_DEFAULT);
+    if (st != CUBLAS_STATUS_SUCCESS) {
+        fprintf(stderr,
+                DS4_GPU_LOG_PREFIX
+                DS4_GPU_BLAS_NAME
+                " cached Q4 attn q_b F16/F16-to-%s matmul failed: "
+                "status %d\n",
+                f16_output ? "F16" : "F32",
+                (int)st);
+        rocm_q4_attn_q_b_f16_release_acquired();
+        rocm_q4_attn_q_b_transient_f16_release_acquired();
+        return rocm_q4_attn_q_b_f16_fallback(required, 0, 1);
+    }
+
+    int tail_ok = 0;
+    if (f16_output) {
+        head_rms_norm_rope_tail_from_half_kernel<<<(uint32_t)head_rows, 256>>>(
+                (float *)out->ptr, qh,
+                n_tok, n_head, head_dim, n_rot, pos0, n_ctx_orig,
+                inverse ? 1 : 0, freq_base, freq_scale, ext_factor,
+                attn_factor, beta_fast, beta_slow, eps);
+        launch_err = cudaGetLastError();
+        tail_ok = launch_err == cudaSuccess;
+    } else {
+        tail_ok = ds4_gpu_head_rms_norm_rope_tail_tensor(
+            out, n_tok, n_head, head_dim, n_rot, pos0, n_ctx_orig,
+            inverse, freq_base, freq_scale, ext_factor, attn_factor,
+            beta_fast, beta_slow, eps);
+    }
+    rocm_q4_attn_q_b_f16_release_acquired();
+    rocm_q4_attn_q_b_transient_f16_release_acquired();
+    if (!tail_ok) {
+        if (f16_output) {
+            fprintf(stderr,
+                    DS4_GPU_LOG_PREFIX
+                    "cached Q4 attn q_b F16-out epilogue launch failed: %s\n",
+                    cudaGetErrorString(launch_err));
+            (void)cudaGetLastError();
+        }
+        /* GEMM has already accepted the output writer.  Never authorize the
+         * caller to replay native Q4 over an asynchronous/partial result. */
+        return rocm_q4_attn_q_b_f16_fallback(1, 0, 1);
+    }
+    return 1;
+}
+
+/* Resident default: expand only the current Q4_K q_b matrix into the shared
+ * 64 MiB W_F16 region, stage X_F16 beside it, and consume both inputs
+ * immediately with hipBLAS. The opt-in F16-output arm also stages Q_F16 before
+ * its fused epilogue. No caller may reuse the allocation between these steps. */
+static int rocm_q4_attn_q_b_transient_f16_head_rms_rope_tail_tensor(
+        ds4_gpu_tensor       *out,
+        ds4_gpu_tensor       *q_half,
+        const void           *model_map,
+        uint64_t              model_size,
+        uint64_t              weight_offset,
+        uint64_t              in_dim,
+        uint64_t              out_dim,
+        const ds4_gpu_tensor *x,
+        uint32_t              n_tok,
+        uint32_t              n_head,
+        uint32_t              head_dim,
+        uint32_t              n_rot,
+        uint32_t              pos0,
+        uint32_t              n_ctx_orig,
+        bool                  inverse,
+        float                 freq_base,
+        float                 freq_scale,
+        float                 ext_factor,
+        float                 attn_factor,
+        float                 beta_fast,
+        float                 beta_slow,
+        float                 eps) {
+    if ((uint64_t)n_tok <
+            rocm_q4_attn_q_b_transient_f16_min_tokens()) {
+        return 0;
+    }
+    rocm_q4_attn_q_b_f16_note_candidate();
+    const int f16_output = rocm_q4_attn_q_b_f16_output_enabled();
+
+    uint64_t x_elems = 0;
+    uint64_t out_elems = 0;
+    uint64_t head_rows = 0;
+    uint64_t x_bytes = 0;
+    uint64_t out_bytes = 0;
+    uint64_t out_f16_bytes = 0;
+    if (!rocm_q4_attn_q_b_transient_f16_policy_allowed() ||
+        rocm_q4_attn_q_b_f16_circuit_open() ||
+        !g_cublas_ready || !out || !out->ptr ||
+        !x || !x->ptr || !model_map ||
+        model_map != g_model_host_base ||
+        model_size != g_model_registered_size ||
+        in_dim != DS4_ROCM_Q4_ATTN_Q_B_IN_DIM ||
+        out_dim != DS4_ROCM_Q4_ATTN_Q_B_OUT_DIM ||
+        n_head == 0u || head_dim == 0u ||
+        out_dim != (uint64_t)n_head * head_dim ||
+        n_rot > head_dim || (n_rot & 1u) != 0u ||
+        n_tok > (uint32_t)INT_MAX ||
+        pos0 > (uint32_t)INT_MAX - n_tok ||
+        !cuda_u64_mul_checked(n_tok, in_dim, &x_elems) ||
+        !cuda_u64_mul_checked(n_tok, out_dim, &out_elems) ||
+        !cuda_u64_mul_checked(n_tok, n_head, &head_rows) ||
+        head_rows > UINT32_MAX ||
+        (x_elems + 255u) / 256u > UINT32_MAX ||
+        !cuda_u64_mul_checked(x_elems, sizeof(float), &x_bytes) ||
+        !cuda_u64_mul_checked(out_elems, sizeof(float), &out_bytes) ||
+        !cuda_u64_mul_checked(out_elems, sizeof(__half), &out_f16_bytes) ||
+        x->bytes < x_bytes || out->bytes < out_bytes) {
+        return rocm_q4_attn_q_b_f16_fallback(0, 1, 0);
+    }
+
+    uint64_t row_bytes = 0;
+    uint64_t weight_bytes = 0;
+    const uint64_t blocks_per_row = in_dim / CUDA_QK_K;
+    if (!cuda_u64_mul_checked(blocks_per_row,
+                              sizeof(cuda_block_q4_K), &row_bytes) ||
+        !cuda_u64_mul_checked(out_dim, row_bytes, &weight_bytes) ||
+        !cuda_model_range_fits(model_size, weight_offset, weight_bytes)) {
+        return rocm_q4_attn_q_b_f16_fallback(0, 1, 0);
+    }
+
+    __half *w_f16 = NULL;
+    __half *x_f16 = NULL;
+    __half *q_scratch = NULL;
+    if (!rocm_q4_attn_q_b_transient_f16_acquire(
+            n_tok, f16_output, &w_f16, &x_f16, &q_scratch)) {
+        return rocm_q4_attn_q_b_f16_fallback(0, 0, 0);
+    }
+    const int use_graph_q_half =
+        f16_output && q_half && q_half->ptr &&
+        q_half->bytes >= out_f16_bytes &&
+        !rocm_q4_attn_q_b_prefixes_overlap(
+            q_half->ptr, out_f16_bytes, out->ptr, out_bytes) &&
+        !rocm_q4_attn_q_b_prefixes_overlap(
+            q_half->ptr, out_f16_bytes, x->ptr, x_bytes);
+    __half *const qh = use_graph_q_half
+                     ? (__half *)q_half->ptr : q_scratch;
+
+    const char *w_q4 = rocm_q4_attn_q_b_device_resident_source(
+        model_map, weight_offset, weight_bytes);
+    if (!w_q4) {
+        rocm_q4_attn_q_b_transient_f16_release_acquired();
+        return rocm_q4_attn_q_b_f16_fallback(0, 0, 1);
+    }
+
+    const uint64_t total_chunks = out_dim * (in_dim / 16u);
+    rocm_dequant_q4_K_attn_q_b_f16_kernel<<<
+        (uint32_t)((total_chunks + 255u) / 256u), 256>>>(
+            w_f16, (const cuda_block_q4_K *)w_q4,
+            in_dim, out_dim, blocks_per_row);
+    cudaError_t launch_err = cudaGetLastError();
+    if (launch_err != cudaSuccess) {
+        fprintf(stderr,
+                DS4_GPU_LOG_PREFIX
+                "Q4 attn_q_b transient dequant launch failed: %s\n",
+                cudaGetErrorString(launch_err));
+        (void)cudaGetLastError();
+        rocm_q4_attn_q_b_transient_f16_release_acquired();
+        return rocm_q4_attn_q_b_f16_fallback(0, 0, 1);
+    }
+
+    f32_to_f16_kernel<<<(x_elems + 255u) / 256u, 256>>>(
+        x_f16, (const float *)x->ptr, x_elems);
+    launch_err = cudaGetLastError();
+    if (launch_err != cudaSuccess) {
+        fprintf(stderr,
+                DS4_GPU_LOG_PREFIX
+                "Q4 attn_q_b transient activation conversion failed: %s\n",
+                cudaGetErrorString(launch_err));
+        (void)cudaGetLastError();
+        rocm_q4_attn_q_b_transient_f16_release_acquired();
+        return rocm_q4_attn_q_b_f16_fallback(0, 0, 1);
+    }
+
+    const float alpha = 1.0f;
+    const float beta = 0.0f;
+    /* Keep the release path's F32 output by default. The explicit F16-output
+     * experiment uses the same boundary for cached and transient weights. */
+    const cublasStatus_t st = cublasGemmEx(
+        g_cublas,
+        CUBLAS_OP_T,
+        CUBLAS_OP_N,
+        (int)out_dim,
+        (int)n_tok,
+        (int)in_dim,
+        &alpha,
+        w_f16,
+        CUDA_R_16F,
+        (int)in_dim,
+        x_f16,
+        CUDA_R_16F,
+        (int)in_dim,
+        &beta,
+        f16_output ? (void *)qh : out->ptr,
+        f16_output ? CUDA_R_16F : CUDA_R_32F,
+        (int)out_dim,
+        CUBLAS_COMPUTE_32F,
+        CUBLAS_GEMM_DEFAULT);
+    if (st != CUBLAS_STATUS_SUCCESS) {
+        fprintf(stderr,
+                DS4_GPU_LOG_PREFIX
+                DS4_GPU_BLAS_NAME
+                " transient Q4 attn q_b F16/F16-to-%s matmul failed: "
+                "status %d\n",
+                f16_output ? "F16" : "F32",
+                (int)st);
+        rocm_q4_attn_q_b_transient_f16_release_acquired();
+        return rocm_q4_attn_q_b_f16_fallback(0, 0, 1);
+    }
+
+    int tail_ok = 0;
+    if (f16_output) {
+        head_rms_norm_rope_tail_from_half_kernel<<<(uint32_t)head_rows, 256>>>(
+                (float *)out->ptr, qh,
+                n_tok, n_head, head_dim, n_rot, pos0, n_ctx_orig,
+                inverse ? 1 : 0, freq_base, freq_scale, ext_factor,
+                attn_factor, beta_fast, beta_slow, eps);
+        launch_err = cudaGetLastError();
+        tail_ok = launch_err == cudaSuccess;
+    } else {
+        tail_ok = ds4_gpu_head_rms_norm_rope_tail_tensor(
+            out, n_tok, n_head, head_dim, n_rot, pos0, n_ctx_orig,
+            inverse, freq_base, freq_scale, ext_factor, attn_factor,
+            beta_fast, beta_slow, eps);
+    }
+    rocm_q4_attn_q_b_transient_f16_release_acquired();
+    if (!tail_ok) {
+        if (f16_output) {
+            fprintf(stderr,
+                    DS4_GPU_LOG_PREFIX
+                    "transient Q4 attn q_b F16-out epilogue launch failed: %s\n",
+                    cudaGetErrorString(launch_err));
+            (void)cudaGetLastError();
+        }
+        /* GEMM was accepted and may already be executing.  Returning zero
+         * would make the graph replay native Q4 over an in-flight writer. */
+        return rocm_q4_attn_q_b_f16_fallback(1, 0, 1);
+    }
+    return 1;
+}
+
+/* A strict Q8_K-wave32 request belongs to the exact Q4 path.  The F16
+ * sidecars must yield before validation, allocation, dequantization or GEMM;
+ * the caller can then enter the canonical Q4 fallback, whose selector either
+ * launches the required quantizer or fails closed on an unsupported device. */
+enum {
+    ROCM_Q4_ATTN_Q_B_Q8_WAVE32_CONFLICT = -1,
+    ROCM_Q4_ATTN_Q_B_Q8_WAVE32_KEEP_F16 = 0,
+    ROCM_Q4_ATTN_Q_B_Q8_WAVE32_YIELD = 1,
+};
+
+static int rocm_q4_attn_q_b_required_q8_wave32_policy(
+        uint32_t weight_type,
+        uint32_t n_tok,
+        int q8_wave32_required,
+        int f16_cache_required) {
+    if (weight_type != DS4_ROCM_Q4_K_TYPE || n_tok <= 8u ||
+        !q8_wave32_required) {
+        return ROCM_Q4_ATTN_Q_B_Q8_WAVE32_KEEP_F16;
+    }
+    return f16_cache_required
+        ? ROCM_Q4_ATTN_Q_B_Q8_WAVE32_CONFLICT
+        : ROCM_Q4_ATTN_Q_B_Q8_WAVE32_YIELD;
+}
+
+extern "C" int ds4_rocm_test_q4_attn_q_b_yield_to_q8_wave32_policy(
+        uint32_t weight_type,
+        uint32_t n_tok,
+        int q8_wave32_required,
+        int f16_cache_required) {
+    return rocm_q4_attn_q_b_required_q8_wave32_policy(
+        weight_type, n_tok, q8_wave32_required != 0,
+        f16_cache_required != 0);
+}
+
 extern "C" int ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor(
         ds4_gpu_tensor       *out,
         ds4_gpu_tensor       *q_half,
         const void           *model_map,
         uint64_t              model_size,
         uint64_t              weight_offset,
+        uint32_t              weight_type,
         uint64_t              in_dim,
         uint64_t              out_dim,
         const ds4_gpu_tensor *x,
@@ -552,7 +941,41 @@ extern "C" int ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor(
         float                 beta_fast,
         float                 beta_slow,
         float                 eps) {
-    if (!g_cublas_ready || !out || !q_half || !x || !model_map || n_tok == 0 ||
+    const int q8_wave32_required = rocm_q4_attn_q_b_env_bool(
+        "DS4_ROCM_REQUIRE_Q4_PREFILL_Q8_K_WAVE32") == 1;
+    const int f16_cache_required = rocm_q4_attn_q_b_f16_required();
+    const int q8_wave32_policy =
+        rocm_q4_attn_q_b_required_q8_wave32_policy(
+            weight_type, n_tok, q8_wave32_required, f16_cache_required);
+    if (q8_wave32_policy == ROCM_Q4_ATTN_Q_B_Q8_WAVE32_CONFLICT) {
+        fprintf(stderr,
+                DS4_GPU_LOG_PREFIX
+                "Q4 attn_q_b prefill cannot require both the F16 cache "
+                "and the Q8_K wave32 quantizer\n");
+        return -1;
+    }
+    if (q8_wave32_policy == ROCM_Q4_ATTN_Q_B_Q8_WAVE32_YIELD) {
+        return 0;
+    }
+    if (weight_type == DS4_ROCM_Q4_K_TYPE) {
+        const int persistent_requested =
+            f16_cache_required ||
+            (rocm_q4_attn_q_b_f16_enabled() &&
+             !rocm_q4_attn_q_b_f16_disabled());
+        if (persistent_requested) {
+            return rocm_q4_attn_q_b_f16_head_rms_rope_tail_tensor(
+                out, q_half, model_map, model_size, weight_offset,
+                in_dim, out_dim, x, n_tok, n_head, head_dim, n_rot, pos0,
+                n_ctx_orig, inverse, freq_base, freq_scale, ext_factor,
+                attn_factor, beta_fast, beta_slow, eps);
+        }
+        return rocm_q4_attn_q_b_transient_f16_head_rms_rope_tail_tensor(
+            out, q_half, model_map, model_size, weight_offset,
+            in_dim, out_dim, x, n_tok, n_head, head_dim, n_rot, pos0,
+            n_ctx_orig, inverse, freq_base, freq_scale, ext_factor,
+            attn_factor, beta_fast, beta_slow, eps);
+    }
+    if (weight_type != 8u || !g_cublas_ready || !out || !q_half || !x || !model_map || n_tok == 0 ||
         n_rot > head_dim || (n_rot & 1u) || out_dim != (uint64_t)n_head * head_dim ||
         x->bytes < (uint64_t)n_tok * in_dim * sizeof(float) ||
         out->bytes < (uint64_t)n_tok * out_dim * sizeof(float) ||
diff --git a/rocm/ds4_rocm_q4.cuh b/rocm/ds4_rocm_q4.cuh
new file mode 100644
index 0000000000..d1bf027e55
--- /dev/null
+++ b/rocm/ds4_rocm_q4.cuh
@@ -0,0 +1,2999 @@
+// DS4 ROCm dense Q4_K kernels and launch wrappers.
+//
+// This module is included after ds4_rocm_moe.cuh so it can reuse the
+// canonical Q8_K activation quantizer, Q4_K/Q8_K block dot product, and
+// quarter-wave reduction already used by the routed-MoE implementation.
+// Launches intentionally stay on ROCm's default stream: cuda_tmp_alloc() is a
+// single reusable scratch arena whose lifetime is protected by that ordering.
+
+static_assert(sizeof(cuda_block_q4_K) == 144u,
+              "ROCm Q4_K block layout must match GGUF");
+static_assert(sizeof(cuda_block_q8_K) == 292u,
+              "ROCm Q8_K activation block layout must match the dot kernel");
+
+__global__ static void rocm_matmul_q4_K_dense_kernel(
+        float *out,
+        const char *w_base,
+        const cuda_block_q8_K *xq,
+        uint64_t row_bytes,
+        uint32_t xq_blocks,
+        uint32_t out_dim,
+        uint32_t n_tok) {
+    const uint32_t lane = threadIdx.x & 7u;
+    const uint32_t row_lane = threadIdx.x >> 3u;
+    const uint32_t tok = blockIdx.y;
+    const uint32_t row = blockIdx.x * 32u + row_lane;
+    if (tok >= n_tok || row >= out_dim) return;
+
+    const cuda_block_q8_K *xqb = xq + (uint64_t)tok * xq_blocks;
+    const cuda_block_q4_K *wr = reinterpret_cast<const cuda_block_q4_K *>(
+            w_base + (uint64_t)row * row_bytes);
+    float acc = 0.0f;
+    for (uint32_t b = lane; b < xq_blocks; b += 8u) {
+        acc += dev_dot_q4_K_q8_K_block(wr + b, xqb + b);
+    }
+    acc = quarter_warp_sum_f32(acc, lane);
+    if (lane == 0u) out[(uint64_t)tok * out_dim + row] = acc;
+}
+
+/* Latency-oriented pair variant of the canonical dense kernel.  Concatenating
+ * the two row-tile domains keeps exactly the same per-row dot and reduction
+ * order while sharing one launch between Q-A and KV. */
+__global__ static void rocm_matmul_q4_K_dense_pair_kernel(
+        float *out0,
+        float *out1,
+        const char *w0,
+        const char *w1,
+        const cuda_block_q8_K *xq,
+        uint64_t row_bytes,
+        uint32_t xq_blocks,
+        uint32_t out0_dim,
+        uint32_t out1_dim,
+        uint32_t n_tok) {
+    const uint32_t lane = threadIdx.x & 7u;
+    const uint32_t row_lane = threadIdx.x >> 3u;
+    const uint32_t out0_tiles = (out0_dim - 1u) / 32u + 1u;
+    const bool second = blockIdx.x >= out0_tiles;
+    const uint32_t row_tile = second ? blockIdx.x - out0_tiles : blockIdx.x;
+    const uint32_t row = row_tile * 32u + row_lane;
+    const uint32_t tok = blockIdx.y;
+    const uint32_t out_dim = second ? out1_dim : out0_dim;
+    if (tok >= n_tok || row >= out_dim) return;
+
+    float *const out = second ? out1 : out0;
+    const char *const w_base = second ? w1 : w0;
+    const cuda_block_q8_K *xqb = xq + (uint64_t)tok * xq_blocks;
+    const cuda_block_q4_K *wr = reinterpret_cast<const cuda_block_q4_K *>(
+            w_base + (uint64_t)row * row_bytes);
+    float acc = 0.0f;
+    for (uint32_t b = lane; b < xq_blocks; b += 8u) {
+        acc += dev_dot_q4_K_q8_K_block(wr + b, xqb + b);
+    }
+    acc = quarter_warp_sum_f32(acc, lane);
+    if (lane == 0u) out[(uint64_t)tok * out_dim + row] = acc;
+}
+
+/* One independent activation row and Q4_K matrix per output group.  This is
+ * the canonical dense decode walk with only a group grid dimension added. */
+__global__ static void rocm_matmul_q4_K_dense_grouped_decode_kernel(
+        float *out, const char *w_base, const cuda_block_q8_K *xq,
+        uint64_t row_bytes, uint32_t xq_blocks, uint32_t out_dim,
+        uint32_t n_groups) {
+    const uint32_t lane = threadIdx.x & 7u;
+    const uint32_t row_lane = threadIdx.x >> 3u;
+    const uint32_t row = blockIdx.x * 32u + row_lane;
+    const uint32_t group = blockIdx.z;
+    if (group >= n_groups || row >= out_dim) return;
+    const cuda_block_q8_K *xqb = xq + (uint64_t)group * xq_blocks;
+    const cuda_block_q4_K *wr = reinterpret_cast<const cuda_block_q4_K *>(
+        w_base + ((uint64_t)group * out_dim + row) * row_bytes);
+    float acc = 0.0f;
+    for (uint32_t b = lane; b < xq_blocks; b += 8u) {
+        acc += dev_dot_q4_K_q8_K_block(wr + b, xqb + b);
+    }
+    acc = quarter_warp_sum_f32(acc, lane);
+    if (lane == 0u) out[(uint64_t)group * out_dim + row] = acc;
+}
+
+/* Dense Q4_K prefill tile for RDNA/ROCm.
+ *
+ * The legacy kernel gives one eight-lane group a single (token,row) dot.  As
+ * a result every token walks the complete Q4_K row independently and every
+ * one of the 32 row groups in a workgroup also fetches the same Q8_K input
+ * blocks.  Prefill is therefore dominated by redundant global reads.
+ *
+ * This kernel keeps the legacy eight-lane block assignment and reduction
+ * order, but computes eight token columns at once.  A Q4_K block is decoded
+ * once into eight integer dot products, and an 8x8 token/K-block tile of the
+ * canonical Q8_K activations is staged in LDS for reuse by all 32 rows.  The
+ * K loop advances in groups of eight so lane L still accumulates blocks
+ * L,L+8,... in exactly the order used by rocm_matmul_q4_K_dense_kernel.
+ *
+ * LDS footprint: 8 tokens * 8 K blocks * 292 bytes = 18,688 bytes.
+ * The final, partial token and K tiles are handled without an early return;
+ * every thread must reach both barriers.
+ */
+enum {
+    ROCM_Q4_PREFILL_TOKEN_TILE = 8u,
+    ROCM_Q4_PREFILL_KBLOCK_TILE = 8u,
+    ROCM_Q4_PREFILL_K1024_KBLOCK_TILE = 4u,
+    ROCM_Q4_PREFILL_K1024_ROWS = 64u,
+    ROCM_Q4_Q8K_WORDS = sizeof(cuda_block_q8_K) / sizeof(uint32_t),
+};
+static_assert((sizeof(cuda_block_q8_K) % sizeof(uint32_t)) == 0u,
+              "ROCm Q8_K LDS copies require a whole number of words");
+
+#if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__)
+/* Direct-Q4 WMMA for resident gfx1151 prefill.
+ *
+ * This deliberately mirrors the live Q8 WMMA kernel: each wave32 owns 16
+ * output rows, the workgroup computes 64 tokens, K advances by 32, only the
+ * 64x32 activation tile is staged in 4 KiB of LDS, and accumulator fragments
+ * are written straight to the output.  The row tile is shape-selected to 64,
+ * 128, or 256 so every activation tile is reused as broadly as the output
+ * shape permits; the retained 64-row instantiation is also an A/B control.
+ * K64 can stage two adjacent F32 activations per loader iteration, matching
+ * the established Q8 WMMA load/conversion pattern; aligned 256-row K128
+ * launches stage four.  Each lane
+ * dequantizes one Q4_K row/group directly into two F16 register vectors, so
+ * there is no Q8_K activation scratch and no persistent F16 weight sidecar.
+ *
+ * Arithmetic is not bit-identical to Q4_K x Q8_K: activations and transient
+ * weights round to F16 before F32 WMMA accumulation.  Host policy therefore
+ * enables this path by default only for physically device-resident, gfx1151
+ * wave32 prefill outside quality mode.  SSD streaming remains separately
+ * gated, and DISABLE provides an authoritative rollback.
+ */
+enum {
+    ROCM_Q4_WMMA_TOKEN_TILE = 64u,
+    ROCM_Q4_WMMA_K_TILE = 32u,
+    ROCM_Q4_WMMA_K64_TILE = 64u,
+    ROCM_Q4_WMMA_K64_LDS_PITCH = 80u,
+    ROCM_Q4_WMMA_K128_TILE = 128u,
+    ROCM_Q4_WMMA_K128_LDS_PITCH = 144u,
+    ROCM_Q4_WMMA_FRAGMENT = 16u,
+};
+static_assert(ROCM_Q4_WMMA_K64_TILE == 2u * ROCM_Q4_WMMA_K_TILE,
+              "K64 must combine exactly two adjacent Q4_K qgroups");
+static_assert(ROCM_Q4_WMMA_K64_LDS_PITCH >= ROCM_Q4_WMMA_K64_TILE &&
+              (ROCM_Q4_WMMA_K64_LDS_PITCH % ROCM_Q4_WMMA_FRAGMENT) == 0u,
+              "K64 LDS rows must preserve aligned half16 WMMA loads");
+static_assert(ROCM_Q4_WMMA_K128_TILE == 2u * ROCM_Q4_WMMA_K64_TILE,
+              "K128 must combine exactly four adjacent Q4_K qgroups");
+static_assert(ROCM_Q4_WMMA_K128_LDS_PITCH >= ROCM_Q4_WMMA_K128_TILE &&
+              (ROCM_Q4_WMMA_K128_LDS_PITCH % ROCM_Q4_WMMA_FRAGMENT) == 0u,
+              "K128 LDS rows must preserve aligned half16 WMMA loads");
+
+#if defined(__HIP_DEVICE_COMPILE__) && __HIP_DEVICE_COMPILE__ && \
+    defined(__gfx1151__) && \
+    (!defined(__AMDGCN_WAVEFRONT_SIZE__) || \
+     __AMDGCN_WAVEFRONT_SIZE__ == 32)
+#define DS4_ROCM_Q4_GFX1151_WMMA_ROWTILE_DEVICE 1
+typedef _Float16 __attribute__((ext_vector_type(16))) ds4_q4_half16_t;
+typedef float __attribute__((ext_vector_type(8))) ds4_q4_float8_t;
+typedef uint8_t __attribute__((ext_vector_type(16))) ds4_q4_uchar16_t;
+#else
+#define DS4_ROCM_Q4_GFX1151_WMMA_ROWTILE_DEVICE 0
+#endif
+
+template <uint32_t ROW_TILE, uint32_t WAVES, uint32_t MIN_BLOCKS,
+          bool LOAD2>
+__launch_bounds__(WAVES * 32u, MIN_BLOCKS)
+__global__ static void rocm_matmul_q4_K_prefill_wmma_rowtile_strided_kernel(
+        float *out,
+        const char *w_base,
+        const float *x,
+        uint32_t n_tok,
+        uint32_t n_groups,
+        uint32_t in_dim,
+        uint32_t out_dim,
+        uint64_t row_bytes,
+        uint64_t x_token_stride,
+        uint64_t x_group_stride,
+        uint64_t out_token_stride) {
+#if DS4_ROCM_Q4_GFX1151_WMMA_ROWTILE_DEVICE
+    if (warpSize != 32) return;
+
+    const uint32_t tid = threadIdx.x;
+    const uint32_t wave = tid >> 5u;
+    const uint32_t lane = tid & 31u;
+    const uint32_t lane16 = lane & 15u;
+    const uint32_t group = blockIdx.z;
+    static_assert(ROW_TILE == WAVES * ROCM_Q4_WMMA_FRAGMENT,
+                  "one Q4 WMMA wave must own exactly 16 output rows");
+    const uint32_t row0 = blockIdx.x * ROW_TILE;
+    const uint32_t tok0 = blockIdx.y * ROCM_Q4_WMMA_TOKEN_TILE;
+    if (group >= n_groups) return;
+
+    const uint32_t wave_row0 = row0 + wave * ROCM_Q4_WMMA_FRAGMENT;
+    const uint32_t my_row = wave_row0 + lane16;
+    const uint32_t safe_row = my_row < out_dim ? my_row : out_dim - 1u;
+    const cuda_block_q4_K *row_blocks =
+        reinterpret_cast<const cuda_block_q4_K *>(
+            w_base + ((uint64_t)group * out_dim + safe_row) * row_bytes);
+    const uint32_t q4_blocks = in_dim / CUDA_QK_K;
+
+    ds4_q4_float8_t acc0 = {0.0f, 0.0f, 0.0f, 0.0f,
+                            0.0f, 0.0f, 0.0f, 0.0f};
+    ds4_q4_float8_t acc1 = acc0;
+    ds4_q4_float8_t acc2 = acc0;
+    ds4_q4_float8_t acc3 = acc0;
+    /* The 16-lane _Float16 vector loads below have 32-byte alignment on the
+     * AMDGPU target; half2 itself would require only four bytes. */
+    __shared__ __align__(32) _Float16
+        lds_x[ROCM_Q4_WMMA_TOKEN_TILE * ROCM_Q4_WMMA_K_TILE];
+
+    for (uint32_t block_index = 0u; block_index < q4_blocks;
+         block_index++) {
+        const cuda_block_q4_K *block = row_blocks + block_index;
+        const float block_d = dev_f16_to_f32(block->d);
+        const float block_dm = dev_f16_to_f32(block->dmin);
+#pragma unroll
+        for (uint32_t qpair = 0u; qpair < 4u; qpair++) {
+            /* Adjacent 32-value groups are the low/high nibbles of the same
+             * 32 payload bytes.  Keep them in registers across both K tiles
+             * instead of issuing the same global loads twice. */
+            ds4_q4_uchar16_t packed0;
+            ds4_q4_uchar16_t packed1;
+            __builtin_memcpy(
+                &packed0, block->qs + qpair * 32u, sizeof(packed0));
+            __builtin_memcpy(
+                &packed1,
+                block->qs + qpair * 32u + ROCM_Q4_WMMA_FRAGMENT,
+                sizeof(packed1));
+
+            #pragma unroll
+            for (uint32_t nibble = 0u; nibble < 2u; nibble++) {
+                const uint32_t qgroup = qpair * 2u + nibble;
+                const uint32_t group32 = block_index * 8u + qgroup;
+                if constexpr (LOAD2) {
+                    for (uint32_t j = tid * 2u;
+                         j < ROCM_Q4_WMMA_TOKEN_TILE * ROCM_Q4_WMMA_K_TILE;
+                         j += blockDim.x * 2u) {
+                        const uint32_t tok_local = j >> 5u;
+                        const uint32_t kk = j & 31u;
+                        const uint32_t tok = tok0 + tok_local;
+                        half2 value = __floats2half2_rn(0.0f, 0.0f);
+                        if (tok < n_tok) {
+                            const float2 pair =
+                                *reinterpret_cast<const float2 *>(
+                                    x + (uint64_t)tok * x_token_stride +
+                                    (uint64_t)group * x_group_stride +
+                                    (uint64_t)group32 *
+                                        ROCM_Q4_WMMA_K_TILE +
+                                    kk);
+                            value = __floats2half2_rn(pair.x, pair.y);
+                        }
+                        *reinterpret_cast<half2 *>(lds_x + j) = value;
+                    }
+                } else {
+                    for (uint32_t j = tid;
+                         j < ROCM_Q4_WMMA_TOKEN_TILE * ROCM_Q4_WMMA_K_TILE;
+                         j += blockDim.x) {
+                        const uint32_t tok_local = j >> 5u;
+                        const uint32_t kk = j & 31u;
+                        const uint32_t tok = tok0 + tok_local;
+                        float value = 0.0f;
+                        if (tok < n_tok) {
+                            value = x[(uint64_t)tok * x_token_stride +
+                                      (uint64_t)group * x_group_stride +
+                                      (uint64_t)group32 *
+                                          ROCM_Q4_WMMA_K_TILE +
+                                      kk];
+                        }
+                        lds_x[j] = (_Float16)value;
+                    }
+                }
+                __syncthreads();
+
+                uint8_t scale = 0u;
+                uint8_t minimum = 0u;
+                dev_q4_K_get_scale_min(
+                    qgroup, block->scales, &scale, &minimum);
+                const float d = block_d * (float)scale;
+                const float dm = block_dm * (float)minimum;
+                const uint32_t shift = nibble * 4u;
+                ds4_q4_half16_t weights0;
+                ds4_q4_half16_t weights1;
+                #pragma unroll
+                for (uint32_t i = 0u; i < ROCM_Q4_WMMA_FRAGMENT; i++) {
+                    const uint8_t q0 = (packed0[i] >> shift) & 0x0fu;
+                    const uint8_t q1 = (packed1[i] >> shift) & 0x0fu;
+                    weights0[i] = (_Float16)(d * (float)q0 - dm);
+                    weights1[i] = (_Float16)(d * (float)q1 - dm);
+                }
+
+                #pragma unroll
+                for (uint32_t token_tile = 0u; token_tile < 4u;
+                     token_tile++) {
+                    const uint32_t token_local =
+                        token_tile * ROCM_Q4_WMMA_FRAGMENT + lane16;
+                    const _Float16 *activation =
+                        lds_x + token_local * ROCM_Q4_WMMA_K_TILE;
+                    const ds4_q4_half16_t activation0 =
+                        *reinterpret_cast<const ds4_q4_half16_t *>(activation);
+                    const ds4_q4_half16_t activation1 =
+                        *reinterpret_cast<const ds4_q4_half16_t *>(
+                            activation + ROCM_Q4_WMMA_FRAGMENT);
+                    if (token_tile == 0u) {
+                        acc0 = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(
+                            weights0, activation0, acc0);
+                        acc0 = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(
+                            weights1, activation1, acc0);
+                    } else if (token_tile == 1u) {
+                        acc1 = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(
+                            weights0, activation0, acc1);
+                        acc1 = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(
+                            weights1, activation1, acc1);
+                    } else if (token_tile == 2u) {
+                        acc2 = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(
+                            weights0, activation0, acc2);
+                        acc2 = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(
+                            weights1, activation1, acc2);
+                    } else {
+                        acc3 = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(
+                            weights0, activation0, acc3);
+                        acc3 = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(
+                            weights1, activation1, acc3);
+                    }
+                }
+                __syncthreads();
+            }
+        }
+    }
+
+    #pragma unroll
+    for (uint32_t token_tile = 0u; token_tile < 4u; token_tile++) {
+        const uint32_t tok =
+            tok0 + token_tile * ROCM_Q4_WMMA_FRAGMENT + lane16;
+        if (tok >= n_tok) continue;
+        const ds4_q4_float8_t acc = token_tile == 0u
+            ? acc0
+            : (token_tile == 1u ? acc1
+                                : (token_tile == 2u ? acc2 : acc3));
+        #pragma unroll
+        for (uint32_t j = 0u; j < 8u; j++) {
+            const uint32_t row = wave_row0 + 2u * j + (lane >> 4u);
+            if (row < out_dim) {
+                out[(uint64_t)tok * out_token_stride +
+                    (uint64_t)group * out_dim + row] = acc[j];
+            }
+        }
+    }
+#else
+    (void)out;
+    (void)w_base;
+    (void)x;
+    (void)n_tok;
+    (void)n_groups;
+    (void)in_dim;
+    (void)out_dim;
+    (void)row_bytes;
+    (void)x_token_stride;
+    (void)x_group_stride;
+    (void)out_token_stride;
+#endif
+}
+
+/* Default K64 variant.  The K32 kernel above intentionally remains
+ * byte-for-byte unchanged so its code generation stays a stable rollback and
+ * A/B baseline.  K64 stages two adjacent 32-value activation groups
+ * at once and consumes the low/high Q4 nibbles in their original order,
+ * reducing the LDS barrier pairs from sixteen to eight per Q4_K block.
+ *
+ * A padded 80-half LDS pitch keeps every 16-half WMMA load 32-byte aligned
+ * while rotating successive token rows across LDS banks.  Natural pitch 64
+ * would map every 128-byte row to the same bank pattern on RDNA wave32. */
+template <uint32_t ROW_TILE, uint32_t WAVES, uint32_t MIN_BLOCKS,
+          bool LOAD2>
+__launch_bounds__(WAVES * 32u, MIN_BLOCKS)
+__global__ static void
+rocm_matmul_q4_K_prefill_wmma_k64_p80_rowtile_strided_kernel(
+        float *out,
+        const char *w_base,
+        const float *x,
+        uint32_t n_tok,
+        uint32_t n_groups,
+        uint32_t in_dim,
+        uint32_t out_dim,
+        uint64_t row_bytes,
+        uint64_t x_token_stride,
+        uint64_t x_group_stride,
+        uint64_t out_token_stride) {
+#if DS4_ROCM_Q4_GFX1151_WMMA_ROWTILE_DEVICE
+    if (warpSize != 32) return;
+
+    const uint32_t tid = threadIdx.x;
+    const uint32_t wave = tid >> 5u;
+    const uint32_t lane = tid & 31u;
+    const uint32_t lane16 = lane & 15u;
+    const uint32_t group = blockIdx.z;
+    static_assert(ROW_TILE == WAVES * ROCM_Q4_WMMA_FRAGMENT,
+                  "one Q4 WMMA wave must own exactly 16 output rows");
+    const uint32_t row0 = blockIdx.x * ROW_TILE;
+    const uint32_t tok0 = blockIdx.y * ROCM_Q4_WMMA_TOKEN_TILE;
+    if (group >= n_groups) return;
+
+    const uint32_t wave_row0 = row0 + wave * ROCM_Q4_WMMA_FRAGMENT;
+    const uint32_t my_row = wave_row0 + lane16;
+    const uint32_t safe_row = my_row < out_dim ? my_row : out_dim - 1u;
+    const cuda_block_q4_K *row_blocks =
+        reinterpret_cast<const cuda_block_q4_K *>(
+            w_base + ((uint64_t)group * out_dim + safe_row) * row_bytes);
+    const uint32_t q4_blocks = in_dim / CUDA_QK_K;
+
+    ds4_q4_float8_t acc0 = {0.0f, 0.0f, 0.0f, 0.0f,
+                            0.0f, 0.0f, 0.0f, 0.0f};
+    ds4_q4_float8_t acc1 = acc0;
+    ds4_q4_float8_t acc2 = acc0;
+    ds4_q4_float8_t acc3 = acc0;
+    __shared__ __align__(32) _Float16
+        lds_x[ROCM_Q4_WMMA_TOKEN_TILE * ROCM_Q4_WMMA_K64_LDS_PITCH];
+
+    for (uint32_t block_index = 0u; block_index < q4_blocks;
+         block_index++) {
+        const cuda_block_q4_K *block = row_blocks + block_index;
+        const float block_d = dev_f16_to_f32(block->d);
+        const float block_dm = dev_f16_to_f32(block->dmin);
+#pragma unroll
+        for (uint32_t qpair = 0u; qpair < 4u; qpair++) {
+            ds4_q4_uchar16_t packed0;
+            ds4_q4_uchar16_t packed1;
+            __builtin_memcpy(
+                &packed0, block->qs + qpair * 32u, sizeof(packed0));
+            __builtin_memcpy(
+                &packed1,
+                block->qs + qpair * 32u + ROCM_Q4_WMMA_FRAGMENT,
+                sizeof(packed1));
+
+            const uint32_t group32_base = block_index * 8u + qpair * 2u;
+            if constexpr (LOAD2) {
+                for (uint32_t j = tid * 2u;
+                     j < ROCM_Q4_WMMA_TOKEN_TILE * ROCM_Q4_WMMA_K64_TILE;
+                     j += blockDim.x * 2u) {
+                    const uint32_t tok_local = j >> 6u;
+                    const uint32_t kk = j & 63u;
+                    const uint32_t tok = tok0 + tok_local;
+                    half2 value = __floats2half2_rn(0.0f, 0.0f);
+                    if (tok < n_tok) {
+                        const float2 pair =
+                            *reinterpret_cast<const float2 *>(
+                                x + (uint64_t)tok * x_token_stride +
+                                (uint64_t)group * x_group_stride +
+                                (uint64_t)group32_base *
+                                    ROCM_Q4_WMMA_K_TILE +
+                                kk);
+                        value = __floats2half2_rn(pair.x, pair.y);
+                    }
+                    *reinterpret_cast<half2 *>(
+                        lds_x + tok_local * ROCM_Q4_WMMA_K64_LDS_PITCH +
+                        kk) = value;
+                }
+            } else {
+                for (uint32_t j = tid;
+                     j < ROCM_Q4_WMMA_TOKEN_TILE * ROCM_Q4_WMMA_K64_TILE;
+                     j += blockDim.x) {
+                    const uint32_t tok_local = j >> 6u;
+                    const uint32_t kk = j & 63u;
+                    const uint32_t tok = tok0 + tok_local;
+                    float value = 0.0f;
+                    if (tok < n_tok) {
+                        value = x[(uint64_t)tok * x_token_stride +
+                                  (uint64_t)group * x_group_stride +
+                                  (uint64_t)group32_base *
+                                      ROCM_Q4_WMMA_K_TILE +
+                                  kk];
+                    }
+                    lds_x[tok_local * ROCM_Q4_WMMA_K64_LDS_PITCH + kk] =
+                        (_Float16)value;
+                }
+            }
+            __syncthreads();
+
+            /* Do not unroll the two nibbles: one pair of half16 weight
+             * vectors must die before the next one is materialized.  This
+             * caps VGPR pressure while preserving qgroup accumulation order. */
+#pragma unroll 1
+            for (uint32_t nibble = 0u; nibble < 2u; nibble++) {
+                const uint32_t qgroup = qpair * 2u + nibble;
+                uint8_t scale = 0u;
+                uint8_t minimum = 0u;
+                dev_q4_K_get_scale_min(
+                    qgroup, block->scales, &scale, &minimum);
+                const float d = block_d * (float)scale;
+                const float dm = block_dm * (float)minimum;
+                const uint32_t shift = nibble * 4u;
+                ds4_q4_half16_t weights0;
+                ds4_q4_half16_t weights1;
+#pragma unroll
+                for (uint32_t i = 0u; i < ROCM_Q4_WMMA_FRAGMENT; i++) {
+                    const uint8_t q0 = (packed0[i] >> shift) & 0x0fu;
+                    const uint8_t q1 = (packed1[i] >> shift) & 0x0fu;
+                    weights0[i] = (_Float16)(d * (float)q0 - dm);
+                    weights1[i] = (_Float16)(d * (float)q1 - dm);
+                }
+
+#pragma unroll
+                for (uint32_t token_tile = 0u; token_tile < 4u;
+                     token_tile++) {
+                    const uint32_t token_local =
+                        token_tile * ROCM_Q4_WMMA_FRAGMENT + lane16;
+                    const _Float16 *activation =
+                        lds_x +
+                        token_local * ROCM_Q4_WMMA_K64_LDS_PITCH +
+                        nibble * ROCM_Q4_WMMA_K_TILE;
+                    const ds4_q4_half16_t activation0 =
+                        *reinterpret_cast<const ds4_q4_half16_t *>(activation);
+                    const ds4_q4_half16_t activation1 =
+                        *reinterpret_cast<const ds4_q4_half16_t *>(
+                            activation + ROCM_Q4_WMMA_FRAGMENT);
+                    if (token_tile == 0u) {
+                        acc0 = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(
+                            weights0, activation0, acc0);
+                        acc0 = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(
+                            weights1, activation1, acc0);
+                    } else if (token_tile == 1u) {
+                        acc1 = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(
+                            weights0, activation0, acc1);
+                        acc1 = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(
+                            weights1, activation1, acc1);
+                    } else if (token_tile == 2u) {
+                        acc2 = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(
+                            weights0, activation0, acc2);
+                        acc2 = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(
+                            weights1, activation1, acc2);
+                    } else {
+                        acc3 = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(
+                            weights0, activation0, acc3);
+                        acc3 = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(
+                            weights1, activation1, acc3);
+                    }
+                }
+            }
+            __syncthreads();
+        }
+    }
+
+#pragma unroll
+    for (uint32_t token_tile = 0u; token_tile < 4u; token_tile++) {
+        const uint32_t tok =
+            tok0 + token_tile * ROCM_Q4_WMMA_FRAGMENT + lane16;
+        if (tok >= n_tok) continue;
+        const ds4_q4_float8_t acc = token_tile == 0u
+            ? acc0
+            : (token_tile == 1u ? acc1
+                                : (token_tile == 2u ? acc2 : acc3));
+#pragma unroll
+        for (uint32_t j = 0u; j < 8u; j++) {
+            const uint32_t row = wave_row0 + 2u * j + (lane >> 4u);
+            if (row < out_dim) {
+                out[(uint64_t)tok * out_token_stride +
+                    (uint64_t)group * out_dim + row] = acc[j];
+            }
+        }
+    }
+#else
+    (void)out;
+    (void)w_base;
+    (void)x;
+    (void)n_tok;
+    (void)n_groups;
+    (void)in_dim;
+    (void)out_dim;
+    (void)row_bytes;
+    (void)x_token_stride;
+    (void)x_group_stride;
+    (void)out_token_stride;
+#endif
+}
+
+/* Long-prefill q_b K128 stage.  Four adjacent 32-value activation groups are
+ * staged behind one barrier pair, reducing the K64/P80 synchronization count
+ * by another 2x.  P144 retains a 32-byte-aligned half16 base while rotating
+ * consecutive token rows across LDS banks.  float4 loads also halve the
+ * activation-load instruction count relative to K64's float2 loader without
+ * changing any F32->F16 rounding or the qgroup accumulation order.
+ *
+ * Keep this as a separate kernel and instantiate it only for the 256-row
+ * geometry: its 18 KiB LDS tile is appropriate for q_b's 32768 output rows,
+ * but could reduce occupancy on the smaller projections. */
+template <uint32_t ROW_TILE, uint32_t WAVES, uint32_t MIN_BLOCKS>
+__launch_bounds__(WAVES * 32u, MIN_BLOCKS)
+__global__ static void
+rocm_matmul_q4_K_prefill_wmma_k128_p144_rowtile_strided_kernel(
+        float *out,
+        const char *w_base,
+        const float *x,
+        uint32_t n_tok,
+        uint32_t n_groups,
+        uint32_t in_dim,
+        uint32_t out_dim,
+        uint64_t row_bytes,
+        uint64_t x_token_stride,
+        uint64_t x_group_stride,
+        uint64_t out_token_stride) {
+#if DS4_ROCM_Q4_GFX1151_WMMA_ROWTILE_DEVICE
+    if (warpSize != 32) return;
+
+    const uint32_t tid = threadIdx.x;
+    const uint32_t wave = tid >> 5u;
+    const uint32_t lane = tid & 31u;
+    const uint32_t lane16 = lane & 15u;
+    const uint32_t group = blockIdx.z;
+    static_assert(ROW_TILE == WAVES * ROCM_Q4_WMMA_FRAGMENT,
+                  "one Q4 WMMA wave must own exactly 16 output rows");
+    const uint32_t row0 = blockIdx.x * ROW_TILE;
+    const uint32_t tok0 = blockIdx.y * ROCM_Q4_WMMA_TOKEN_TILE;
+    if (group >= n_groups) return;
+
+    const uint32_t wave_row0 = row0 + wave * ROCM_Q4_WMMA_FRAGMENT;
+    const uint32_t my_row = wave_row0 + lane16;
+    const uint32_t safe_row = my_row < out_dim ? my_row : out_dim - 1u;
+    const cuda_block_q4_K *row_blocks =
+        reinterpret_cast<const cuda_block_q4_K *>(
+            w_base + ((uint64_t)group * out_dim + safe_row) * row_bytes);
+    const uint32_t q4_blocks = in_dim / CUDA_QK_K;
+
+    ds4_q4_float8_t acc0 = {0.0f, 0.0f, 0.0f, 0.0f,
+                            0.0f, 0.0f, 0.0f, 0.0f};
+    ds4_q4_float8_t acc1 = acc0;
+    ds4_q4_float8_t acc2 = acc0;
+    ds4_q4_float8_t acc3 = acc0;
+    __shared__ __align__(32) _Float16
+        lds_x[ROCM_Q4_WMMA_TOKEN_TILE * ROCM_Q4_WMMA_K128_LDS_PITCH];
+
+    for (uint32_t block_index = 0u; block_index < q4_blocks;
+         block_index++) {
+        const cuda_block_q4_K *block = row_blocks + block_index;
+        const float block_d = dev_f16_to_f32(block->d);
+        const float block_dm = dev_f16_to_f32(block->dmin);
+
+        /* Each Q4_K block contains eight qgroups.  Stage groups 0..3 and
+         * 4..7 in two passes; the nested loops still consume qgroups in the
+         * exact 0,1,...,7 order used by K32 and K64. */
+#pragma unroll
+        for (uint32_t qpair_base = 0u; qpair_base < 4u;
+             qpair_base += 2u) {
+            const uint32_t group32_base =
+                block_index * 8u + qpair_base * 2u;
+            for (uint32_t j = tid * 4u;
+                 j < ROCM_Q4_WMMA_TOKEN_TILE * ROCM_Q4_WMMA_K128_TILE;
+                 j += blockDim.x * 4u) {
+                const uint32_t tok_local = j >> 7u;
+                const uint32_t kk = j & 127u;
+                const uint32_t tok = tok0 + tok_local;
+                half2 value01 = __floats2half2_rn(0.0f, 0.0f);
+                half2 value23 = value01;
+                if (tok < n_tok) {
+                    const float4 values =
+                        *reinterpret_cast<const float4 *>(
+                            x + (uint64_t)tok * x_token_stride +
+                            (uint64_t)group * x_group_stride +
+                            (uint64_t)group32_base *
+                                ROCM_Q4_WMMA_K_TILE +
+                            kk);
+                    value01 = __floats2half2_rn(values.x, values.y);
+                    value23 = __floats2half2_rn(values.z, values.w);
+                }
+                _Float16 *const dst =
+                    lds_x +
+                    tok_local * ROCM_Q4_WMMA_K128_LDS_PITCH + kk;
+                *reinterpret_cast<half2 *>(dst) = value01;
+                *reinterpret_cast<half2 *>(dst + 2u) = value23;
+            }
+            __syncthreads();
+
+            /* Do not retain both packed qpair payloads at once.  Their
+             * lifetime would increase VGPR pressure in the 16-wave q_b
+             * workgroup and erase the synchronization win. */
+#pragma unroll 1
+            for (uint32_t qpair_offset = 0u; qpair_offset < 2u;
+                 qpair_offset++) {
+                const uint32_t qpair = qpair_base + qpair_offset;
+                ds4_q4_uchar16_t packed0;
+                ds4_q4_uchar16_t packed1;
+                __builtin_memcpy(
+                    &packed0, block->qs + qpair * 32u, sizeof(packed0));
+                __builtin_memcpy(
+                    &packed1,
+                    block->qs + qpair * 32u + ROCM_Q4_WMMA_FRAGMENT,
+                    sizeof(packed1));
+
+#pragma unroll 1
+                for (uint32_t nibble = 0u; nibble < 2u; nibble++) {
+                    const uint32_t qgroup = qpair * 2u + nibble;
+                    uint8_t scale = 0u;
+                    uint8_t minimum = 0u;
+                    dev_q4_K_get_scale_min(
+                        qgroup, block->scales, &scale, &minimum);
+                    const float d = block_d * (float)scale;
+                    const float dm = block_dm * (float)minimum;
+                    const uint32_t shift = nibble * 4u;
+                    ds4_q4_half16_t weights0;
+                    ds4_q4_half16_t weights1;
+#pragma unroll
+                    for (uint32_t i = 0u; i < ROCM_Q4_WMMA_FRAGMENT; i++) {
+                        const uint8_t q0 = (packed0[i] >> shift) & 0x0fu;
+                        const uint8_t q1 = (packed1[i] >> shift) & 0x0fu;
+                        weights0[i] = (_Float16)(d * (float)q0 - dm);
+                        weights1[i] = (_Float16)(d * (float)q1 - dm);
+                    }
+
+#pragma unroll
+                    for (uint32_t token_tile = 0u; token_tile < 4u;
+                         token_tile++) {
+                        const uint32_t token_local =
+                            token_tile * ROCM_Q4_WMMA_FRAGMENT + lane16;
+                        const uint32_t activation_offset =
+                            (qpair_offset * 2u + nibble) *
+                            ROCM_Q4_WMMA_K_TILE;
+                        const _Float16 *activation =
+                            lds_x +
+                            token_local * ROCM_Q4_WMMA_K128_LDS_PITCH +
+                            activation_offset;
+                        const ds4_q4_half16_t activation0 =
+                            *reinterpret_cast<const ds4_q4_half16_t *>(
+                                activation);
+                        const ds4_q4_half16_t activation1 =
+                            *reinterpret_cast<const ds4_q4_half16_t *>(
+                                activation + ROCM_Q4_WMMA_FRAGMENT);
+                        if (token_tile == 0u) {
+                            acc0 =
+                                __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(
+                                    weights0, activation0, acc0);
+                            acc0 =
+                                __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(
+                                    weights1, activation1, acc0);
+                        } else if (token_tile == 1u) {
+                            acc1 =
+                                __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(
+                                    weights0, activation0, acc1);
+                            acc1 =
+                                __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(
+                                    weights1, activation1, acc1);
+                        } else if (token_tile == 2u) {
+                            acc2 =
+                                __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(
+                                    weights0, activation0, acc2);
+                            acc2 =
+                                __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(
+                                    weights1, activation1, acc2);
+                        } else {
+                            acc3 =
+                                __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(
+                                    weights0, activation0, acc3);
+                            acc3 =
+                                __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(
+                                    weights1, activation1, acc3);
+                        }
+                    }
+                }
+            }
+            __syncthreads();
+        }
+    }
+
+#pragma unroll
+    for (uint32_t token_tile = 0u; token_tile < 4u; token_tile++) {
+        const uint32_t tok =
+            tok0 + token_tile * ROCM_Q4_WMMA_FRAGMENT + lane16;
+        if (tok >= n_tok) continue;
+        const ds4_q4_float8_t acc = token_tile == 0u
+            ? acc0
+            : (token_tile == 1u ? acc1
+                                : (token_tile == 2u ? acc2 : acc3));
+#pragma unroll
+        for (uint32_t j = 0u; j < 8u; j++) {
+            const uint32_t row = wave_row0 + 2u * j + (lane >> 4u);
+            if (row < out_dim) {
+                out[(uint64_t)tok * out_token_stride +
+                    (uint64_t)group * out_dim + row] = acc[j];
+            }
+        }
+    }
+#else
+    (void)out;
+    (void)w_base;
+    (void)x;
+    (void)n_tok;
+    (void)n_groups;
+    (void)in_dim;
+    (void)out_dim;
+    (void)row_bytes;
+    (void)x_token_stride;
+    (void)x_group_stride;
+    (void)out_token_stride;
+#endif
+}
+#undef DS4_ROCM_Q4_GFX1151_WMMA_ROWTILE_DEVICE
+#endif
+
+__device__ __forceinline__ static void
+rocm_dot_q4_K_q8_K_block8_reuse_weights(
+        const cuda_block_q4_K *x,
+        const cuda_block_q8_K *y0,
+        const cuda_block_q8_K *y1,
+        const cuda_block_q8_K *y2,
+        const cuda_block_q8_K *y3,
+        const cuda_block_q8_K *y4,
+        const cuda_block_q8_K *y5,
+        const cuda_block_q8_K *y6,
+        const cuda_block_q8_K *y7,
+        uint32_t n,
+        float acc[ROCM_Q4_PREFILL_TOKEN_TILE]) {
+    const cuda_block_q8_K *ys[ROCM_Q4_PREFILL_TOKEN_TILE] = {
+        y0, y1, y2, y3, y4, y5, y6, y7,
+    };
+    const float xd = dev_f16_to_f32(x->d);
+    const float xmin = dev_f16_to_f32(x->dmin);
+    int32_t isum[ROCM_Q4_PREFILL_TOKEN_TILE] = {0, 0, 0, 0, 0, 0, 0, 0};
+    int32_t summs[ROCM_Q4_PREFILL_TOKEN_TILE] = {0, 0, 0, 0, 0, 0, 0, 0};
+
+    /* A 32-byte Q4 payload stores the low and high nibbles for two adjacent
+     * 32-value groups.  Load those eight packed words once, then reuse them
+     * for both groups and every token in the tile.  This makes weight reuse
+     * explicit instead of relying on the compiler or vector cache to hoist
+     * repeated loads out of the token loop. */
+    #pragma unroll
+    for (uint32_t jp = 0u; jp < 4u; jp++) {
+        const uint32_t j0 = 2u * jp;
+        const uint32_t j1 = j0 + 1u;
+        uint8_t sc0, m0, sc1, m1;
+        dev_q4_K_get_scale_min(j0, x->scales, &sc0, &m0);
+        dev_q4_K_get_scale_min(j1, x->scales, &sc1, &m1);
+
+        int32_t qw[8];
+        #pragma unroll
+        for (uint32_t i = 0u; i < 8u; i++) {
+            qw[i] = *reinterpret_cast<const int32_t *>(
+                x->qs + jp * 32u + i * 4u);
+        }
+
+        #pragma unroll
+        for (uint32_t p = 0u; p < ROCM_Q4_PREFILL_TOKEN_TILE; p++) {
+            if (p < n) {
+                const cuda_block_q8_K *y = ys[p];
+                int32_t dot0 = 0;
+                int32_t dot1 = 0;
+                #pragma unroll
+                for (uint32_t i = 0u; i < 8u; i++) {
+                    const int32_t w0 = qw[i] & 0x0f0f0f0f;
+                    const int32_t w1 = (qw[i] >> 4) & 0x0f0f0f0f;
+                    dot0 = __dp4a(w0, *reinterpret_cast<const int32_t *>(
+                                           y->qs + j0 * 32u + i * 4u), dot0);
+                    dot1 = __dp4a(w1, *reinterpret_cast<const int32_t *>(
+                                           y->qs + j1 * 32u + i * 4u), dot1);
+                }
+                isum[p] += (int32_t)sc0 * dot0;
+                isum[p] += (int32_t)sc1 * dot1;
+                summs[p] += (int32_t)m0 *
+                    (int32_t)(y->bsums[2u * j0] + y->bsums[2u * j0 + 1u]);
+                summs[p] += (int32_t)m1 *
+                    (int32_t)(y->bsums[2u * j1] + y->bsums[2u * j1 + 1u]);
+            }
+        }
+    }
+
+    #pragma unroll
+    for (uint32_t p = 0u; p < ROCM_Q4_PREFILL_TOKEN_TILE; p++) {
+        if (p < n) {
+            const float yd = ys[p]->d;
+            acc[p] += yd * xd * (float)isum[p] -
+                      yd * xmin * (float)summs[p];
+        }
+    }
+}
+
+/* K=1024 has exactly four Q8_K blocks.  The generic TILE8 kernel leaves half
+ * of each eight-lane row group idle and still reserves LDS for eight blocks.
+ * Four-lane groups preserve the legacy block/reduction order while doubling
+ * the rows produced by a 256-thread workgroup and halving the LDS footprint
+ * to 8 tokens * 4 K blocks * 292 bytes = 9,344 bytes. */
+__device__ __forceinline__ static float
+rocm_q4_K_lane4_sum_f32(float v) {
+    /* Build the active-lane mask relative to the physical wave.  A 32-bit
+     * mask repeats lanes 0..31 for the upper half of an AMD wave64 and
+     * violates HIP's __shfl_down_sync contract even though width=4 keeps the
+     * data exchange inside the intended subgroup. */
+    const uint32_t wave_lane = threadIdx.x & (warpSize - 1u);
+    const MASK_T mask = static_cast<MASK_T>(0x0fu) << (wave_lane & ~3u);
+    v += __shfl_down_sync(mask, v, 2, 4);
+    v += __shfl_down_sync(mask, v, 1, 4);
+    return v;
+}
+
+__global__ static void rocm_matmul_q4_K_prefill_k1024_tile4_kernel(
+        float *out,
+        const char *w_base,
+        const cuda_block_q8_K *xq,
+        uint64_t row_bytes,
+        uint32_t out_dim,
+        uint32_t n_tok) {
+    __shared__ cuda_block_q8_K sxq[ROCM_Q4_PREFILL_TOKEN_TILE]
+                                         [ROCM_Q4_PREFILL_K1024_KBLOCK_TILE];
+
+    const uint32_t tid = threadIdx.x;
+    const uint32_t lane = tid & 3u;
+    const uint32_t row_lane = tid >> 2u;
+    const uint32_t row = blockIdx.x * ROCM_Q4_PREFILL_K1024_ROWS + row_lane;
+    const uint32_t tok0 = blockIdx.y * ROCM_Q4_PREFILL_TOKEN_TILE;
+    const uint32_t nt = n_tok - tok0 < ROCM_Q4_PREFILL_TOKEN_TILE
+                      ? n_tok - tok0 : ROCM_Q4_PREFILL_TOKEN_TILE;
+    const bool row_valid = row < out_dim;
+    const cuda_block_q4_K *wr = row_valid
+        ? reinterpret_cast<const cuda_block_q4_K *>(
+              w_base + (uint64_t)row * row_bytes)
+        : NULL;
+    float acc[ROCM_Q4_PREFILL_TOKEN_TILE] = {
+        0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f,
+    };
+
+    /* The complete K dimension fits one LDS tile.  Flatten the copy so
+     * neighboring threads read consecutive words across token/block rows. */
+    const uint32_t tile_words = nt * ROCM_Q4_PREFILL_K1024_KBLOCK_TILE *
+                                ROCM_Q4_Q8K_WORDS;
+    uint32_t *const sxq_words = reinterpret_cast<uint32_t *>(sxq);
+    for (uint32_t i = tid; i < tile_words; i += blockDim.x) {
+        const uint32_t block_slot = i / ROCM_Q4_Q8K_WORDS;
+        const uint32_t word = i - block_slot * ROCM_Q4_Q8K_WORDS;
+        const uint32_t p = block_slot >> 2u;
+        const uint32_t bb = block_slot & 3u;
+        const uint32_t *const src_words =
+            reinterpret_cast<const uint32_t *>(
+                xq + (uint64_t)(tok0 + p) *
+                          ROCM_Q4_PREFILL_K1024_KBLOCK_TILE + bb);
+        sxq_words[i] = src_words[word];
+    }
+    __syncthreads();
+
+    if (row_valid) {
+        rocm_dot_q4_K_q8_K_block8_reuse_weights(
+            wr + lane,
+            sxq[0] + lane, sxq[1] + lane,
+            sxq[2] + lane, sxq[3] + lane,
+            sxq[4] + lane, sxq[5] + lane,
+            sxq[6] + lane, sxq[7] + lane,
+            nt, acc);
+
+        #pragma unroll
+        for (uint32_t p = 0u; p < ROCM_Q4_PREFILL_TOKEN_TILE; p++) {
+            if (p < nt) {
+                const float v = rocm_q4_K_lane4_sum_f32(acc[p]);
+                if (lane == 0u) {
+                    out[(uint64_t)(tok0 + p) * out_dim + row] = v;
+                }
+            }
+        }
+    }
+}
+
+__global__ static void rocm_matmul_q4_K_prefill_tile8_strided_kernel(
+        float *out,
+        const char *w_base,
+        const cuda_block_q8_K *xq,
+        uint64_t row_bytes,
+        uint32_t xq_blocks,
+        uint32_t out_dim,
+        uint32_t n_tok,
+        uint64_t xq_token_stride,
+        uint64_t out_token_stride) {
+    __shared__ cuda_block_q8_K sxq[ROCM_Q4_PREFILL_TOKEN_TILE]
+                                         [ROCM_Q4_PREFILL_KBLOCK_TILE];
+
+    const uint32_t tid = threadIdx.x;
+    const uint32_t lane = tid & 7u;
+    const uint32_t row_lane = tid >> 3u;
+    const uint32_t row = blockIdx.x * 32u + row_lane;
+    const uint32_t tok0 = blockIdx.y * ROCM_Q4_PREFILL_TOKEN_TILE;
+    const uint32_t group = blockIdx.z;
+    const uint32_t nt = n_tok - tok0 < ROCM_Q4_PREFILL_TOKEN_TILE
+                      ? n_tok - tok0 : ROCM_Q4_PREFILL_TOKEN_TILE;
+    const bool row_valid = row < out_dim;
+    const cuda_block_q4_K *wr = row_valid
+        ? reinterpret_cast<const cuda_block_q4_K *>(
+              w_base + ((uint64_t)group * out_dim + row) * row_bytes)
+        : NULL;
+    float acc[ROCM_Q4_PREFILL_TOKEN_TILE] = {
+        0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f,
+    };
+
+    for (uint32_t b0 = 0u; b0 < xq_blocks;
+         b0 += ROCM_Q4_PREFILL_KBLOCK_TILE) {
+        const uint32_t nb = xq_blocks - b0 < ROCM_Q4_PREFILL_KBLOCK_TILE
+                          ? xq_blocks - b0
+                          : ROCM_Q4_PREFILL_KBLOCK_TILE;
+
+        /* Copy consecutive 32-bit words cooperatively.  Assigning one 292-B
+         * struct per lane makes adjacent lanes issue 292-B-strided global
+         * loads; flattening the packed blocks gives the memory coalescer long
+         * contiguous runs while preserving the fixed eight-block LDS layout. */
+        const uint32_t tile_words = nt * ROCM_Q4_PREFILL_KBLOCK_TILE *
+                                    ROCM_Q4_Q8K_WORDS;
+        uint32_t *const sxq_words = reinterpret_cast<uint32_t *>(sxq);
+        for (uint32_t i = tid; i < tile_words; i += blockDim.x) {
+            const uint32_t block_slot = i / ROCM_Q4_Q8K_WORDS;
+            const uint32_t word = i - block_slot * ROCM_Q4_Q8K_WORDS;
+            const uint32_t p = block_slot >> 3u;
+            const uint32_t bb = block_slot & 7u;
+            if (bb < nb) {
+                const uint64_t src_block =
+                    (uint64_t)(tok0 + p) * xq_token_stride +
+                    (uint64_t)group * xq_blocks + b0 + bb;
+                const uint32_t *const src_words =
+                    reinterpret_cast<const uint32_t *>(xq + src_block);
+                sxq_words[i] = src_words[word];
+            }
+        }
+        __syncthreads();
+
+        if (row_valid && lane < nb) {
+            rocm_dot_q4_K_q8_K_block8_reuse_weights(
+                wr + b0 + lane,
+                sxq[0] + lane, sxq[1] + lane,
+                sxq[2] + lane, sxq[3] + lane,
+                sxq[4] + lane, sxq[5] + lane,
+                sxq[6] + lane, sxq[7] + lane,
+                nt, acc);
+        }
+        __syncthreads();
+    }
+
+    if (row_valid) {
+        #pragma unroll
+        for (uint32_t p = 0u; p < ROCM_Q4_PREFILL_TOKEN_TILE; p++) {
+            if (p < nt) {
+                const float v = quarter_warp_sum_f32(acc[p], lane);
+                if (lane == 0u) {
+                    out[(uint64_t)(tok0 + p) * out_token_stride +
+                        (uint64_t)group * out_dim + row] = v;
+                }
+            }
+        }
+    }
+}
+
+/* Two independent dense projections over the same activation tile.  The
+ * row-tile ranges are concatenated in grid.x, so Q/KV prefill shares both
+ * the Q8_K quantization and a single launch without padding the smaller
+ * projection up to the larger one's row count.  Each workgroup still handles
+ * only one weight matrix: this preserves the standalone TILE8 block walk and
+ * its accumulation order while removing the second host launch. */
+__global__ static void rocm_matmul_q4_K_prefill_tile8_pair_kernel(
+        float *out0,
+        float *out1,
+        const char *w0,
+        const char *w1,
+        const cuda_block_q8_K *xq,
+        uint64_t row_bytes,
+        uint32_t xq_blocks,
+        uint32_t out0_dim,
+        uint32_t out1_dim,
+        uint32_t n_tok) {
+    __shared__ cuda_block_q8_K sxq[ROCM_Q4_PREFILL_TOKEN_TILE]
+                                         [ROCM_Q4_PREFILL_KBLOCK_TILE];
+
+    const uint32_t tid = threadIdx.x;
+    const uint32_t lane = tid & 7u;
+    const uint32_t row_lane = tid >> 3u;
+    const uint32_t out0_tiles = (out0_dim - 1u) / 32u + 1u;
+    const bool second = blockIdx.x >= out0_tiles;
+    const uint32_t row_tile = second ? blockIdx.x - out0_tiles : blockIdx.x;
+    const uint32_t row = row_tile * 32u + row_lane;
+    const uint32_t tok0 = blockIdx.y * ROCM_Q4_PREFILL_TOKEN_TILE;
+    const uint32_t out_dim = second ? out1_dim : out0_dim;
+    float *const out = second ? out1 : out0;
+    const char *const w_base = second ? w1 : w0;
+    const uint32_t nt = n_tok - tok0 < ROCM_Q4_PREFILL_TOKEN_TILE
+                      ? n_tok - tok0 : ROCM_Q4_PREFILL_TOKEN_TILE;
+    const bool row_valid = row < out_dim;
+    const cuda_block_q4_K *wr = row_valid
+        ? reinterpret_cast<const cuda_block_q4_K *>(
+              w_base + (uint64_t)row * row_bytes)
+        : NULL;
+    float acc[ROCM_Q4_PREFILL_TOKEN_TILE] = {
+        0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f,
+    };
+
+    for (uint32_t b0 = 0u; b0 < xq_blocks;
+         b0 += ROCM_Q4_PREFILL_KBLOCK_TILE) {
+        const uint32_t nb = xq_blocks - b0 < ROCM_Q4_PREFILL_KBLOCK_TILE
+                          ? xq_blocks - b0
+                          : ROCM_Q4_PREFILL_KBLOCK_TILE;
+        const uint32_t tile_words = nt * ROCM_Q4_PREFILL_KBLOCK_TILE *
+                                    ROCM_Q4_Q8K_WORDS;
+        uint32_t *const sxq_words = reinterpret_cast<uint32_t *>(sxq);
+        for (uint32_t i = tid; i < tile_words; i += blockDim.x) {
+            const uint32_t block_slot = i / ROCM_Q4_Q8K_WORDS;
+            const uint32_t word = i - block_slot * ROCM_Q4_Q8K_WORDS;
+            const uint32_t p = block_slot >> 3u;
+            const uint32_t bb = block_slot & 7u;
+            if (bb < nb) {
+                const uint64_t src_block =
+                    (uint64_t)(tok0 + p) * xq_blocks + b0 + bb;
+                const uint32_t *const src_words =
+                    reinterpret_cast<const uint32_t *>(xq + src_block);
+                sxq_words[i] = src_words[word];
+            }
+        }
+        __syncthreads();
+
+        if (row_valid && lane < nb) {
+            rocm_dot_q4_K_q8_K_block8_reuse_weights(
+                wr + b0 + lane,
+                sxq[0] + lane, sxq[1] + lane,
+                sxq[2] + lane, sxq[3] + lane,
+                sxq[4] + lane, sxq[5] + lane,
+                sxq[6] + lane, sxq[7] + lane,
+                nt, acc);
+        }
+        __syncthreads();
+    }
+
+    if (row_valid) {
+        #pragma unroll
+        for (uint32_t p = 0u; p < ROCM_Q4_PREFILL_TOKEN_TILE; p++) {
+            if (p < nt) {
+                const float v = quarter_warp_sum_f32(acc[p], lane);
+                if (lane == 0u) {
+                    out[(uint64_t)(tok0 + p) * out_dim + row] = v;
+                }
+            }
+        }
+    }
+}
+
+static int rocm_q4_K_dense_validate(
+        const ds4_gpu_tensor *out,
+        const void *model_map,
+        uint64_t model_size,
+        uint64_t weight_offset,
+        uint64_t in_dim,
+        uint64_t out_dim,
+        const ds4_gpu_tensor *x,
+        uint64_t n_tok,
+        uint64_t *blocks_out,
+        uint64_t *row_bytes_out,
+        uint64_t *weight_bytes_out) {
+    if (!out || !x || !model_map || !blocks_out || !row_bytes_out ||
+        !weight_bytes_out || in_dim == 0u || out_dim == 0u || n_tok == 0u ||
+        in_dim > UINT32_MAX || out_dim > UINT32_MAX || n_tok > UINT32_MAX ||
+        (in_dim % CUDA_QK_K) != 0u) {
+        return 0;
+    }
+
+    const uint64_t blocks = in_dim / CUDA_QK_K;
+    uint64_t row_bytes = 0;
+    uint64_t weight_bytes = 0;
+    if (blocks == 0u ||
+        !cuda_u64_mul_checked(blocks, sizeof(cuda_block_q4_K), &row_bytes) ||
+        !cuda_u64_mul_checked(out_dim, row_bytes, &weight_bytes) ||
+        !cuda_model_range_fits(model_size, weight_offset, weight_bytes) ||
+        !cuda_tensor_has_elems2(x, n_tok, in_dim, sizeof(float)) ||
+        !cuda_tensor_has_elems2(out, n_tok, out_dim, sizeof(float))) {
+        return 0;
+    }
+
+    *blocks_out = blocks;
+    *row_bytes_out = row_bytes;
+    *weight_bytes_out = weight_bytes;
+    return 1;
+}
+
+static cuda_block_q8_K *rocm_q4_K_prequant_alloc(
+        uint64_t n_tok,
+        uint64_t blocks,
+        const char *what) {
+    uint64_t bytes = 0;
+    if (!cuda_u64_mul3_checked(n_tok, blocks,
+                               sizeof(cuda_block_q8_K), &bytes)) {
+        return NULL;
+    }
+    return reinterpret_cast<cuda_block_q8_K *>(cuda_tmp_alloc(bytes, what));
+}
+
+static int rocm_q4_K_byte_ranges_overlap(
+        const void *ptr0, uint64_t bytes0,
+        const void *ptr1, uint64_t bytes1) {
+    const uintptr_t p0 = reinterpret_cast<uintptr_t>(ptr0);
+    const uintptr_t p1 = reinterpret_cast<uintptr_t>(ptr1);
+    return p0 <= p1 ? (uint64_t)(p1 - p0) < bytes0
+                    : (uint64_t)(p0 - p1) < bytes1;
+}
+
+static int rocm_q4_K_dense_pair_requested(void) {
+    return getenv("DS4_ROCM_ENABLE_Q4_DENSE_PAIR") != NULL &&
+           getenv("DS4_ROCM_DISABLE_Q4_DENSE_PAIR") == NULL;
+}
+
+enum {
+    ROCM_Q4_GROUPED_ATTN_A_DEFAULT_K = 4096u,
+    ROCM_Q4_GROUPED_ATTN_A_DEFAULT_M = 1024u,
+    ROCM_Q4_GROUPED_ATTN_A_DEFAULT_GROUPS = 8u,
+};
+
+static int rocm_q4_K_grouped_attn_a_resident_default_scope(
+        uint64_t group_dim,
+        uint64_t rank,
+        uint32_t group0,
+        uint32_t group_cnt,
+        int resident_decode) {
+    /* A batch fallback can pass one row at a time through this same API, so
+     * the caller explicitly identifies true decode.  Default only the
+     * production decode shape after the complete model has been made
+     * resident; explicit ENABLE keeps the existing experimental surface. */
+    return resident_decode &&
+           !g_ssd_streaming_mode &&
+           group_dim == ROCM_Q4_GROUPED_ATTN_A_DEFAULT_K &&
+           rank == ROCM_Q4_GROUPED_ATTN_A_DEFAULT_M &&
+           group0 == 0u &&
+           group_cnt == ROCM_Q4_GROUPED_ATTN_A_DEFAULT_GROUPS;
+}
+
+static int rocm_q4_K_prefill_tile8_scope(uint64_t n_tok) {
+    /* Keep decode/speculative micro-batches on the latency-oriented legacy
+     * kernel.  4096 is DS4's largest supported prefill chunk and bounds the
+     * validated tiled-prefill surface. */
+    return n_tok > 8u && n_tok <= 4096u;
+}
+
+static int rocm_q4_K_prefill_tile8_requested(void) {
+    /* TILE8 is the ROCm Q4 prefill default.  Keep the old ENABLE variable
+     * harmlessly compatible and retain one authoritative rollback switch. */
+    return getenv("DS4_ROCM_DISABLE_Q4_PREFILL_TILE8") == NULL;
+}
+
+static int rocm_q4_K_prefill_tile8_required(void) {
+    return getenv("DS4_ROCM_REQUIRE_Q4_PREFILL_TILE8") != NULL;
+}
+
+enum {
+    ROCM_Q4_PREFILL_Q8_WAVE32_REQUIRED_FAILURE = -1,
+    ROCM_Q4_PREFILL_Q8_WAVE32_FALLBACK = 0,
+    ROCM_Q4_PREFILL_Q8_WAVE32_USE = 1,
+};
+
+static uint64_t g_rocm_q4_prefill_q8_wave32_launches;
+
+extern "C" void ds4_rocm_test_q4_prefill_q8_wave32_reset(void) {
+    __atomic_store_n(&g_rocm_q4_prefill_q8_wave32_launches, 0u,
+                     __ATOMIC_RELAXED);
+}
+
+extern "C" uint64_t ds4_rocm_test_q4_prefill_q8_wave32_get_calls(void) {
+    return __atomic_load_n(&g_rocm_q4_prefill_q8_wave32_launches,
+                           __ATOMIC_RELAXED);
+}
+
+/* Pure policy kept separate from device discovery so the precedence and
+ * fail-closed contract remain testable on hosts without a visible GPU.
+ * REQUIRE is also an opt-in; DISABLE is authoritative. */
+static int rocm_q4_K_prefill_q8_wave32_policy(
+        int prefill_scope,
+        int runtime_compatible,
+        int enabled,
+        int disabled,
+        int required) {
+    if (!enabled && !required) {
+        return ROCM_Q4_PREFILL_Q8_WAVE32_FALLBACK;
+    }
+    if (disabled || !prefill_scope || !runtime_compatible) {
+        return required ? ROCM_Q4_PREFILL_Q8_WAVE32_REQUIRED_FAILURE
+                        : ROCM_Q4_PREFILL_Q8_WAVE32_FALLBACK;
+    }
+    return ROCM_Q4_PREFILL_Q8_WAVE32_USE;
+}
+
+extern "C" int ds4_rocm_test_q4_prefill_q8_wave32_policy(
+        int prefill_scope,
+        int runtime_compatible,
+        int enabled,
+        int disabled,
+        int required) {
+    return rocm_q4_K_prefill_q8_wave32_policy(
+        prefill_scope != 0, runtime_compatible != 0, enabled != 0,
+        disabled != 0, required != 0);
+}
+
+static int rocm_q4_K_prefill_q8_wave32_required(void) {
+    return rocm_q4_attn_q_b_env_bool(
+        "DS4_ROCM_REQUIRE_Q4_PREFILL_Q8_K_WAVE32") == 1;
+}
+
+static int rocm_q4_K_prefill_q8_wave32_select(uint64_t n_tok) {
+    const int enabled = rocm_q4_attn_q_b_env_bool(
+        "DS4_ROCM_ENABLE_Q4_PREFILL_Q8_K_WAVE32") == 1;
+    const int disabled = rocm_q4_attn_q_b_env_bool(
+        "DS4_ROCM_DISABLE_Q4_PREFILL_Q8_K_WAVE32") == 1;
+    const int required = rocm_q4_K_prefill_q8_wave32_required();
+    const int prefill_scope = rocm_q4_K_prefill_tile8_scope(n_tok);
+    const int requested = enabled || required;
+    const int runtime_compatible = requested && !disabled && prefill_scope
+        ? rocm_attention_runtime_is_gfx1151_wave32(0)
+        : 0;
+    const int decision = rocm_q4_K_prefill_q8_wave32_policy(
+        prefill_scope, runtime_compatible, enabled, disabled, required);
+    if (decision == ROCM_Q4_PREFILL_Q8_WAVE32_REQUIRED_FAILURE) {
+        fprintf(stderr,
+                DS4_GPU_LOG_PREFIX
+                "required Q4 prefill Q8_K wave32 quantizer is unavailable "
+                "(N=%llu scope=%d compatible=%d disabled=%d)\n",
+                (unsigned long long)n_tok, prefill_scope,
+                runtime_compatible, disabled);
+    }
+    return decision;
+}
+
+static int rocm_q4_K_q8_quantize_launch(
+        cuda_block_q8_K *out,
+        const float *x,
+        uint32_t in_dim,
+        uint32_t n_rows,
+        int q8_wave32,
+        int q8_wave32_required,
+        const char *label) {
+    const uint32_t blocks = in_dim / CUDA_QK_K;
+    if (q8_wave32 == ROCM_Q4_PREFILL_Q8_WAVE32_USE) {
+        /* Re-read the active device immediately before enqueue.  Selection
+         * may have happened before range resolution/allocation, and HIP's
+         * active device is thread-local.  Optional use falls back safely;
+         * REQUIRE cannot silently enqueue either the wrong kernel or the
+         * canonical one. */
+        if (!rocm_attention_runtime_is_gfx1151_wave32(0)) {
+            if (q8_wave32_required) {
+                fprintf(stderr,
+                        DS4_GPU_LOG_PREFIX
+                        "required Q8_K wave32 quantizer lost its gfx1151 "
+                        "wave32 device before enqueue\n");
+                return 0;
+            }
+            q8_wave32 = ROCM_Q4_PREFILL_Q8_WAVE32_FALLBACK;
+        }
+    }
+    if (q8_wave32 == ROCM_Q4_PREFILL_Q8_WAVE32_USE) {
+        const dim3 grid(
+            (unsigned)((blocks + ROCM_Q8_K_WAVE32_WAVES_PER_BLOCK - 1u) /
+                       ROCM_Q8_K_WAVE32_WAVES_PER_BLOCK),
+            n_rows, 1u);
+        q8_K_quantize_wave32_kernel<<<
+            grid, ROCM_Q8_K_WAVE32_BLOCK_THREADS>>>(
+                out, x, in_dim, n_rows);
+        const int ok = cuda_ok(cudaGetLastError(), label);
+        if (ok) {
+            __atomic_fetch_add(&g_rocm_q4_prefill_q8_wave32_launches, 1u,
+                               __ATOMIC_RELAXED);
+        }
+        return ok;
+    }
+
+    const dim3 grid((unsigned)blocks, n_rows, 1u);
+    q8_K_quantize_kernel<<<grid, 256u>>>(out, x, in_dim, n_rows);
+    return cuda_ok(cudaGetLastError(), label);
+}
+
+/* Raw-layout oracle used only by the ROCm Q4 parity test.  It lets the test
+ * compare every Q8_K byte instead of relying solely on downstream dot
+ * products to expose a quantizer mismatch. */
+extern "C" int ds4_rocm_test_q8_K_quantize_tensor(
+        ds4_gpu_tensor *out,
+        const ds4_gpu_tensor *x,
+        uint32_t in_dim,
+        uint32_t n_rows,
+        int use_wave32) {
+    if (!out || !x || !out->ptr || !x->ptr || in_dim == 0u || n_rows == 0u ||
+        (in_dim % CUDA_QK_K) != 0u) {
+        return 0;
+    }
+    uint64_t x_bytes = 0;
+    uint64_t out_bytes = 0;
+    if (!cuda_u64_mul3_checked(n_rows, in_dim, sizeof(float), &x_bytes) ||
+        !cuda_u64_mul3_checked(n_rows, in_dim / CUDA_QK_K,
+                               sizeof(cuda_block_q8_K), &out_bytes) ||
+        x->bytes < x_bytes || out->bytes < out_bytes) {
+        return 0;
+    }
+    const int decision = use_wave32
+        ? (rocm_attention_runtime_is_gfx1151_wave32(0)
+               ? ROCM_Q4_PREFILL_Q8_WAVE32_USE
+               : ROCM_Q4_PREFILL_Q8_WAVE32_REQUIRED_FAILURE)
+        : ROCM_Q4_PREFILL_Q8_WAVE32_FALLBACK;
+    if (decision == ROCM_Q4_PREFILL_Q8_WAVE32_REQUIRED_FAILURE) return 0;
+    return rocm_q4_K_q8_quantize_launch(
+        reinterpret_cast<cuda_block_q8_K *>(out->ptr),
+        reinterpret_cast<const float *>(x->ptr), in_dim, n_rows, decision,
+        use_wave32 != 0,
+        use_wave32 ? "Q8_K wave32 raw oracle launch"
+                   : "Q8_K canonical raw oracle launch");
+}
+
+/* Benchmark-only enqueue hook.  The resident ROCm harness validates the
+ * active gfx1151/wave32 device, dimensions, allocations and output guards
+ * before entering its HIP-event interval.  Keep this entry deliberately free
+ * of device queries, allocation, environment parsing and cudaGetLastError so
+ * the measured interval contains only the selected quantizer dispatch.  The
+ * following end event/synchronization reports any asynchronous failure. */
+extern "C" void ds4_rocm_bench_q8_K_quantize_enqueue(
+        void *out,
+        const void *x,
+        uint32_t in_dim,
+        uint32_t n_rows,
+        int use_wave32) {
+    const uint32_t blocks = in_dim / CUDA_QK_K;
+    if (use_wave32) {
+        const dim3 grid(
+            (unsigned)((blocks + ROCM_Q8_K_WAVE32_WAVES_PER_BLOCK - 1u) /
+                       ROCM_Q8_K_WAVE32_WAVES_PER_BLOCK),
+            n_rows, 1u);
+        q8_K_quantize_wave32_kernel<<<
+            grid, ROCM_Q8_K_WAVE32_BLOCK_THREADS>>>(
+                reinterpret_cast<cuda_block_q8_K *>(out),
+                reinterpret_cast<const float *>(x), in_dim, n_rows);
+        return;
+    }
+
+    const dim3 grid((unsigned)blocks, n_rows, 1u);
+    q8_K_quantize_kernel<<<grid, 256u>>>(
+        reinterpret_cast<cuda_block_q8_K *>(out),
+        reinterpret_cast<const float *>(x), in_dim, n_rows);
+}
+
+enum {
+    ROCM_Q4_PREFILL_WMMA_REQUIRED_FAILURE = -1,
+    ROCM_Q4_PREFILL_WMMA_FALLBACK = 0,
+    ROCM_Q4_PREFILL_WMMA_USE = 1,
+};
+
+/* Test oracle for strict dispatch: REQUIRE must attest this launch wrapper,
+ * not merely produce output through a canonical fallback. */
+static uint64_t g_rocm_q4_prefill_wmma_launches;
+static uint64_t g_rocm_q4_prefill_wmma_k64_launches;
+static uint64_t g_rocm_q4_prefill_wmma_k128_launches;
+static int g_rocm_q4_prefill_tile8_report_registered;
+
+static void rocm_q4_K_prefill_tile8_report(void);
+
+static void rocm_q4_K_prefill_stats_register(void) {
+    if (getenv("DS4_ROCM_Q4_PREFILL_TILE8_STATS") == NULL) return;
+    if (__atomic_exchange_n(&g_rocm_q4_prefill_tile8_report_registered, 1,
+                            __ATOMIC_ACQ_REL) == 0) {
+        (void)atexit(rocm_q4_K_prefill_tile8_report);
+    }
+}
+
+extern "C" void ds4_rocm_test_q4_prefill_wmma_reset(void) {
+    __atomic_store_n(&g_rocm_q4_prefill_wmma_launches, 0u,
+                     __ATOMIC_RELAXED);
+    __atomic_store_n(&g_rocm_q4_prefill_wmma_k64_launches, 0u,
+                     __ATOMIC_RELAXED);
+    __atomic_store_n(&g_rocm_q4_prefill_wmma_k128_launches, 0u,
+                     __ATOMIC_RELAXED);
+}
+
+extern "C" uint64_t ds4_rocm_test_q4_prefill_wmma_get_calls(void) {
+    return __atomic_load_n(&g_rocm_q4_prefill_wmma_launches,
+                           __ATOMIC_RELAXED);
+}
+
+extern "C" uint64_t ds4_rocm_test_q4_prefill_wmma_k64_get_calls(void) {
+    return __atomic_load_n(&g_rocm_q4_prefill_wmma_k64_launches,
+                           __ATOMIC_RELAXED);
+}
+
+extern "C" uint64_t ds4_rocm_test_q4_prefill_wmma_k128_get_calls(void) {
+    return __atomic_load_n(&g_rocm_q4_prefill_wmma_k128_launches,
+                           __ATOMIC_RELAXED);
+}
+
+static int rocm_q4_K_prefill_wmma_k64_control_policy(int control) {
+    return control != 0;
+}
+
+extern "C" int ds4_rocm_test_q4_prefill_wmma_k64_control_policy(
+        int control) {
+    return rocm_q4_K_prefill_wmma_k64_control_policy(control);
+}
+
+/* K128 is the production default for its aligned q_b-style 256-row scope.
+ * Keep a value-aware opt-out for tester rollback, layer it on top of the K64
+ * control so K64=0 remains a reliable K32 rollback, and retain K64 for every
+ * incompatible launch. */
+static int rocm_q4_K_prefill_wmma_k128_policy(
+        int disabled,
+        int k64_enabled,
+        uint32_t row_tile,
+        int load4_compatible) {
+    return disabled != 1 && k64_enabled && row_tile == 256u &&
+           load4_compatible;
+}
+
+extern "C" int ds4_rocm_test_q4_prefill_wmma_k128_policy(
+        int disabled,
+        int k64_enabled,
+        uint32_t row_tile,
+        int load4_compatible) {
+    return rocm_q4_K_prefill_wmma_k128_policy(
+        disabled, k64_enabled, row_tile, load4_compatible);
+}
+
+/* Resident execution is automatic after validation.  Preserve an explicit
+ * ENABLE=0 as a compatibility opt-out, while REQUIRE remains a strict
+ * assertion rather than the switch that turns the candidate on.  SSD
+ * streaming deliberately keeps its separate opt-in and residency contract. */
+static int rocm_q4_K_prefill_wmma_requested_policy(
+        int ssd_streaming,
+        int enabled,
+        int ssd_enabled,
+        int disabled,
+        int required) {
+    if (required) return 1;
+    if (disabled) return 0;
+    return ssd_streaming ? ssd_enabled == 1 : enabled != 0;
+}
+
+/* Attention output is a two-stage A -> B projection.  A retains the validated
+ * standalone policy: resident execution is automatic, while SSD keeps its
+ * explicit gate and physical-device-residency contract. */
+static int rocm_q4_K_prefill_wmma_attention_a_requested_policy(
+        int ssd_streaming,
+        int enabled,
+        int ssd_enabled,
+        int disabled,
+        int required) {
+    return rocm_q4_K_prefill_wmma_requested_policy(
+        ssd_streaming, enabled, ssd_enabled, disabled, required);
+}
+
+/* Applying direct WMMA to both stages compounds the two F16 approximations.
+ * Keep attention-output B behind REQUIRE so ordinary ENABLE A/B measurements
+ * retain the exact Q8_K+TILE8 second stage.  Test REQUIRE before DISABLE so
+ * DISABLE+REQUIRE reaches the normal selector and fails closed. */
+static int rocm_q4_K_prefill_wmma_attention_b_requested_policy(
+        int ssd_streaming,
+        int enabled,
+        int ssd_enabled,
+        int disabled,
+        int required) {
+    (void)ssd_streaming;
+    (void)enabled;
+    (void)ssd_enabled;
+    (void)disabled;
+    return required != 0;
+}
+
+/* An explicitly selected exact Q8_K quantizer owns an optional WMMA default.
+ * REQUIRE_WMMA is the only control that may override an optional Q8 request;
+ * dual REQUIRE is rejected by each public dispatch before enqueue. */
+static int rocm_q4_K_prefill_wmma_yields_to_q8_wave32(
+        int q8_wave32,
+        int wmma_required) {
+    return q8_wave32 == ROCM_Q4_PREFILL_Q8_WAVE32_USE && !wmma_required;
+}
+
+extern "C" int ds4_rocm_test_q4_prefill_wmma_yields_to_q8_wave32(
+        int q8_selected,
+        int wmma_required) {
+    const int q8_wave32 = q8_selected
+        ? ROCM_Q4_PREFILL_Q8_WAVE32_USE
+        : ROCM_Q4_PREFILL_Q8_WAVE32_FALLBACK;
+    return rocm_q4_K_prefill_wmma_yields_to_q8_wave32(
+        q8_wave32, wmma_required != 0);
+}
+
+extern "C" int ds4_rocm_test_q4_prefill_wmma_requested_policy(
+        int ssd_streaming,
+        int enabled,
+        int ssd_enabled,
+        int disabled,
+        int required) {
+    return rocm_q4_K_prefill_wmma_requested_policy(
+        ssd_streaming != 0, enabled, ssd_enabled, disabled != 0,
+        required != 0);
+}
+
+extern "C" int
+ds4_rocm_test_q4_prefill_wmma_attention_a_requested_policy(
+        int ssd_streaming,
+        int enabled,
+        int ssd_enabled,
+        int disabled,
+        int required) {
+    return rocm_q4_K_prefill_wmma_attention_a_requested_policy(
+        ssd_streaming != 0, enabled, ssd_enabled, disabled != 0,
+        required != 0);
+}
+
+extern "C" int
+ds4_rocm_test_q4_prefill_wmma_attention_b_requested_policy(
+        int ssd_streaming,
+        int enabled,
+        int ssd_enabled,
+        int disabled,
+        int required) {
+    return rocm_q4_K_prefill_wmma_attention_b_requested_policy(
+        ssd_streaming != 0, enabled, ssd_enabled, disabled != 0,
+        required != 0);
+}
+
+static int rocm_q4_K_prefill_wmma_select(
+        uint64_t n_tok,
+        uint64_t in_dim,
+        uint64_t out_dim,
+        int weight_device_resident) {
+    const int enabled = rocm_q4_attn_q_b_env_bool(
+        "DS4_ROCM_ENABLE_Q4_PREFILL_WMMA");
+    const int ssd_enabled = rocm_q4_attn_q_b_env_bool(
+        "DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_SSD") == 1;
+    const int disabled = rocm_q4_attn_q_b_env_bool(
+        "DS4_ROCM_DISABLE_Q4_PREFILL_WMMA") == 1;
+    const int required = rocm_q4_attn_q_b_env_bool(
+        "DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA") == 1;
+    const int requested = rocm_q4_K_prefill_wmma_requested_policy(
+        g_ssd_streaming_mode, enabled, ssd_enabled, disabled, required);
+    if (!requested) return ROCM_Q4_PREFILL_WMMA_FALLBACK;
+
+    const int shape_ok =
+        n_tok >= 256u && n_tok <= 4096u &&
+        in_dim != 0u && (in_dim % CUDA_QK_K) == 0u &&
+        out_dim != 0u && in_dim <= UINT32_MAX &&
+        out_dim <= UINT32_MAX && n_tok <= UINT32_MAX;
+    const int storage_ok = !g_ssd_streaming_mode ||
+                           ((ssd_enabled || required) &&
+                            weight_device_resident);
+    const int explicit_request = enabled == 1 || ssd_enabled || required;
+    const int eligible = !disabled && shape_ok && storage_ok &&
+        !g_quality_mode &&
+        rocm_attention_runtime_is_gfx1151_wave32(explicit_request);
+    if (eligible) return ROCM_Q4_PREFILL_WMMA_USE;
+    if (!required) return ROCM_Q4_PREFILL_WMMA_FALLBACK;
+
+    fprintf(stderr,
+            DS4_GPU_LOG_PREFIX
+            "required Q4_K prefill WMMA is unavailable "
+            "(N=%llu K=%llu M=%llu disabled=%d ssd=%d ssd_opt=%d "
+            "resident=%d quality=%d)\n",
+            (unsigned long long)n_tok,
+            (unsigned long long)in_dim,
+            (unsigned long long)out_dim,
+            disabled,
+            g_ssd_streaming_mode ? 1 : 0,
+            ssd_enabled,
+            weight_device_resident,
+            g_quality_mode ? 1 : 0);
+    return ROCM_Q4_PREFILL_WMMA_REQUIRED_FAILURE;
+}
+
+static int rocm_q4_K_prefill_wmma_load2_compatible(
+        const float *x,
+        uint64_t x_token_stride,
+        uint64_t x_group_stride) {
+    return x && (((uintptr_t)x & 7u) == 0u) &&
+           ((x_token_stride & 1u) == 0u) &&
+           ((x_group_stride & 1u) == 0u);
+}
+
+static int rocm_q4_K_prefill_wmma_load4_compatible(
+        const float *x,
+        uint64_t x_token_stride,
+        uint64_t x_group_stride) {
+    return x && (((uintptr_t)x & 15u) == 0u) &&
+           ((x_token_stride & 3u) == 0u) &&
+           ((x_group_stride & 3u) == 0u);
+}
+
+static void rocm_q4_K_prefill_wmma_enqueue(
+        float *out,
+        const char *w,
+        const float *x,
+        uint32_t n_tok,
+        uint32_t n_groups,
+        uint32_t in_dim,
+        uint32_t out_dim,
+        uint64_t row_bytes,
+        uint64_t x_token_stride,
+        uint64_t x_group_stride,
+        uint64_t out_token_stride,
+        uint32_t row_tile,
+        int load2) {
+    const dim3 grid(
+        (unsigned)(((uint64_t)out_dim + row_tile - 1u) / row_tile),
+        (unsigned)(((uint64_t)n_tok + ROCM_Q4_WMMA_TOKEN_TILE - 1u) /
+                   ROCM_Q4_WMMA_TOKEN_TILE),
+        n_groups);
+    if (row_tile == 256u) {
+        if (load2) {
+            rocm_matmul_q4_K_prefill_wmma_rowtile_strided_kernel<
+                256u, 16u, 1u, true><<<grid, 512u>>>(
+                    out, w, x, n_tok, n_groups, in_dim, out_dim, row_bytes,
+                    x_token_stride, x_group_stride, out_token_stride);
+        } else {
+            rocm_matmul_q4_K_prefill_wmma_rowtile_strided_kernel<
+                256u, 16u, 1u, false><<<grid, 512u>>>(
+                    out, w, x, n_tok, n_groups, in_dim, out_dim, row_bytes,
+                    x_token_stride, x_group_stride, out_token_stride);
+        }
+    } else if (row_tile == 128u) {
+        if (load2) {
+            rocm_matmul_q4_K_prefill_wmma_rowtile_strided_kernel<
+                128u, 8u, 1u, true><<<grid, 256u>>>(
+                    out, w, x, n_tok, n_groups, in_dim, out_dim, row_bytes,
+                    x_token_stride, x_group_stride, out_token_stride);
+        } else {
+            rocm_matmul_q4_K_prefill_wmma_rowtile_strided_kernel<
+                128u, 8u, 1u, false><<<grid, 256u>>>(
+                    out, w, x, n_tok, n_groups, in_dim, out_dim, row_bytes,
+                    x_token_stride, x_group_stride, out_token_stride);
+        }
+    } else {
+        /* Preserve the original kernel's occupancy contract so the retained
+         * 64-row benchmark arm differs only in row geometry. */
+        rocm_matmul_q4_K_prefill_wmma_rowtile_strided_kernel<
+            64u, 4u, 2u, false>
+            <<<grid, 128u>>>(
+                out, w, x, n_tok, n_groups, in_dim, out_dim, row_bytes,
+                x_token_stride, x_group_stride, out_token_stride);
+    }
+}
+
+/* Keep the K64 launch family separate from the established K32 entry points.
+ * Besides making rollback a single environment change, this prevents compiler
+ * changes caused by a template K_TILE branch from moving the A/B baseline. */
+static void rocm_q4_K_prefill_wmma_k64_enqueue(
+        float *out,
+        const char *w,
+        const float *x,
+        uint32_t n_tok,
+        uint32_t n_groups,
+        uint32_t in_dim,
+        uint32_t out_dim,
+        uint64_t row_bytes,
+        uint64_t x_token_stride,
+        uint64_t x_group_stride,
+        uint64_t out_token_stride,
+        uint32_t row_tile,
+        int load2) {
+    const dim3 grid(
+        (unsigned)(((uint64_t)out_dim + row_tile - 1u) / row_tile),
+        (unsigned)(((uint64_t)n_tok + ROCM_Q4_WMMA_TOKEN_TILE - 1u) /
+                   ROCM_Q4_WMMA_TOKEN_TILE),
+        n_groups);
+    if (row_tile == 256u) {
+        if (load2) {
+            rocm_matmul_q4_K_prefill_wmma_k64_p80_rowtile_strided_kernel<
+                256u, 16u, 1u, true><<<grid, 512u>>>(
+                    out, w, x, n_tok, n_groups, in_dim, out_dim, row_bytes,
+                    x_token_stride, x_group_stride, out_token_stride);
+        } else {
+            rocm_matmul_q4_K_prefill_wmma_k64_p80_rowtile_strided_kernel<
+                256u, 16u, 1u, false><<<grid, 512u>>>(
+                    out, w, x, n_tok, n_groups, in_dim, out_dim, row_bytes,
+                    x_token_stride, x_group_stride, out_token_stride);
+        }
+    } else if (row_tile == 128u) {
+        if (load2) {
+            rocm_matmul_q4_K_prefill_wmma_k64_p80_rowtile_strided_kernel<
+                128u, 8u, 1u, true><<<grid, 256u>>>(
+                    out, w, x, n_tok, n_groups, in_dim, out_dim, row_bytes,
+                    x_token_stride, x_group_stride, out_token_stride);
+        } else {
+            rocm_matmul_q4_K_prefill_wmma_k64_p80_rowtile_strided_kernel<
+                128u, 8u, 1u, false><<<grid, 256u>>>(
+                    out, w, x, n_tok, n_groups, in_dim, out_dim, row_bytes,
+                    x_token_stride, x_group_stride, out_token_stride);
+        }
+    } else {
+        rocm_matmul_q4_K_prefill_wmma_k64_p80_rowtile_strided_kernel<
+            64u, 4u, 2u, false><<<grid, 128u>>>(
+                out, w, x, n_tok, n_groups, in_dim, out_dim, row_bytes,
+                x_token_stride, x_group_stride, out_token_stride);
+    }
+}
+
+/* The K128 path is deliberately a single q_b-oriented instantiation.
+ * Keeping it out of the generic K32/K64 template family preserves the codegen
+ * of both established benchmark arms. */
+static void rocm_q4_K_prefill_wmma_k128_enqueue(
+        float *out,
+        const char *w,
+        const float *x,
+        uint32_t n_tok,
+        uint32_t n_groups,
+        uint32_t in_dim,
+        uint32_t out_dim,
+        uint64_t row_bytes,
+        uint64_t x_token_stride,
+        uint64_t x_group_stride,
+        uint64_t out_token_stride) {
+    const dim3 grid(
+        (unsigned)(((uint64_t)out_dim + 255u) / 256u),
+        (unsigned)(((uint64_t)n_tok + ROCM_Q4_WMMA_TOKEN_TILE - 1u) /
+                   ROCM_Q4_WMMA_TOKEN_TILE),
+        n_groups);
+    rocm_matmul_q4_K_prefill_wmma_k128_p144_rowtile_strided_kernel<
+        256u, 16u, 1u><<<grid, 512u>>>(
+            out, w, x, n_tok, n_groups, in_dim, out_dim, row_bytes,
+            x_token_stride, x_group_stride, out_token_stride);
+}
+
+static uint32_t rocm_q4_K_prefill_wmma_row_tile(uint32_t out_dim) {
+    const uint32_t shape_tile = out_dim >= 8192u
+        ? 256u
+        : (out_dim >= 1024u ? 128u : 64u);
+    const char *raw = getenv("DS4_ROCM_Q4_PREFILL_WMMA_ROW_TILE");
+    if (raw) {
+        while (isspace((unsigned char)*raw)) raw++;
+        /* strtoull accepts a leading minus and wraps some values into the
+         * whitelist.  Treat every negative spelling as an invalid unsigned
+         * override so the benchmark cannot silently select the wrong arm. */
+        if (*raw == '-') return shape_tile;
+    }
+    const uint64_t requested = rocm_q4_attn_q_b_env_u64(
+        "DS4_ROCM_Q4_PREFILL_WMMA_ROW_TILE", 0u, 0u, UINT64_MAX);
+    if (requested == 64u || requested == 128u || requested == 256u) {
+        return (uint32_t)requested;
+    }
+    return shape_tile;
+}
+
+extern "C" uint32_t ds4_rocm_test_q4_prefill_wmma_row_tile(
+        uint32_t out_dim) {
+    return rocm_q4_K_prefill_wmma_row_tile(out_dim);
+}
+
+static int rocm_q4_K_prefill_wmma_launch(
+        float *out,
+        const char *w,
+        const float *x,
+        uint32_t n_tok,
+        uint32_t n_groups,
+        uint32_t in_dim,
+        uint32_t out_dim,
+        uint64_t row_bytes,
+        uint64_t x_token_stride,
+        uint64_t x_group_stride,
+        uint64_t out_token_stride,
+        const char *label) {
+    if (!out || !w || !x || n_groups == 0u) return 0;
+
+    /* Match the proven Q8 row geometry so a 64x32 activation tile is loaded
+     * once for 128 rows, or once for 256 rows on the large q_b/output shapes.
+     * Small projections retain 64 rows instead of launching mostly idle waves;
+     * that instantiation is also the deterministic previous-kernel A/B control.
+     * Invalid overrides deliberately fall back to shape selection. */
+    const uint32_t row_tile = rocm_q4_K_prefill_wmma_row_tile(out_dim);
+    const int load2 = row_tile != 64u &&
+        rocm_q4_K_prefill_wmma_load2_compatible(
+            x, x_token_stride, x_group_stride);
+    /* K64 is the base staging mode once the normal direct-Q4 WMMA selector
+     * has accepted this launch; eligible 256-row launches layer K128 on it.
+     * Preserve a value-aware K32 rollback: unset/true permits the wider
+     * stages and 0/false/no/off selects K32. */
+    const int k64_control = rocm_q4_attn_q_b_env_bool(
+        "DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_K64");
+    const int use_k64 =
+        rocm_q4_K_prefill_wmma_k64_control_policy(k64_control);
+    const int k128_disabled = rocm_q4_attn_q_b_env_bool(
+        "DS4_ROCM_DISABLE_Q4_PREFILL_WMMA_K128");
+    const int load4 = rocm_q4_K_prefill_wmma_load4_compatible(
+        x, x_token_stride, x_group_stride);
+    const int use_k128 = rocm_q4_K_prefill_wmma_k128_policy(
+        k128_disabled, use_k64, row_tile, load4);
+    if (use_k128) {
+        rocm_q4_K_prefill_wmma_k128_enqueue(
+            out, w, x, n_tok, n_groups, in_dim, out_dim, row_bytes,
+            x_token_stride, x_group_stride, out_token_stride);
+    } else if (use_k64) {
+        rocm_q4_K_prefill_wmma_k64_enqueue(
+            out, w, x, n_tok, n_groups, in_dim, out_dim, row_bytes,
+            x_token_stride, x_group_stride, out_token_stride, row_tile, load2);
+    } else {
+        rocm_q4_K_prefill_wmma_enqueue(
+            out, w, x, n_tok, n_groups, in_dim, out_dim, row_bytes,
+            x_token_stride, x_group_stride, out_token_stride, row_tile, load2);
+    }
+    const char *launch_label = use_k128
+        ? "q4_K prefill WMMA K128/P144 rowtile launch"
+        : (use_k64
+               ? "q4_K prefill WMMA K64/P80 rowtile launch"
+               : (label ? label : "q4_K prefill WMMA rowtile launch"));
+    const int ok = cuda_ok(cudaGetLastError(), launch_label);
+    if (ok) {
+        rocm_q4_K_prefill_stats_register();
+        __atomic_fetch_add(&g_rocm_q4_prefill_wmma_launches, 1u,
+                           __ATOMIC_RELAXED);
+        if (use_k128) {
+            __atomic_fetch_add(&g_rocm_q4_prefill_wmma_k128_launches, 1u,
+                               __ATOMIC_RELAXED);
+        } else if (use_k64) {
+            __atomic_fetch_add(&g_rocm_q4_prefill_wmma_k64_launches, 1u,
+                               __ATOMIC_RELAXED);
+        }
+    }
+    return ok;
+}
+
+/* Resident, enqueue-only hooks for the ROCm microbenchmark.  Pointer lookup,
+ * policy, environment parsing, and launch-error reporting stay outside HIP
+ * event intervals; the explicit tile and loader arguments make each arm
+ * self-attesting instead of relying on a mutable process environment. */
+extern "C" const void *ds4_rocm_bench_q4_K_resident_weight_ptr(
+        const void *model_map,
+        uint64_t model_size,
+        uint64_t weight_offset,
+        uint64_t weight_bytes) {
+    if (!cuda_model_range_fits(model_size, weight_offset, weight_bytes)) {
+        return NULL;
+    }
+    return rocm_q4_attn_q_b_device_resident_source(
+        model_map, weight_offset, weight_bytes);
+}
+
+extern "C" int ds4_rocm_bench_q4_K_wmma_enqueue(
+        void *out,
+        const void *w,
+        const void *x,
+        uint32_t n_tok,
+        uint32_t n_groups,
+        uint32_t in_dim,
+        uint32_t out_dim,
+        uint64_t row_bytes,
+        uint64_t x_token_stride,
+        uint64_t x_group_stride,
+        uint64_t out_token_stride,
+        uint32_t row_tile,
+        int load2) {
+    if (!out || !w || !x || n_tok == 0u || n_groups == 0u ||
+        in_dim == 0u || out_dim == 0u ||
+        (in_dim % CUDA_QK_K) != 0u ||
+        (load2 != 0 && load2 != 1) ||
+        (load2 && row_tile == 64u) ||
+        (row_tile != 64u && row_tile != 128u && row_tile != 256u)) {
+        return 0;
+    }
+    const uint64_t minimum_row_bytes =
+        ((uint64_t)in_dim / CUDA_QK_K) * sizeof(cuda_block_q4_K);
+    if (row_bytes < minimum_row_bytes ||
+        (load2 && !rocm_q4_K_prefill_wmma_load2_compatible(
+            reinterpret_cast<const float *>(x),
+            x_token_stride, x_group_stride))) {
+        return 0;
+    }
+    rocm_q4_K_prefill_wmma_enqueue(
+        reinterpret_cast<float *>(out),
+        reinterpret_cast<const char *>(w),
+        reinterpret_cast<const float *>(x),
+        n_tok, n_groups, in_dim, out_dim, row_bytes,
+        x_token_stride, x_group_stride, out_token_stride, row_tile, load2);
+    return 1;
+}
+
+/* Variant hook for a resident, same-process K32/K64 comparison.  The legacy
+ * benchmark hook above remains a strict K32 wrapper so existing binaries and
+ * source call sites retain their meaning. */
+extern "C" int ds4_rocm_bench_q4_K_wmma_variant_enqueue(
+        void *out,
+        const void *w,
+        const void *x,
+        uint32_t n_tok,
+        uint32_t n_groups,
+        uint32_t in_dim,
+        uint32_t out_dim,
+        uint64_t row_bytes,
+        uint64_t x_token_stride,
+        uint64_t x_group_stride,
+        uint64_t out_token_stride,
+        uint32_t row_tile,
+        uint32_t k_tile,
+        int load2) {
+    if (k_tile != ROCM_Q4_WMMA_K_TILE &&
+        k_tile != ROCM_Q4_WMMA_K64_TILE) {
+        return 0;
+    }
+    if (!out || !w || !x || n_tok == 0u || n_groups == 0u ||
+        in_dim == 0u || out_dim == 0u ||
+        (in_dim % CUDA_QK_K) != 0u ||
+        (load2 != 0 && load2 != 1) ||
+        (load2 && row_tile == 64u) ||
+        (row_tile != 64u && row_tile != 128u && row_tile != 256u)) {
+        return 0;
+    }
+    const uint64_t minimum_row_bytes =
+        ((uint64_t)in_dim / CUDA_QK_K) * sizeof(cuda_block_q4_K);
+    if (row_bytes < minimum_row_bytes ||
+        (load2 && !rocm_q4_K_prefill_wmma_load2_compatible(
+            reinterpret_cast<const float *>(x),
+            x_token_stride, x_group_stride))) {
+        return 0;
+    }
+    if (k_tile == ROCM_Q4_WMMA_K64_TILE) {
+        rocm_q4_K_prefill_wmma_k64_enqueue(
+            reinterpret_cast<float *>(out),
+            reinterpret_cast<const char *>(w),
+            reinterpret_cast<const float *>(x),
+            n_tok, n_groups, in_dim, out_dim, row_bytes,
+            x_token_stride, x_group_stride, out_token_stride, row_tile, load2);
+    } else {
+        rocm_q4_K_prefill_wmma_enqueue(
+            reinterpret_cast<float *>(out),
+            reinterpret_cast<const char *>(w),
+            reinterpret_cast<const float *>(x),
+            n_tok, n_groups, in_dim, out_dim, row_bytes,
+            x_token_stride, x_group_stride, out_token_stride, row_tile, load2);
+    }
+    return 1;
+}
+
+/* Strict direct hook for the q_b K128/P144 candidate.  Unlike production
+ * dispatch, incompatibility fails instead of silently falling back to K64 so
+ * benchmark output cannot be mislabeled. */
+extern "C" int ds4_rocm_bench_q4_K_wmma_k128_enqueue(
+        void *out,
+        const void *w,
+        const void *x,
+        uint32_t n_tok,
+        uint32_t n_groups,
+        uint32_t in_dim,
+        uint32_t out_dim,
+        uint64_t row_bytes,
+        uint64_t x_token_stride,
+        uint64_t x_group_stride,
+        uint64_t out_token_stride) {
+    if (!out || !w || !x || n_tok == 0u || n_groups == 0u ||
+        in_dim == 0u || out_dim == 0u ||
+        (in_dim % CUDA_QK_K) != 0u || out_dim < 8192u ||
+        !rocm_q4_K_prefill_wmma_load4_compatible(
+            reinterpret_cast<const float *>(x),
+            x_token_stride, x_group_stride)) {
+        return 0;
+    }
+    const uint64_t minimum_row_bytes =
+        ((uint64_t)in_dim / CUDA_QK_K) * sizeof(cuda_block_q4_K);
+    if (row_bytes < minimum_row_bytes) return 0;
+    rocm_q4_K_prefill_wmma_k128_enqueue(
+        reinterpret_cast<float *>(out),
+        reinterpret_cast<const char *>(w),
+        reinterpret_cast<const float *>(x),
+        n_tok, n_groups, in_dim, out_dim, row_bytes,
+        x_token_stride, x_group_stride, out_token_stride);
+    return 1;
+}
+
+enum {
+    ROCM_Q4_PREFILL_K1024_TILE4_REQUIRED_FAILURE = -1,
+    ROCM_Q4_PREFILL_K1024_TILE4_FALLBACK = 0,
+    ROCM_Q4_PREFILL_K1024_TILE4_USE = 1,
+};
+
+/* Keep this decision independent from the device lookup so the complete
+ * policy matrix has a hardware-free oracle.  Resident execution preserves
+ * the established automatic default.  SSD execution stays opt-in and may
+ * only select TILE4 after the exact weight range has been found in actual
+ * device storage; mapped/registered host memory is deliberately insufficient.
+ * REQUIRE requests the candidate as well as asserting it, while DISABLE is
+ * authoritative in both modes. */
+static int rocm_q4_K_prefill_k1024_tile4_policy(
+        int ssd_streaming,
+        int weight_device_resident,
+        int ssd_enabled,
+        int disabled,
+        int required) {
+    if (disabled) {
+        return required ? ROCM_Q4_PREFILL_K1024_TILE4_REQUIRED_FAILURE
+                        : ROCM_Q4_PREFILL_K1024_TILE4_FALLBACK;
+    }
+    if (!ssd_streaming) return ROCM_Q4_PREFILL_K1024_TILE4_USE;
+    if (!ssd_enabled && !required) {
+        return ROCM_Q4_PREFILL_K1024_TILE4_FALLBACK;
+    }
+    if (!weight_device_resident) {
+        return required ? ROCM_Q4_PREFILL_K1024_TILE4_REQUIRED_FAILURE
+                        : ROCM_Q4_PREFILL_K1024_TILE4_FALLBACK;
+    }
+    return ROCM_Q4_PREFILL_K1024_TILE4_USE;
+}
+
+/* Test-only pure-policy entry point.  It intentionally performs no HIP call,
+ * so hosts with a ROCm toolchain but no visible device can still validate the
+ * SSD default, residency gate, and DISABLE/REQUIRE precedence. */
+extern "C" int ds4_rocm_test_q4_prefill_k1024_tile4_policy(
+        int ssd_streaming,
+        int weight_device_resident,
+        int ssd_enabled,
+        int disabled,
+        int required) {
+    return rocm_q4_K_prefill_k1024_tile4_policy(
+        ssd_streaming != 0, weight_device_resident != 0,
+        ssd_enabled != 0, disabled != 0, required != 0);
+}
+
+static int rocm_q4_K_prefill_k1024_tile4_resolve(
+        uint64_t blocks,
+        uint64_t out_dim,
+        const void *model_map,
+        uint64_t weight_offset,
+        uint64_t weight_bytes,
+        const char *weight_ptr) {
+    if (blocks != ROCM_Q4_PREFILL_K1024_KBLOCK_TILE ||
+        out_dim != DS4_ROCM_Q4_ATTN_Q_B_OUT_DIM) {
+        return ROCM_Q4_PREFILL_K1024_TILE4_FALLBACK;
+    }
+
+    const int enabled = rocm_q4_attn_q_b_env_bool(
+        "DS4_ROCM_ENABLE_Q4_PREFILL_K1024_TILE4_SSD") == 1;
+    const int disabled = rocm_q4_attn_q_b_env_bool(
+        "DS4_ROCM_DISABLE_Q4_PREFILL_K1024_TILE4") == 1;
+    const int required = rocm_q4_attn_q_b_env_bool(
+        "DS4_ROCM_REQUIRE_Q4_PREFILL_K1024_TILE4") == 1;
+    const char *resident_ptr = g_ssd_streaming_mode
+        ? rocm_q4_attn_q_b_device_resident_source(
+              model_map, weight_offset, weight_bytes)
+        : weight_ptr;
+    const int weight_device_resident =
+        resident_ptr != NULL && resident_ptr == weight_ptr;
+    const int decision = rocm_q4_K_prefill_k1024_tile4_policy(
+        g_ssd_streaming_mode, weight_device_resident, enabled, disabled,
+        required);
+    if (decision == ROCM_Q4_PREFILL_K1024_TILE4_REQUIRED_FAILURE) {
+        if (disabled) {
+            fprintf(stderr,
+                    "ds4: required ROCm Q4_K prefill K1024 tile4 is "
+                    "disabled\n");
+        } else {
+            fprintf(stderr,
+                    "ds4: required ROCm Q4_K prefill K1024 tile4 has no "
+                    "device-resident SSD weight range "
+                    "(offset=%llu bytes=%llu)\n",
+                    (unsigned long long)weight_offset,
+                    (unsigned long long)weight_bytes);
+        }
+    }
+    return decision;
+}
+
+static uint64_t g_rocm_q4_prefill_tile8_dense_calls;
+static uint64_t g_rocm_q4_prefill_tile8_pair_calls;
+static uint64_t g_rocm_q4_prefill_tile8_attention_batch_calls;
+static uint64_t g_rocm_q4_prefill_k1024_tile4_calls;
+static uint64_t g_rocm_q4_prefill_k1024_tile4_ssd_calls;
+static uint64_t g_rocm_q4_prefill_tile8_tokens;
+static pthread_mutex_t g_rocm_q4_prefill_tile8_stats_mutex =
+    PTHREAD_MUTEX_INITIALIZER;
+
+static uint64_t g_rocm_q4_grouped_attn_a_calls;
+static uint64_t g_rocm_q4_grouped_attn_a_dispatches;
+static uint64_t g_rocm_q4_grouped_attn_a_groups;
+static uint64_t g_rocm_q4_grouped_attn_a_fallbacks;
+static uint64_t g_rocm_q4_grouped_attn_a_failures;
+static int g_rocm_q4_grouped_attn_a_report_registered;
+static pthread_mutex_t g_rocm_q4_grouped_attn_a_stats_mutex =
+    PTHREAD_MUTEX_INITIALIZER;
+
+static void rocm_q4_K_grouped_attn_a_report(void) {
+    pthread_mutex_lock(&g_rocm_q4_grouped_attn_a_stats_mutex);
+    const uint64_t calls = g_rocm_q4_grouped_attn_a_calls;
+    const uint64_t dispatches = g_rocm_q4_grouped_attn_a_dispatches;
+    const uint64_t groups = g_rocm_q4_grouped_attn_a_groups;
+    const uint64_t fallbacks = g_rocm_q4_grouped_attn_a_fallbacks;
+    const uint64_t failures = g_rocm_q4_grouped_attn_a_failures;
+    pthread_mutex_unlock(&g_rocm_q4_grouped_attn_a_stats_mutex);
+    fprintf(stderr,
+            "ds4: ROCm Q4_K grouped attention-A decode stats: "
+            "calls=%llu dispatches=%llu groups=%llu fallbacks=%llu failures=%llu\n",
+            (unsigned long long)calls,
+            (unsigned long long)dispatches,
+            (unsigned long long)groups,
+            (unsigned long long)fallbacks,
+            (unsigned long long)failures);
+}
+
+static int rocm_q4_K_grouped_attn_a_result(int rc, uint32_t n_groups) {
+    if (getenv("DS4_ROCM_Q4_GROUPED_ATTN_A_STATS") != NULL) {
+        pthread_mutex_lock(&g_rocm_q4_grouped_attn_a_stats_mutex);
+        if (!g_rocm_q4_grouped_attn_a_report_registered) {
+            g_rocm_q4_grouped_attn_a_report_registered = 1;
+            (void)atexit(rocm_q4_K_grouped_attn_a_report);
+        }
+        g_rocm_q4_grouped_attn_a_calls++;
+        if (rc > 0) {
+            g_rocm_q4_grouped_attn_a_dispatches++;
+            g_rocm_q4_grouped_attn_a_groups += n_groups;
+        } else if (rc < 0) {
+            g_rocm_q4_grouped_attn_a_failures++;
+        } else {
+            g_rocm_q4_grouped_attn_a_fallbacks++;
+        }
+        pthread_mutex_unlock(&g_rocm_q4_grouped_attn_a_stats_mutex);
+    }
+    return rc;
+}
+
+static void rocm_q4_K_prefill_tile8_report(void) {
+    const uint64_t wmma_calls =
+        __atomic_load_n(&g_rocm_q4_prefill_wmma_launches,
+                        __ATOMIC_RELAXED);
+    const uint64_t wmma_k64_calls =
+        __atomic_load_n(&g_rocm_q4_prefill_wmma_k64_launches,
+                        __ATOMIC_RELAXED);
+    const uint64_t wmma_k128_calls =
+        __atomic_load_n(&g_rocm_q4_prefill_wmma_k128_launches,
+                        __ATOMIC_RELAXED);
+    const uint64_t staged_calls = wmma_k64_calls + wmma_k128_calls;
+    const uint64_t wmma_k32_calls =
+        wmma_calls >= staged_calls ? wmma_calls - staged_calls : 0u;
+    pthread_mutex_lock(&g_rocm_q4_prefill_tile8_stats_mutex);
+    const uint64_t dense_calls = g_rocm_q4_prefill_tile8_dense_calls;
+    const uint64_t pair_calls = g_rocm_q4_prefill_tile8_pair_calls;
+    const uint64_t attention_batch_calls =
+        g_rocm_q4_prefill_tile8_attention_batch_calls;
+    const uint64_t k1024_tile4_calls =
+        g_rocm_q4_prefill_k1024_tile4_calls;
+    const uint64_t k1024_tile4_ssd_calls =
+        g_rocm_q4_prefill_k1024_tile4_ssd_calls;
+    const uint64_t tokens = g_rocm_q4_prefill_tile8_tokens;
+    pthread_mutex_unlock(&g_rocm_q4_prefill_tile8_stats_mutex);
+    fprintf(stderr,
+            "ds4: ROCm Q4_K tiled-prefill stats: "
+            "dense_calls=%llu pair_calls=%llu attention_batch_calls=%llu "
+            "k1024_tile4_calls=%llu k1024_tile4_ssd_calls=%llu "
+            "wmma_calls=%llu wmma_k32_calls=%llu wmma_k64_calls=%llu "
+            "wmma_k128_calls=%llu tokens=%llu\n",
+            (unsigned long long)dense_calls,
+            (unsigned long long)pair_calls,
+            (unsigned long long)attention_batch_calls,
+            (unsigned long long)k1024_tile4_calls,
+            (unsigned long long)k1024_tile4_ssd_calls,
+            (unsigned long long)wmma_calls,
+            (unsigned long long)wmma_k32_calls,
+            (unsigned long long)wmma_k64_calls,
+            (unsigned long long)wmma_k128_calls,
+            (unsigned long long)tokens);
+}
+
+static void rocm_q4_K_prefill_tile8_note(
+        uint32_t dense_calls,
+        uint32_t pair_calls,
+        uint32_t attention_batch_calls,
+        uint32_t k1024_tile4_calls,
+        uint64_t tokens) {
+    if (getenv("DS4_ROCM_Q4_PREFILL_TILE8_STATS") == NULL) return;
+    rocm_q4_K_prefill_stats_register();
+    pthread_mutex_lock(&g_rocm_q4_prefill_tile8_stats_mutex);
+    g_rocm_q4_prefill_tile8_dense_calls += dense_calls;
+    g_rocm_q4_prefill_tile8_pair_calls += pair_calls;
+    g_rocm_q4_prefill_tile8_attention_batch_calls += attention_batch_calls;
+    g_rocm_q4_prefill_k1024_tile4_calls += k1024_tile4_calls;
+    if (k1024_tile4_calls && g_ssd_streaming_mode) {
+        g_rocm_q4_prefill_k1024_tile4_ssd_calls += k1024_tile4_calls;
+    }
+    g_rocm_q4_prefill_tile8_tokens += tokens;
+    pthread_mutex_unlock(&g_rocm_q4_prefill_tile8_stats_mutex);
+}
+
+extern "C" int ds4_rocm_matmul_q4_K_tensor(
+        ds4_gpu_tensor *out,
+        const void *model_map,
+        uint64_t model_size,
+        uint64_t weight_offset,
+        uint64_t in_dim,
+        uint64_t out_dim,
+        const ds4_gpu_tensor *x,
+        uint64_t n_tok) {
+    uint64_t blocks = 0;
+    uint64_t row_bytes = 0;
+    uint64_t weight_bytes = 0;
+    if (!rocm_q4_K_dense_validate(out, model_map, model_size, weight_offset,
+                                  in_dim, out_dim, x, n_tok, &blocks,
+                                  &row_bytes, &weight_bytes)) {
+        return 0;
+    }
+
+    const int prefill_scope = rocm_q4_K_prefill_tile8_scope(n_tok);
+    const int prefill_tile8 = rocm_q4_K_prefill_tile8_requested();
+    const int prefill_tile8_required = rocm_q4_K_prefill_tile8_required();
+    const int q8_wave32 = rocm_q4_K_prefill_q8_wave32_select(n_tok);
+    const int q8_wave32_required =
+        rocm_q4_K_prefill_q8_wave32_required();
+    if (q8_wave32 == ROCM_Q4_PREFILL_Q8_WAVE32_REQUIRED_FAILURE) {
+        return 0;
+    }
+    const int k1024_tile4_shape =
+        blocks == ROCM_Q4_PREFILL_K1024_KBLOCK_TILE &&
+        out_dim == DS4_ROCM_Q4_ATTN_Q_B_OUT_DIM;
+    const int k1024_tile4_required = rocm_q4_attn_q_b_env_bool(
+        "DS4_ROCM_REQUIRE_Q4_PREFILL_K1024_TILE4") == 1;
+    if (prefill_scope && prefill_tile8_required && !prefill_tile8) {
+        fprintf(stderr,
+                "ds4: required ROCm Q4_K prefill tile8 is disabled "
+                "(n_tok=%llu)\n",
+                (unsigned long long)n_tok);
+        return 0;
+    }
+    if (prefill_scope && k1024_tile4_shape && k1024_tile4_required &&
+        !prefill_tile8) {
+        fprintf(stderr,
+                "ds4: required ROCm Q4_K prefill K1024 tile4 cannot run "
+                "because tiled prefill is disabled (n_tok=%llu)\n",
+                (unsigned long long)n_tok);
+        return 0;
+    }
+
+    const char *wptr = cuda_model_range_ptr(model_map, weight_offset,
+                                            weight_bytes, "q4_K dense");
+    if (!wptr) return 0;
+    const char *resident_wptr = g_ssd_streaming_mode
+        ? rocm_q4_attn_q_b_device_resident_source(
+              model_map, weight_offset, weight_bytes)
+        : wptr;
+    const int weight_device_resident =
+        resident_wptr != NULL && resident_wptr == wptr;
+    int prefill_wmma = rocm_q4_K_prefill_wmma_select(
+        n_tok, in_dim, out_dim, weight_device_resident);
+    if (prefill_wmma == ROCM_Q4_PREFILL_WMMA_REQUIRED_FAILURE) return 0;
+    if (prefill_wmma == ROCM_Q4_PREFILL_WMMA_USE &&
+        q8_wave32 == ROCM_Q4_PREFILL_Q8_WAVE32_USE) {
+        const int wmma_required = rocm_q4_attn_q_b_env_bool(
+            "DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA") == 1;
+        if (wmma_required && q8_wave32_required) {
+            fprintf(stderr,
+                    DS4_GPU_LOG_PREFIX
+                    "Q4_K prefill cannot require both direct WMMA and "
+                    "the Q8_K wave32 quantizer\n");
+            return 0;
+        }
+        /* The direct F16 WMMA path has no Q8_K RHS.  An explicitly selected
+         * exact quantizer therefore owns an optional automatic WMMA path. */
+        if (rocm_q4_K_prefill_wmma_yields_to_q8_wave32(
+                q8_wave32, wmma_required)) {
+            prefill_wmma = ROCM_Q4_PREFILL_WMMA_FALLBACK;
+        }
+    }
+    if (prefill_wmma == ROCM_Q4_PREFILL_WMMA_USE &&
+        prefill_scope && prefill_tile8_required) {
+        if (rocm_q4_attn_q_b_env_bool(
+                "DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA") == 1) {
+            fprintf(stderr,
+                    DS4_GPU_LOG_PREFIX
+                    "Q4_K prefill cannot require both WMMA and TILE8\n");
+            return 0;
+        }
+        /* REQUIRE_TILE8 owns the dispatch when WMMA is only an optional
+         * experiment. */
+        prefill_wmma = ROCM_Q4_PREFILL_WMMA_FALLBACK;
+    }
+    if (prefill_wmma == ROCM_Q4_PREFILL_WMMA_USE &&
+        k1024_tile4_shape && k1024_tile4_required) {
+        if (rocm_q4_attn_q_b_env_bool(
+                "DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA") == 1) {
+            fprintf(stderr,
+                    DS4_GPU_LOG_PREFIX
+                    "Q4_K prefill cannot require both WMMA and K1024 TILE4\n");
+            return 0;
+        }
+        prefill_wmma = ROCM_Q4_PREFILL_WMMA_FALLBACK;
+    }
+    if (prefill_wmma == ROCM_Q4_PREFILL_WMMA_USE) {
+        return rocm_q4_K_prefill_wmma_launch(
+            reinterpret_cast<float *>(out->ptr), wptr,
+            reinterpret_cast<const float *>(x->ptr),
+            (uint32_t)n_tok, 1u, (uint32_t)in_dim, (uint32_t)out_dim,
+            row_bytes, in_dim, 0u, out_dim,
+            "q4_K dense prefill WMMA rowtile launch");
+    }
+    int k1024_tile4 = ROCM_Q4_PREFILL_K1024_TILE4_FALLBACK;
+    if (prefill_scope && prefill_tile8) {
+        k1024_tile4 = rocm_q4_K_prefill_k1024_tile4_resolve(
+            blocks, out_dim, model_map, weight_offset, weight_bytes, wptr);
+        if (k1024_tile4 ==
+            ROCM_Q4_PREFILL_K1024_TILE4_REQUIRED_FAILURE) {
+            return 0;
+        }
+    }
+    cuda_block_q8_K *xq = rocm_q4_K_prequant_alloc(
+            n_tok, blocks, "q4_K dense prequant");
+    if (!xq) return 0;
+
+    if (!rocm_q4_K_q8_quantize_launch(
+            xq, reinterpret_cast<const float *>(x->ptr),
+            (uint32_t)in_dim, (uint32_t)n_tok, q8_wave32,
+            q8_wave32_required,
+            "q4_K dense quantize launch")) {
+        return 0;
+    }
+
+    if (prefill_scope && prefill_tile8) {
+        if (k1024_tile4 == ROCM_Q4_PREFILL_K1024_TILE4_USE) {
+            const dim3 tiled_grid(
+                (unsigned)((out_dim - 1u) /
+                           ROCM_Q4_PREFILL_K1024_ROWS + 1u),
+                (unsigned)((n_tok - 1u) /
+                           ROCM_Q4_PREFILL_TOKEN_TILE + 1u),
+                1u);
+            rocm_matmul_q4_K_prefill_k1024_tile4_kernel<<<tiled_grid, 256>>>(
+                    reinterpret_cast<float *>(out->ptr), wptr, xq,
+                    row_bytes, (uint32_t)out_dim, (uint32_t)n_tok);
+            const int ok = cuda_ok(
+                    cudaGetLastError(),
+                    "q4_K dense prefill K1024 tile4 launch");
+            if (ok) {
+                rocm_q4_K_prefill_tile8_note(1u, 0u, 0u, 1u, n_tok);
+            }
+            return ok;
+        }
+        const dim3 tiled_grid((unsigned)((out_dim - 1u) / 32u + 1u),
+                              (unsigned)((n_tok - 1u) /
+                                         ROCM_Q4_PREFILL_TOKEN_TILE + 1u),
+                              1u);
+        rocm_matmul_q4_K_prefill_tile8_strided_kernel<<<tiled_grid, 256>>>(
+                reinterpret_cast<float *>(out->ptr), wptr, xq, row_bytes,
+                (uint32_t)blocks, (uint32_t)out_dim, (uint32_t)n_tok,
+                blocks, out_dim);
+        const int ok = cuda_ok(cudaGetLastError(),
+                               "q4_K dense prefill tile8 launch");
+        if (ok) rocm_q4_K_prefill_tile8_note(1u, 0u, 0u, 0u, n_tok);
+        return ok;
+    }
+
+    const dim3 grid((unsigned)((out_dim - 1u) / 32u + 1u),
+                    (unsigned)n_tok, 1u);
+    rocm_matmul_q4_K_dense_kernel<<<grid, 256>>>(
+            reinterpret_cast<float *>(out->ptr), wptr, xq, row_bytes,
+            (uint32_t)blocks, (uint32_t)out_dim, (uint32_t)n_tok);
+    return cuda_ok(cudaGetLastError(), "q4_K dense matmul launch");
+}
+
+/* Only REQUIRE_TILE8 makes the fused pair itself a strict contract.  A
+ * Q8-wave32-only REQUIRE belongs to the quantizer and can still be satisfied
+ * by the graph's two dense fallbacks.  Keep this pre-enqueue policy in one
+ * place so validation, alias/range rejection and scratch allocation cannot
+ * silently weaken the pair contract.  Decode is outside prefill scope and
+ * therefore keeps its legacy optional status. */
+static int rocm_q4_K_pair_pre_enqueue_failure_policy(
+        int prefill_scope,
+        int tile8_required,
+        int q8_wave32_required) {
+    (void)q8_wave32_required;
+    return prefill_scope && tile8_required ? -1 : 0;
+}
+
+extern "C" int ds4_rocm_test_q4_pair_pre_enqueue_failure_policy(
+        int prefill_scope,
+        int tile8_required,
+        int q8_wave32_required) {
+    return rocm_q4_K_pair_pre_enqueue_failure_policy(
+        prefill_scope != 0, tile8_required != 0,
+        q8_wave32_required != 0);
+}
+
+extern "C" int ds4_gpu_matmul_q4_K_pair_tensor(
+        ds4_gpu_tensor *out0,
+        ds4_gpu_tensor *out1,
+        const void *model_map,
+        uint64_t model_size,
+        uint64_t weight0_offset,
+        uint64_t weight1_offset,
+        uint64_t in_dim,
+        uint64_t out0_dim,
+        uint64_t out1_dim,
+        const ds4_gpu_tensor *x,
+        uint64_t n_tok) {
+    const int prefill_scope = rocm_q4_K_prefill_tile8_scope(n_tok);
+    const int tile8_required = rocm_q4_K_prefill_tile8_required();
+    const int prefill_required = prefill_scope && tile8_required;
+    const int wmma_required = rocm_q4_attn_q_b_env_bool(
+        "DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA") == 1;
+    const int q8_wave32 = rocm_q4_K_prefill_q8_wave32_select(n_tok);
+    const int q8_wave32_required =
+        rocm_q4_K_prefill_q8_wave32_required();
+    if (q8_wave32 == ROCM_Q4_PREFILL_Q8_WAVE32_REQUIRED_FAILURE) {
+        return -1;
+    }
+    const int pre_enqueue_failure =
+        rocm_q4_K_pair_pre_enqueue_failure_policy(
+            prefill_scope, tile8_required, q8_wave32_required);
+    if (q8_wave32_required && wmma_required) {
+        fprintf(stderr,
+                DS4_GPU_LOG_PREFIX
+                "Q4_K prefill pair cannot require both direct WMMA and "
+                "the Q8_K wave32 quantizer\n");
+        return -1;
+    }
+
+    /* The fused pair consumes a shared Q8_K activation tile.  When the direct
+     * F16 WMMA path is selected, return before validation/enqueue so the
+     * graph's established fallback issues two dense calls and both can take
+     * the strict WMMA path.  This also prevents REQUIRE from falsely passing
+     * after silently measuring the TILE8 pair.  SSD preflight is deliberately
+     * optimistic about residency: it only decides whether to yield; each dense
+     * fallback then proves its exact physical device range before enqueue. */
+    if (!prefill_required &&
+        !rocm_q4_K_prefill_wmma_yields_to_q8_wave32(
+            q8_wave32, wmma_required)) {
+        const int wmma0 = rocm_q4_K_prefill_wmma_select(
+            n_tok, in_dim, out0_dim, 1);
+        const int wmma1 = rocm_q4_K_prefill_wmma_select(
+            n_tok, in_dim, out1_dim, 1);
+        if (wmma0 == ROCM_Q4_PREFILL_WMMA_REQUIRED_FAILURE ||
+            wmma1 == ROCM_Q4_PREFILL_WMMA_REQUIRED_FAILURE) {
+            return -1;
+        }
+        if (wmma0 == ROCM_Q4_PREFILL_WMMA_USE ||
+            wmma1 == ROCM_Q4_PREFILL_WMMA_USE) {
+            return 0;
+        }
+    } else if (wmma_required) {
+        fprintf(stderr,
+                DS4_GPU_LOG_PREFIX
+                "Q4_K prefill pair cannot require both WMMA and TILE8\n");
+        return -1;
+    }
+
+    const int prefill_pair = prefill_scope &&
+                             rocm_q4_K_prefill_tile8_requested();
+    if (prefill_required && !prefill_pair) {
+        fprintf(stderr,
+                "ds4: required ROCm Q4_K prefill tile8 pair is disabled "
+                "(n_tok=%llu)\n",
+                (unsigned long long)n_tok);
+        return -1;
+    }
+
+    /* Decode keeps its original, separately gated pair path.  Prefill uses
+     * the common tile8 gate and shares one canonical Q8_K quantization and
+     * one tiled launch across the two projections. */
+    const int decode_pair = n_tok <= 8u &&
+                            rocm_q4_K_dense_pair_requested();
+    if (!prefill_pair && !decode_pair) {
+        return pre_enqueue_failure;
+    }
+    if (!out0 || !out1 || out0 == out1 ||
+        (out0 && out1 && out0->ptr == out1->ptr)) {
+        return pre_enqueue_failure;
+    }
+
+    uint64_t blocks0 = 0, blocks1 = 0;
+    uint64_t row_bytes0 = 0, row_bytes1 = 0;
+    uint64_t weight0_bytes = 0, weight1_bytes = 0;
+    if (!rocm_q4_K_dense_validate(out0, model_map, model_size, weight0_offset,
+                                  in_dim, out0_dim, x, n_tok, &blocks0,
+                                  &row_bytes0, &weight0_bytes) ||
+        !rocm_q4_K_dense_validate(out1, model_map, model_size, weight1_offset,
+                                  in_dim, out1_dim, x, n_tok, &blocks1,
+                                  &row_bytes1, &weight1_bytes) ||
+        blocks0 != blocks1 || row_bytes0 != row_bytes1) {
+        return pre_enqueue_failure;
+    }
+    uint64_t out0_bytes = 0;
+    uint64_t out1_bytes = 0;
+    if (!cuda_u64_mul3_checked(n_tok, out0_dim, sizeof(float), &out0_bytes) ||
+        !cuda_u64_mul3_checked(n_tok, out1_dim, sizeof(float), &out1_bytes) ||
+        rocm_q4_K_byte_ranges_overlap(out0->ptr, out0_bytes,
+                                      out1->ptr, out1_bytes)) {
+        return pre_enqueue_failure;
+    }
+
+    const char *w0 = cuda_model_range_ptr(model_map, weight0_offset,
+                                          weight0_bytes, "q4_K dense pair0");
+    const char *w1 = cuda_model_range_ptr(model_map, weight1_offset,
+                                          weight1_bytes, "q4_K dense pair1");
+    if (!w0 || !w1) return pre_enqueue_failure;
+    cuda_block_q8_K *xq = rocm_q4_K_prequant_alloc(
+            n_tok, blocks0, "q4_K dense pair prequant");
+    if (!xq) return pre_enqueue_failure;
+
+    if (!rocm_q4_K_q8_quantize_launch(
+            xq, reinterpret_cast<const float *>(x->ptr),
+            (uint32_t)in_dim, (uint32_t)n_tok, q8_wave32,
+            q8_wave32_required,
+            "q4_K dense pair quantize launch")) {
+        return -1;
+    }
+
+    if (prefill_pair) {
+        const uint64_t out0_tiles = (out0_dim - 1u) / 32u + 1u;
+        const uint64_t out1_tiles = (out1_dim - 1u) / 32u + 1u;
+        const dim3 grid((unsigned)(out0_tiles + out1_tiles),
+                        (unsigned)((n_tok - 1u) /
+                                   ROCM_Q4_PREFILL_TOKEN_TILE + 1u),
+                        1u);
+        rocm_matmul_q4_K_prefill_tile8_pair_kernel<<<grid, 256>>>(
+                reinterpret_cast<float *>(out0->ptr),
+                reinterpret_cast<float *>(out1->ptr), w0, w1, xq,
+                row_bytes0, (uint32_t)blocks0, (uint32_t)out0_dim,
+                (uint32_t)out1_dim, (uint32_t)n_tok);
+        const int ok = cuda_ok(cudaGetLastError(),
+                               "q4_K dense prefill pair tile8 launch");
+        if (ok) rocm_q4_K_prefill_tile8_note(0u, 1u, 0u, 0u, n_tok);
+        return ok ? 1 : -1;
+    }
+
+    const uint64_t out0_tiles = (out0_dim - 1u) / 32u + 1u;
+    const uint64_t out1_tiles = (out1_dim - 1u) / 32u + 1u;
+    const dim3 grid((unsigned)(out0_tiles + out1_tiles),
+                    (unsigned)n_tok, 1u);
+    rocm_matmul_q4_K_dense_pair_kernel<<<grid, 256>>>(
+            reinterpret_cast<float *>(out0->ptr),
+            reinterpret_cast<float *>(out1->ptr), w0, w1, xq,
+            row_bytes0, (uint32_t)blocks0, (uint32_t)out0_dim,
+            (uint32_t)out1_dim, (uint32_t)n_tok);
+    return cuda_ok(cudaGetLastError(), "q4_K dense pair matmul launch")
+        ? 1 : -1;
+}
+
+extern "C" int ds4_gpu_attention_output_low_q4_K_slice_tensor(
+        ds4_gpu_tensor *low, const void *model_map, uint64_t model_size,
+        uint64_t out_a_offset, uint64_t group_dim, uint64_t rank,
+        uint32_t group0, uint32_t group_cnt,
+        const ds4_gpu_tensor *heads, int resident_decode) {
+    const int disabled =
+        getenv("DS4_ROCM_DISABLE_Q4_GROUPED_ATTN_A") != NULL;
+    const int required =
+        getenv("DS4_ROCM_REQUIRE_Q4_GROUPED_ATTN_A") != NULL;
+    const int enabled =
+        getenv("DS4_ROCM_ENABLE_Q4_GROUPED_ATTN_A") != NULL;
+    const int resident_default =
+        rocm_q4_K_grouped_attn_a_resident_default_scope(
+            group_dim, rank, group0, group_cnt, resident_decode);
+    /* DISABLE is authoritative; REQUIRE reports that rollback as a failure
+     * instead of allowing the graph to false-green through its fallback. */
+    if (disabled) {
+        if (required) {
+            fprintf(stderr,
+                    "ds4: required ROCm Q4_K grouped attention-A decode "
+                    "is disabled\n");
+        }
+        return rocm_q4_K_grouped_attn_a_result(required ? -1 : 0, 0u);
+    }
+    if (!resident_default && !enabled && !required) {
+        return rocm_q4_K_grouped_attn_a_result(0, 0u);
+    }
+    const int pre_enqueue_failure = required ? -1 : 0;
+    if (!low || !heads || !model_map || group_dim == 0u || rank == 0u ||
+        group_cnt == 0u || group_dim > UINT32_MAX || rank > UINT32_MAX ||
+        group_cnt > UINT16_MAX || (group_dim % CUDA_QK_K) != 0u ||
+        group0 > UINT32_MAX - group_cnt) {
+        return rocm_q4_K_grouped_attn_a_result(pre_enqueue_failure, 0u);
+    }
+
+    const uint64_t blocks = group_dim / CUDA_QK_K;
+    uint64_t row_bytes = 0, group_weight_bytes = 0, group_skip = 0;
+    uint64_t selected_weight_bytes = 0, selected_offset = 0;
+    if (blocks == 0u ||
+        !cuda_u64_mul_checked(blocks, sizeof(cuda_block_q4_K), &row_bytes) ||
+        !cuda_u64_mul_checked(rank, row_bytes, &group_weight_bytes) ||
+        !cuda_u64_mul_checked(group0, group_weight_bytes, &group_skip) ||
+        !cuda_u64_mul_checked(group_cnt, group_weight_bytes,
+                              &selected_weight_bytes) ||
+        !cuda_u64_add_checked(out_a_offset, group_skip, &selected_offset) ||
+        !cuda_model_range_fits(model_size, selected_offset,
+                               selected_weight_bytes) ||
+        !cuda_tensor_has_elems2(heads, group_cnt, group_dim, sizeof(float)) ||
+        !cuda_tensor_has_elems2(low, group_cnt, rank, sizeof(float))) {
+        return rocm_q4_K_grouped_attn_a_result(pre_enqueue_failure, 0u);
+    }
+
+    const char *w = cuda_model_range_ptr(
+        model_map, selected_offset, selected_weight_bytes,
+        "q4_K grouped attention output A decode");
+    cuda_block_q8_K *xq = rocm_q4_K_prequant_alloc(
+        group_cnt, blocks, "q4_K grouped attention output A decode prequant");
+    if (!w || !xq) {
+        return rocm_q4_K_grouped_attn_a_result(pre_enqueue_failure, 0u);
+    }
+
+    const dim3 qgrid((unsigned)blocks, group_cnt, 1u);
+    q8_K_quantize_kernel<<<qgrid, 256>>>(
+        xq, reinterpret_cast<const float *>(heads->ptr),
+        (uint32_t)group_dim, group_cnt);
+    if (!cuda_ok(cudaGetLastError(),
+                 "q4_K grouped attention output A decode quantize launch")) {
+        return rocm_q4_K_grouped_attn_a_result(-1, 0u);
+    }
+    const dim3 grid((unsigned)((rank - 1u) / 32u + 1u), 1u, group_cnt);
+    rocm_matmul_q4_K_dense_grouped_decode_kernel<<<grid, 256>>>(
+        reinterpret_cast<float *>(low->ptr), w, xq, row_bytes,
+        (uint32_t)blocks, (uint32_t)rank, group_cnt);
+    if (!cuda_ok(cudaGetLastError(),
+                 "q4_K grouped attention output A decode matmul launch")) {
+        return rocm_q4_K_grouped_attn_a_result(-1, 0u);
+    }
+    return rocm_q4_K_grouped_attn_a_result(1, group_cnt);
+}
+
+/* Quantize token-major [token][group][K] rows once, then apply group-major
+ * [group][out_row][K] Q4_K weights directly into token-major output.  A
+ * return of -1 means the quantize launch was accepted and callers must not
+ * replay a row fallback over potentially submitted work. */
+static int rocm_q4_K_prefill_tile8_quant_launch(
+        float *out,
+        const char *w,
+        const float *x,
+        uint32_t n_tok,
+        uint32_t n_groups,
+        uint32_t in_dim,
+        uint32_t out_dim,
+        uint64_t row_bytes,
+        int prefill_wmma,
+        int q8_wave32,
+        int q8_wave32_required,
+        const char *label) {
+    uint64_t n_rows = 0;
+    uint64_t xq_token_stride = 0;
+    uint64_t x_token_stride = 0;
+    uint64_t out_token_stride = 0;
+    const uint64_t blocks = in_dim / CUDA_QK_K;
+    if (!out || !w || !x || n_tok == 0u || n_groups == 0u ||
+        in_dim == 0u || out_dim == 0u || blocks == 0u ||
+        (in_dim % CUDA_QK_K) != 0u ||
+        !cuda_u64_mul_checked(n_tok, n_groups, &n_rows) ||
+        /* HIP keeps the portable grid-y limit at 65535.  Real AProjQ4 uses
+         * eight groups, so even the 4096-token ceiling remains in range. */
+        n_rows > UINT16_MAX ||
+        !cuda_u64_mul_checked(n_groups, blocks, &xq_token_stride) ||
+        !cuda_u64_mul_checked(n_groups, in_dim, &x_token_stride) ||
+        !cuda_u64_mul_checked(n_groups, out_dim, &out_token_stride)) {
+        return 0;
+    }
+
+    if (prefill_wmma == ROCM_Q4_PREFILL_WMMA_USE) {
+        return rocm_q4_K_prefill_wmma_launch(
+                   out, w, x, n_tok, n_groups, in_dim, out_dim, row_bytes,
+                   x_token_stride, in_dim, out_token_stride,
+                   label ? label : "q4_K attention-output WMMA rowtile launch")
+            ? 1 : -1;
+    }
+
+    cuda_block_q8_K *xq = rocm_q4_K_prequant_alloc(
+            n_rows, blocks, label ? label : "q4_K prefill tile8 prequant");
+    if (!xq) return 0;
+
+    if (!rocm_q4_K_q8_quantize_launch(
+            xq, x, in_dim, (uint32_t)n_rows, q8_wave32,
+            q8_wave32_required,
+            "q4_K prefill tile8 quantize launch")) {
+        return 0;
+    }
+
+    const dim3 grid((unsigned)((out_dim - 1u) / 32u + 1u),
+                    (unsigned)((n_tok - 1u) /
+                               ROCM_Q4_PREFILL_TOKEN_TILE + 1u),
+                    n_groups);
+    rocm_matmul_q4_K_prefill_tile8_strided_kernel<<<grid, 256>>>(
+            out, w, xq, row_bytes, (uint32_t)blocks, out_dim, n_tok,
+            xq_token_stride, out_token_stride);
+    if (!cuda_ok(cudaGetLastError(),
+                 "q4_K prefill tile8 matmul launch")) {
+        return -1;
+    }
+    return 1;
+}
+
+extern "C" int ds4_gpu_attention_output_q4_K_batch_tensor(
+        ds4_gpu_tensor *out,
+        ds4_gpu_tensor *low,
+        ds4_gpu_tensor *group_tmp,
+        ds4_gpu_tensor *low_tmp,
+        const void *model_map,
+        uint64_t model_size,
+        uint64_t out_a_offset,
+        uint64_t out_b_offset,
+        uint32_t out_b_type,
+        uint64_t group_dim,
+        uint64_t rank,
+        uint32_t n_groups,
+        uint64_t out_dim,
+        const ds4_gpu_tensor *heads,
+        uint32_t n_tokens) {
+    (void)group_tmp;
+    (void)low_tmp;
+
+    const int tile8_scope = rocm_q4_K_prefill_tile8_scope(n_tokens);
+    const int tile8_requested = rocm_q4_K_prefill_tile8_requested();
+    const int tile8_required = tile8_scope &&
+                               rocm_q4_K_prefill_tile8_required();
+    const int q8_wave32 = rocm_q4_K_prefill_q8_wave32_select(n_tokens);
+    const int q8_wave32_required =
+        rocm_q4_K_prefill_q8_wave32_required();
+    if (q8_wave32 == ROCM_Q4_PREFILL_Q8_WAVE32_REQUIRED_FAILURE) {
+        return -1;
+    }
+    const int wmma_enabled = rocm_q4_attn_q_b_env_bool(
+        "DS4_ROCM_ENABLE_Q4_PREFILL_WMMA");
+    const int wmma_ssd_enabled = rocm_q4_attn_q_b_env_bool(
+        "DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_SSD") == 1;
+    const int wmma_disabled = rocm_q4_attn_q_b_env_bool(
+        "DS4_ROCM_DISABLE_Q4_PREFILL_WMMA") == 1;
+    const int wmma_required = rocm_q4_attn_q_b_env_bool(
+        "DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA") == 1;
+    const int a_wmma_requested =
+        rocm_q4_K_prefill_wmma_attention_a_requested_policy(
+            g_ssd_streaming_mode, wmma_enabled, wmma_ssd_enabled,
+            wmma_disabled, wmma_required);
+    const int b_wmma_requested =
+        rocm_q4_K_prefill_wmma_attention_b_requested_policy(
+            g_ssd_streaming_mode, wmma_enabled, wmma_ssd_enabled,
+            wmma_disabled, wmma_required);
+    const int attention_wmma_requested =
+        a_wmma_requested || b_wmma_requested;
+    if (!tile8_scope) {
+        /* REQUIRE is a strict diagnostic assertion.  Do not let an
+         * unsupported attention-output shape escape into a grouped/per-token
+         * fallback that never attests the requested WMMA kernel. */
+        return wmma_required ? -1 : 0;
+    }
+    if (!tile8_requested && !attention_wmma_requested) {
+        if (tile8_required || q8_wave32_required) {
+            fprintf(stderr,
+                    "ds4: required ROCm Q4_K attention-output prefill "
+                    "exact path is disabled (n_tok=%u)\n",
+                    n_tokens);
+            return -1;
+        }
+        return 0;
+    }
+    const int pre_enqueue_failure =
+        (tile8_required || wmma_required || q8_wave32_required) ? -1 : 0;
+
+    if (!out || !low || !heads || !model_map || group_dim == 0u ||
+        rank == 0u || n_groups == 0u || out_dim == 0u ||
+        n_groups > UINT16_MAX || group_dim > UINT32_MAX ||
+        rank > UINT32_MAX || out_dim > UINT32_MAX ||
+        (group_dim % CUDA_QK_K) != 0u ||
+        (out_b_type != 12u && out_b_type != 8u)) {
+        return pre_enqueue_failure;
+    }
+
+    uint64_t low_dim = 0;
+    uint64_t heads_rows = 0;
+    uint64_t heads_bytes = 0;
+    uint64_t low_bytes = 0;
+    uint64_t out_bytes = 0;
+    if (!cuda_u64_mul_checked(n_groups, rank, &low_dim) ||
+        low_dim == 0u || low_dim > UINT32_MAX ||
+        !cuda_u64_mul_checked(n_tokens, n_groups, &heads_rows) ||
+        !cuda_u64_mul3_checked(heads_rows, group_dim,
+                               sizeof(float), &heads_bytes) ||
+        !cuda_u64_mul3_checked(n_tokens, low_dim,
+                               sizeof(float), &low_bytes) ||
+        !cuda_u64_mul3_checked(n_tokens, out_dim,
+                               sizeof(float), &out_bytes) ||
+        heads->bytes < heads_bytes || low->bytes < low_bytes ||
+        out->bytes < out_bytes) {
+        return pre_enqueue_failure;
+    }
+
+    const uint64_t a_blocks = group_dim / CUDA_QK_K;
+    uint64_t row_a_bytes = 0;
+    uint64_t out_a_bytes = 0;
+    if (!cuda_u64_mul_checked(a_blocks, sizeof(cuda_block_q4_K),
+                              &row_a_bytes) ||
+        !cuda_u64_mul_checked(low_dim, row_a_bytes, &out_a_bytes) ||
+        !cuda_model_range_fits(model_size, out_a_offset, out_a_bytes)) {
+        return pre_enqueue_failure;
+    }
+
+    uint64_t row_b_bytes = 0;
+    uint64_t out_b_bytes = 0;
+    if (out_b_type == 12u) {
+        if ((low_dim % CUDA_QK_K) != 0u ||
+            !cuda_u64_mul_checked(low_dim / CUDA_QK_K,
+                                  sizeof(cuda_block_q4_K), &row_b_bytes)) {
+            return pre_enqueue_failure;
+        }
+    } else {
+        const uint64_t b_blocks = (low_dim + 31u) / 32u;
+        if (!cuda_u64_mul_checked(b_blocks, 34u, &row_b_bytes)) {
+            return pre_enqueue_failure;
+        }
+    }
+    if (!cuda_u64_mul_checked(out_dim, row_b_bytes, &out_b_bytes) ||
+        !cuda_model_range_fits(model_size, out_b_offset, out_b_bytes)) {
+        return pre_enqueue_failure;
+    }
+
+    const char *out_a = cuda_model_range_ptr(
+            model_map, out_a_offset, out_a_bytes, "q4_K attention output A");
+    const char *out_b = cuda_model_range_ptr(
+            model_map, out_b_offset, out_b_bytes, "q4_K attention output B");
+    if (!out_a || !out_b) return pre_enqueue_failure;
+
+    const char *resident_out_a = g_ssd_streaming_mode
+        ? rocm_q4_attn_q_b_device_resident_source(
+              model_map, out_a_offset, out_a_bytes)
+        : out_a;
+    const char *resident_out_b = g_ssd_streaming_mode && out_b_type == 12u
+        ? rocm_q4_attn_q_b_device_resident_source(
+              model_map, out_b_offset, out_b_bytes)
+        : out_b;
+    const int out_a_device_resident =
+        resident_out_a != NULL && resident_out_a == out_a;
+    const int out_b_device_resident =
+        resident_out_b != NULL && resident_out_b == out_b;
+
+    /* Resolve both independently requested stages before A can enqueue work.
+     * This keeps REQUIRE fail-closed: an ineligible B projection can never
+     * make the graph replay a fallback over an already submitted A projection.
+     * A retains the validated resident automatic default; B does not inherit
+     * it because applying direct WMMA to both stages compounds their F16
+     * approximations. */
+    int a_wmma = a_wmma_requested
+        ? rocm_q4_K_prefill_wmma_select(
+              n_tokens, group_dim, rank, out_a_device_resident)
+        : ROCM_Q4_PREFILL_WMMA_FALLBACK;
+    int b_wmma = out_b_type == 12u && b_wmma_requested
+        ? rocm_q4_K_prefill_wmma_select(
+              n_tokens, low_dim, out_dim, out_b_device_resident)
+        : ROCM_Q4_PREFILL_WMMA_FALLBACK;
+    if (a_wmma == ROCM_Q4_PREFILL_WMMA_REQUIRED_FAILURE ||
+        b_wmma == ROCM_Q4_PREFILL_WMMA_REQUIRED_FAILURE) {
+        return -1;
+    }
+    if (q8_wave32 == ROCM_Q4_PREFILL_Q8_WAVE32_USE &&
+        (a_wmma == ROCM_Q4_PREFILL_WMMA_USE ||
+         b_wmma == ROCM_Q4_PREFILL_WMMA_USE)) {
+        if (wmma_required && q8_wave32_required) {
+            fprintf(stderr,
+                    DS4_GPU_LOG_PREFIX
+                    "Q4_K attention-output prefill cannot require both "
+                    "direct WMMA and the Q8_K wave32 quantizer\n");
+            return -1;
+        }
+        if (rocm_q4_K_prefill_wmma_yields_to_q8_wave32(
+                q8_wave32, wmma_required)) {
+            a_wmma = ROCM_Q4_PREFILL_WMMA_FALLBACK;
+            b_wmma = ROCM_Q4_PREFILL_WMMA_FALLBACK;
+        }
+    }
+    if (tile8_required &&
+        (a_wmma == ROCM_Q4_PREFILL_WMMA_USE ||
+         b_wmma == ROCM_Q4_PREFILL_WMMA_USE)) {
+        if (wmma_required) {
+            fprintf(stderr,
+                    DS4_GPU_LOG_PREFIX
+                    "Q4_K attention-output prefill cannot require both "
+                    "WMMA and TILE8\n");
+            return -1;
+        }
+        a_wmma = ROCM_Q4_PREFILL_WMMA_FALLBACK;
+        b_wmma = ROCM_Q4_PREFILL_WMMA_FALLBACK;
+    }
+    if (!tile8_requested &&
+        (a_wmma != ROCM_Q4_PREFILL_WMMA_USE ||
+         (out_b_type == 12u &&
+          b_wmma != ROCM_Q4_PREFILL_WMMA_USE))) {
+        return pre_enqueue_failure;
+    }
+
+    /* A: the WMMA candidate consumes F32 heads directly; TILE8 retains one
+     * quantization over [token,group].  Neither path needs group pack/unpack
+     * buffers or an n_tokens*n_groups dispatch loop. */
+    const int a_rc = rocm_q4_K_prefill_tile8_quant_launch(
+            reinterpret_cast<float *>(low->ptr), out_a,
+            reinterpret_cast<const float *>(heads->ptr), n_tokens, n_groups,
+            (uint32_t)group_dim, (uint32_t)rank, row_a_bytes,
+            a_wmma, q8_wave32, q8_wave32_required,
+            "q4_K attention output A WMMA rowtile/tile8");
+    if (a_rc <= 0) {
+        return a_rc < 0 ? -1 : pre_enqueue_failure;
+    }
+
+    int b_rc = 0;
+    if (out_b_type == 12u) {
+        b_rc = rocm_q4_K_prefill_tile8_quant_launch(
+                reinterpret_cast<float *>(out->ptr), out_b,
+                reinterpret_cast<const float *>(low->ptr), n_tokens, 1u,
+                (uint32_t)low_dim, (uint32_t)out_dim, row_b_bytes,
+                b_wmma, q8_wave32, q8_wave32_required,
+                "q4_K attention output B WMMA rowtile/tile8");
+    } else {
+        b_rc = ds4_gpu_matmul_q8_0_tensor(
+                out, model_map, model_size, out_b_offset, low_dim, out_dim,
+                low, n_tokens);
+    }
+    if (b_rc <= 0) return -1;
+
+    const int used_q4_tile8 =
+        a_wmma != ROCM_Q4_PREFILL_WMMA_USE ||
+        (out_b_type == 12u && b_wmma != ROCM_Q4_PREFILL_WMMA_USE);
+    if (used_q4_tile8) {
+        rocm_q4_K_prefill_tile8_note(0u, 0u, 1u, 0u, n_tokens);
+    }
+    return 1;
+}
diff --git a/rocm/ds4_rocm_q4_qb_sidecar.cuh b/rocm/ds4_rocm_q4_qb_sidecar.cuh
new file mode 100644
index 0000000000..aee8f4c7fb
--- /dev/null
+++ b/rocm/ds4_rocm_q4_qb_sidecar.cuh
@@ -0,0 +1,1204 @@
+// Experimental resident Q4_K attn_q_b -> F16 sidecars for ROCm prefill.
+//
+// The native Q4_K/Q8_K TILE8 path remains the unconditional fallback.  This
+// cache is deliberately resident-only and opt-in: expanding every production
+// attn_q_b matrix costs 64 MiB per layer, so admission happens once, before
+// prefill, with both an explicit cache budget and device-memory headroom.
+
+enum {
+    DS4_ROCM_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES = 80u,
+    DS4_ROCM_Q4_K_TYPE = 12u,
+    DS4_ROCM_Q4_ATTN_Q_B_IN_DIM = 1024u,
+    DS4_ROCM_Q4_ATTN_Q_B_OUT_DIM = 32768u,
+};
+
+struct rocm_q4_attn_q_b_f16_cache_entry {
+    const void *model_map;
+    uint64_t model_size;
+    uint64_t weight_offset;
+    uint64_t weight_bytes;
+    uint64_t in_dim;
+    uint64_t out_dim;
+    uint32_t weight_type;
+    __half *device_ptr;
+    uint64_t f16_bytes;
+    int valid;
+};
+
+struct rocm_q4_attn_q_b_f16_arena {
+    __half *device_ptr;
+    uint64_t bytes;
+};
+
+static rocm_q4_attn_q_b_f16_cache_entry
+    g_rocm_q4_attn_q_b_f16_entries[
+        DS4_ROCM_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES];
+static rocm_q4_attn_q_b_f16_arena
+    g_rocm_q4_attn_q_b_f16_arenas[
+        DS4_ROCM_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES];
+static uint32_t g_rocm_q4_attn_q_b_f16_entry_count;
+static uint32_t g_rocm_q4_attn_q_b_f16_arena_count;
+static uint64_t g_rocm_q4_attn_q_b_f16_bytes;
+static uint64_t g_rocm_q4_attn_q_b_f16_generation = 1u;
+static uint64_t g_rocm_q4_attn_q_b_f16_lookups;
+static uint64_t g_rocm_q4_attn_q_b_f16_hits;
+static uint64_t g_rocm_q4_attn_q_b_f16_misses;
+static uint64_t g_rocm_q4_attn_q_b_f16_builds;
+static uint64_t g_rocm_q4_attn_q_b_f16_build_failures;
+static uint64_t g_rocm_q4_attn_q_b_f16_candidate_calls;
+static uint64_t g_rocm_q4_attn_q_b_f16_fallbacks;
+static uint64_t g_rocm_q4_attn_q_b_f16_rejects;
+static int g_rocm_q4_attn_q_b_f16_hard_failure;
+static int g_rocm_q4_attn_q_b_f16_pending_evict;
+/* The resident default rebuilds one layer at a time into this combined
+ * allocation. The first 64 MiB hold W_F16; the suffix holds the largest
+ * preflighted X_F16 batch and, only for the explicit F16-output experiment,
+ * Q_F16. ROCm currently submits graph work on stream 0, but keep the mutex
+ * through the complete
+ * dequant/copy/GEMM/epilogue enqueue sequence so two host callers cannot
+ * interleave reuse of any region. */
+static void *g_rocm_q4_attn_q_b_transient_f16_scratch;
+static uint64_t g_rocm_q4_attn_q_b_transient_f16_scratch_bytes;
+static uint64_t g_rocm_q4_attn_q_b_transient_f16_weight_bytes;
+static pthread_mutex_t g_rocm_q4_attn_q_b_f16_cache_mu =
+    PTHREAD_MUTEX_INITIALIZER;
+static pthread_mutex_t g_rocm_q4_attn_q_b_f16_build_mu =
+    PTHREAD_MUTEX_INITIALIZER;
+static pthread_mutex_t g_rocm_q4_attn_q_b_transient_f16_mu =
+    PTHREAD_MUTEX_INITIALIZER;
+
+static int rocm_q4_attn_q_b_env_value_eq(
+        const char *value, size_t n, const char *literal) {
+    const size_t literal_n = strlen(literal);
+    if (n != literal_n) return 0;
+    for (size_t i = 0; i < n; i++) {
+        if (tolower((unsigned char)value[i]) !=
+            tolower((unsigned char)literal[i])) {
+            return 0;
+        }
+    }
+    return 1;
+}
+
+/* Return -1 when unset, 0 for an explicit false value, and 1 otherwise.
+ * In particular, VAR=0 / false / no / off must not enable an experimental
+ * path merely because the variable exists. */
+static int rocm_q4_attn_q_b_env_bool(const char *name) {
+    const char *value = getenv(name);
+    if (!value) return -1;
+    while (isspace((unsigned char)*value)) value++;
+    size_t n = strlen(value);
+    while (n != 0u && isspace((unsigned char)value[n - 1u])) n--;
+    if (n == 0u) return 1;
+    if (rocm_q4_attn_q_b_env_value_eq(value, n, "1") ||
+        rocm_q4_attn_q_b_env_value_eq(value, n, "true") ||
+        rocm_q4_attn_q_b_env_value_eq(value, n, "yes") ||
+        rocm_q4_attn_q_b_env_value_eq(value, n, "on")) {
+        return 1;
+    }
+    if (rocm_q4_attn_q_b_env_value_eq(value, n, "0") ||
+        rocm_q4_attn_q_b_env_value_eq(value, n, "false") ||
+        rocm_q4_attn_q_b_env_value_eq(value, n, "no") ||
+        rocm_q4_attn_q_b_env_value_eq(value, n, "off")) {
+        return 0;
+    }
+    /* Preserve the project's traditional presence-enables behavior for
+     * unknown non-empty values while still handling conventional booleans. */
+    return 1;
+}
+
+static uint64_t rocm_q4_attn_q_b_env_u64(
+        const char *name,
+        uint64_t fallback,
+        uint64_t min_value,
+        uint64_t max_value) {
+    const char *value = getenv(name);
+    if (!value) return fallback;
+    while (isspace((unsigned char)*value)) value++;
+    if (!*value) return fallback;
+
+    errno = 0;
+    char *end = NULL;
+    const unsigned long long parsed = strtoull(value, &end, 10);
+    if (end == value || errno == ERANGE) return fallback;
+    while (isspace((unsigned char)*end)) end++;
+    if (*end != '\0' || parsed < min_value) return fallback;
+    const uint64_t result = (uint64_t)parsed;
+    return result > max_value ? max_value : result;
+}
+
+static int rocm_q4_attn_q_b_f16_enabled(void) {
+    return rocm_q4_attn_q_b_env_bool(
+        "DS4_ROCM_ENABLE_Q4_ATTN_Q_B_F16_CACHE") == 1;
+}
+
+static int rocm_q4_attn_q_b_f16_required(void) {
+    return rocm_q4_attn_q_b_env_bool(
+        "DS4_ROCM_REQUIRE_Q4_ATTN_Q_B_F16_CACHE") == 1;
+}
+
+static int rocm_q4_attn_q_b_f16_disabled(void) {
+    return rocm_q4_attn_q_b_env_bool(
+        "DS4_ROCM_DISABLE_Q4_ATTN_Q_B_F16_CACHE") == 1;
+}
+
+static int rocm_q4_attn_q_b_transient_f16_disabled(void) {
+    return rocm_q4_attn_q_b_env_bool(
+        "DS4_ROCM_DISABLE_Q4_ATTN_Q_B_TRANSIENT_F16") == 1;
+}
+
+static int rocm_q4_attn_q_b_f16_output_enabled(void) {
+    const char *value = getenv(
+        "DS4_ROCM_ENABLE_Q4_ATTN_Q_B_F16_OUTPUT");
+    if (!value) return 0;
+    while (isspace((unsigned char)*value)) value++;
+    if (!*value) return 0;
+    return rocm_q4_attn_q_b_env_bool(
+        "DS4_ROCM_ENABLE_Q4_ATTN_Q_B_F16_OUTPUT") == 1;
+}
+
+static uint64_t rocm_q4_attn_q_b_transient_f16_min_tokens(void) {
+    return rocm_q4_attn_q_b_env_u64(
+        "DS4_ROCM_Q4_ATTN_Q_B_TRANSIENT_F16_MIN_TOKENS",
+        4096u, 32u, UINT32_MAX);
+}
+
+static uint64_t rocm_q4_attn_q_b_f16_min_tokens(void) {
+    return rocm_q4_attn_q_b_env_u64(
+        "DS4_ROCM_Q4_ATTN_Q_B_F16_CACHE_MIN_TOKENS",
+        512u, 32u, UINT32_MAX);
+}
+
+static uint64_t rocm_q4_attn_q_b_f16_budget_bytes(void) {
+    const uint64_t budget_mib = rocm_q4_attn_q_b_env_u64(
+        "DS4_ROCM_Q4_ATTN_Q_B_F16_CACHE_MB",
+        3072u, 1u, 65536u);
+    return budget_mib * 1048576u;
+}
+
+static int rocm_q4_attn_q_b_f16_policy_allowed(void) {
+    return (rocm_q4_attn_q_b_f16_enabled() ||
+            rocm_q4_attn_q_b_f16_required()) &&
+           !rocm_q4_attn_q_b_f16_disabled() &&
+           !g_ssd_streaming_mode &&
+           !g_quality_mode &&
+           !g_q8_f16_disabled_for_multi_model;
+}
+
+static int rocm_q4_attn_q_b_transient_f16_policy_allowed(void) {
+    return !rocm_q4_attn_q_b_transient_f16_disabled() &&
+           !g_ssd_streaming_mode &&
+           !g_quality_mode &&
+           !g_q8_f16_disabled_for_multi_model;
+}
+
+/* Read-only lookup for the automatic path.  Normal full-model ROCm loading
+ * may use either a contiguous device image or hipMalloc-backed range arenas.
+ * Accept both, but never mapped/registered host memory and never populate the
+ * range cache here: that would move I/O or page migration into prefill.  Model
+ * cache construction is complete before session preflight begins. */
+static const char *rocm_q4_attn_q_b_device_resident_source(
+        const void *model_map,
+        uint64_t offset,
+        uint64_t bytes) {
+    const char *image =
+        cuda_model_image_range_ptr(model_map, offset, bytes);
+    if (image) return image;
+    if (!model_map || bytes == 0u || offset > UINT64_MAX - bytes) {
+        return NULL;
+    }
+    const uint64_t end = offset + bytes;
+    const auto exact = g_model_range_by_offset.find(offset);
+    if (exact != g_model_range_by_offset.end() &&
+        exact->second < g_model_ranges.size()) {
+        const cuda_model_range &range = g_model_ranges[exact->second];
+        if (range.host_base == model_map && !range.host_registered &&
+            range.device_ptr && range.offset == offset &&
+            bytes <= range.bytes) {
+            return range.device_ptr;
+        }
+    }
+    for (const cuda_model_range &range : g_model_ranges) {
+        if (range.host_base != model_map || range.host_registered ||
+            !range.device_ptr || offset < range.offset ||
+            range.offset > UINT64_MAX - range.bytes) {
+            continue;
+        }
+        const uint64_t range_end = range.offset + range.bytes;
+        if (end <= range_end) {
+            return range.device_ptr + (offset - range.offset);
+        }
+    }
+    return NULL;
+}
+
+static int rocm_q4_attn_q_b_f16_key_equal(
+        const rocm_q4_attn_q_b_f16_cache_entry *entry,
+        const void *model_map,
+        uint64_t model_size,
+        uint64_t weight_offset,
+        uint64_t weight_bytes,
+        uint64_t in_dim,
+        uint64_t out_dim,
+        uint32_t weight_type) {
+    return entry->valid &&
+           entry->model_map == model_map &&
+           entry->model_size == model_size &&
+           entry->weight_offset == weight_offset &&
+           entry->weight_bytes == weight_bytes &&
+           entry->in_dim == in_dim &&
+           entry->out_dim == out_dim &&
+           entry->weight_type == weight_type;
+}
+
+static rocm_q4_attn_q_b_f16_cache_entry *
+rocm_q4_attn_q_b_f16_find_locked(
+        const void *model_map,
+        uint64_t model_size,
+        uint64_t weight_offset,
+        uint64_t weight_bytes,
+        uint64_t in_dim,
+        uint64_t out_dim,
+        uint32_t weight_type) {
+    for (uint32_t i = 0;
+         i < DS4_ROCM_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES;
+         i++) {
+        rocm_q4_attn_q_b_f16_cache_entry *entry =
+            &g_rocm_q4_attn_q_b_f16_entries[i];
+        if (rocm_q4_attn_q_b_f16_key_equal(
+                entry, model_map, model_size, weight_offset, weight_bytes,
+                in_dim, out_dim, weight_type)) {
+            return entry;
+        }
+    }
+    return NULL;
+}
+
+/* On success the cache mutex remains held until the caller has enqueued the
+ * GEMM that consumes the returned pointer.  Release holds the same mutex
+ * across device synchronization and frees, closing the lookup/free race. */
+static const __half *rocm_q4_attn_q_b_f16_acquire(
+        const void *model_map,
+        uint64_t model_size,
+        uint64_t weight_offset,
+        uint64_t weight_bytes,
+        uint64_t in_dim,
+        uint64_t out_dim,
+        uint32_t weight_type) {
+    pthread_mutex_lock(&g_rocm_q4_attn_q_b_f16_cache_mu);
+    g_rocm_q4_attn_q_b_f16_lookups++;
+    /* Recheck the circuit while holding the same mutex that protects the
+     * entries.  A concurrent runtime failure may open it after the caller's
+     * cheap policy check but before this lookup; refusing here also keeps a
+     * partially freed arena unreachable while pending eviction is retried. */
+    if (g_rocm_q4_attn_q_b_f16_hard_failure) {
+        g_rocm_q4_attn_q_b_f16_misses++;
+        pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_cache_mu);
+        return NULL;
+    }
+    rocm_q4_attn_q_b_f16_cache_entry *entry =
+        rocm_q4_attn_q_b_f16_find_locked(
+            model_map, model_size, weight_offset, weight_bytes,
+            in_dim, out_dim, weight_type);
+    if (!entry) {
+        g_rocm_q4_attn_q_b_f16_misses++;
+        pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_cache_mu);
+        return NULL;
+    }
+    g_rocm_q4_attn_q_b_f16_hits++;
+    return entry->device_ptr;
+}
+
+static void rocm_q4_attn_q_b_f16_release_acquired(void) {
+    pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_cache_mu);
+}
+
+static void rocm_q4_attn_q_b_f16_note_candidate(void) {
+    pthread_mutex_lock(&g_rocm_q4_attn_q_b_f16_cache_mu);
+    g_rocm_q4_attn_q_b_f16_candidate_calls++;
+    pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_cache_mu);
+}
+
+static int rocm_q4_attn_q_b_f16_try_runtime_evict(void);
+
+static int rocm_q4_attn_q_b_f16_fallback(
+        int required, int rejected, int build_failure) {
+    pthread_mutex_lock(&g_rocm_q4_attn_q_b_f16_cache_mu);
+    g_rocm_q4_attn_q_b_f16_fallbacks++;
+    if (rejected) g_rocm_q4_attn_q_b_f16_rejects++;
+    if (build_failure) {
+        g_rocm_q4_attn_q_b_f16_build_failures++;
+        /* A submission/launch failure is backend-wide for this optional
+         * specialization. Fail closed for later layers instead of retrying
+         * the same hipBLAS or kernel error dozens of times per prefill. */
+        g_rocm_q4_attn_q_b_f16_hard_failure = 1;
+        g_rocm_q4_attn_q_b_f16_pending_evict = 1;
+    }
+    pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_cache_mu);
+    /* All runtime callers reach this helper after releasing the cache mutex
+     * used to pin the sidecar through GEMM submission, and none holds the
+     * build mutex. Try the synchronized eviction now; if HIP cannot reach a
+     * safe point, pending_evict keeps the cache disabled until a later
+     * lifecycle/prewarm boundary can retry. */
+    if (build_failure) (void)rocm_q4_attn_q_b_f16_try_runtime_evict();
+    return required ? -1 : 0;
+}
+
+static int rocm_q4_attn_q_b_f16_circuit_open(void) {
+    pthread_mutex_lock(&g_rocm_q4_attn_q_b_f16_cache_mu);
+    const int open = g_rocm_q4_attn_q_b_f16_hard_failure;
+    pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_cache_mu);
+    return open;
+}
+
+__device__ __forceinline__ static void
+rocm_q4_attn_q_b_get_scale_min(
+        uint32_t group,
+        const uint8_t *scales,
+        uint8_t *scale,
+        uint8_t *minimum) {
+    if (group < 4u) {
+        *scale = scales[group] & 63u;
+        *minimum = scales[group + 4u] & 63u;
+    } else {
+        *scale = (scales[group + 4u] & 0x0fu) |
+                 ((scales[group - 4u] >> 6u) << 4u);
+        *minimum = (scales[group + 4u] >> 4u) |
+                   ((scales[group] >> 6u) << 4u);
+    }
+}
+
+/* Expand one contiguous 16-value chunk per thread.  Compared with launching a
+ * 256-thread workgroup for every Q4_K block, this cuts the logical thread count
+ * by 16x while preserving the row-major [out_dim, in_dim] layout consumed as
+ * W^T by hipBLAS. */
+__global__ static void rocm_dequant_q4_K_attn_q_b_f16_kernel(
+        __half *dst,
+        const cuda_block_q4_K *src,
+        uint64_t in_dim,
+        uint64_t out_dim,
+        uint64_t blocks_per_row) {
+    const uint64_t chunk =
+        (uint64_t)blockIdx.x * blockDim.x + threadIdx.x;
+    const uint64_t chunks_per_row = in_dim / 16u;
+    const uint64_t total_chunks = out_dim * chunks_per_row;
+    if (chunk >= total_chunks) return;
+
+    const uint64_t row = chunk / chunks_per_row;
+    const uint64_t col0 = (chunk - row * chunks_per_row) * 16u;
+    const uint64_t block_in_row = col0 / CUDA_QK_K;
+    const uint32_t within0 = (uint32_t)(col0 % CUDA_QK_K);
+    const cuda_block_q4_K *xb =
+        src + row * blocks_per_row + block_in_row;
+    const float d = __half2float(
+        __ushort_as_half((unsigned short)xb->d));
+    const float dmin = __half2float(
+        __ushort_as_half((unsigned short)xb->dmin));
+
+#pragma unroll
+    for (uint32_t k = 0; k < 16u; k++) {
+        const uint32_t within = within0 + k;
+        const uint32_t group = within >> 5u;
+        uint8_t scale = 0;
+        uint8_t minimum = 0;
+        rocm_q4_attn_q_b_get_scale_min(
+            group, xb->scales, &scale, &minimum);
+        const uint8_t packed =
+            xb->qs[(group >> 1u) * 32u + (within & 31u)];
+        const uint32_t q =
+            (group & 1u) ? (packed >> 4u) : (packed & 0x0fu);
+        dst[row * in_dim + col0 + k] =
+            __float2half(d * (float)scale * (float)q -
+                         dmin * (float)minimum);
+    }
+}
+
+static int rocm_q4_attn_q_b_f16_desc_valid(
+        const ds4_gpu_q4_attn_q_b_f16_sidecar_desc *desc,
+        uint64_t model_size,
+        uint64_t *f16_bytes) {
+    if (!desc || desc->weight_type != DS4_ROCM_Q4_K_TYPE ||
+        desc->in_dim != DS4_ROCM_Q4_ATTN_Q_B_IN_DIM ||
+        desc->out_dim != DS4_ROCM_Q4_ATTN_Q_B_OUT_DIM) {
+        return 0;
+    }
+    uint64_t row_bytes = 0;
+    uint64_t expected_weight_bytes = 0;
+    uint64_t elems = 0;
+    uint64_t expanded_bytes = 0;
+    if (!cuda_u64_mul_checked(desc->in_dim / CUDA_QK_K,
+                              sizeof(cuda_block_q4_K), &row_bytes) ||
+        !cuda_u64_mul_checked(desc->out_dim, row_bytes,
+                              &expected_weight_bytes) ||
+        !cuda_u64_mul_checked(desc->in_dim, desc->out_dim, &elems) ||
+        !cuda_u64_mul_checked(elems, sizeof(__half), &expanded_bytes)) {
+        return 0;
+    }
+    if (desc->weight_bytes != expected_weight_bytes ||
+        !cuda_model_range_fits(model_size, desc->weight_offset,
+                               desc->weight_bytes)) {
+        return 0;
+    }
+    if (f16_bytes) *f16_bytes = expanded_bytes;
+    return 1;
+}
+
+static int rocm_q4_attn_q_b_f16_memory_has_room(
+        uint64_t sidecar_bytes,
+        uint64_t working_set_reserve_bytes,
+        uint64_t *free_bytes_out,
+        uint64_t *total_bytes_out,
+        uint64_t *reserve_bytes_out) {
+    size_t free_b = 0;
+    size_t total_b = 0;
+    const cudaError_t err = cudaMemGetInfo(&free_b, &total_b);
+    if (err != cudaSuccess || total_b == 0u) {
+        (void)cudaGetLastError();
+        return 0;
+    }
+    const uint64_t free_bytes = (uint64_t)free_b;
+    const uint64_t total_bytes = (uint64_t)total_b;
+    const uint64_t reserve_bytes =
+        cuda_q8_f16_cache_reserve_bytes(total_bytes);
+    uint64_t required_free = 0;
+    if (!cuda_u64_add_checked(sidecar_bytes, reserve_bytes,
+                              &required_free) ||
+        !cuda_u64_add_checked(required_free, working_set_reserve_bytes,
+                              &required_free)) {
+        return 0;
+    }
+    if (free_bytes_out) *free_bytes_out = free_bytes;
+    if (total_bytes_out) *total_bytes_out = total_bytes;
+    if (reserve_bytes_out) *reserve_bytes_out = reserve_bytes;
+    return required_free <= free_bytes;
+}
+
+static int rocm_q4_attn_q_b_transient_f16_layout(
+        uint64_t rows,
+        int include_output,
+        uint64_t *weight_bytes_out,
+        uint64_t *x_bytes_out,
+        uint64_t *q_bytes_out,
+        uint64_t *total_bytes_out) {
+    uint64_t weight_elems = 0;
+    uint64_t weight_bytes = 0;
+    uint64_t x_elems = 0;
+    uint64_t x_bytes = 0;
+    uint64_t q_elems = 0;
+    uint64_t q_bytes = 0;
+    uint64_t total_bytes = 0;
+    if (rows == 0u ||
+        !cuda_u64_mul_checked(DS4_ROCM_Q4_ATTN_Q_B_IN_DIM,
+                              DS4_ROCM_Q4_ATTN_Q_B_OUT_DIM,
+                              &weight_elems) ||
+        !cuda_u64_mul_checked(weight_elems, sizeof(__half),
+                              &weight_bytes) ||
+        !cuda_u64_mul_checked(rows, DS4_ROCM_Q4_ATTN_Q_B_IN_DIM,
+                              &x_elems) ||
+        !cuda_u64_mul_checked(x_elems, sizeof(__half), &x_bytes) ||
+        !cuda_u64_add_checked(weight_bytes, x_bytes, &total_bytes) ||
+        total_bytes > (uint64_t)SIZE_MAX) {
+        return 0;
+    }
+    if (include_output &&
+        (!cuda_u64_mul_checked(rows, DS4_ROCM_Q4_ATTN_Q_B_OUT_DIM,
+                               &q_elems) ||
+         !cuda_u64_mul_checked(q_elems, sizeof(__half), &q_bytes) ||
+         !cuda_u64_add_checked(total_bytes, q_bytes, &total_bytes) ||
+         total_bytes > (uint64_t)SIZE_MAX)) {
+        return 0;
+    }
+    if (weight_bytes_out) *weight_bytes_out = weight_bytes;
+    if (x_bytes_out) *x_bytes_out = x_bytes;
+    if (q_bytes_out) *q_bytes_out = q_bytes;
+    if (total_bytes_out) *total_bytes_out = total_bytes;
+    return 1;
+}
+
+/* Caller holds g_rocm_q4_attn_q_b_transient_f16_mu.  Growth is a preflight
+ * operation. Allocate the replacement first, then synchronize and retire the
+ * old arena: an allocation failure must not silently destroy the capacity
+ * already advertised by the current cache generation. */
+static int rocm_q4_attn_q_b_transient_f16_ensure_locked(
+        uint64_t required_bytes,
+        uint64_t weight_bytes,
+        uint64_t working_set_reserve_bytes,
+        int *allocated_out) {
+    if (allocated_out) *allocated_out = 0;
+    if (required_bytes == 0u || weight_bytes == 0u ||
+        required_bytes > (uint64_t)SIZE_MAX) {
+        return 0;
+    }
+    if (g_rocm_q4_attn_q_b_transient_f16_scratch &&
+        g_rocm_q4_attn_q_b_transient_f16_weight_bytes == weight_bytes &&
+        g_rocm_q4_attn_q_b_transient_f16_scratch_bytes >= required_bytes) {
+        return 1;
+    }
+
+    if (!rocm_q4_attn_q_b_f16_memory_has_room(
+            required_bytes, working_set_reserve_bytes,
+            NULL, NULL, NULL)) {
+        return 0;
+    }
+    void *scratch = NULL;
+    cudaError_t err = cudaMalloc(&scratch, (size_t)required_bytes);
+    if (err != cudaSuccess || !scratch) {
+        fprintf(stderr,
+                DS4_GPU_LOG_PREFIX
+                "Q4 attn_q_b transient scratch allocation failed "
+                "(%.2f MiB): %s\n",
+                (double)required_bytes / 1048576.0,
+                cudaGetErrorString(err));
+        (void)cudaGetLastError();
+        return 0;
+    }
+
+    if (g_rocm_q4_attn_q_b_transient_f16_scratch) {
+        err = cudaDeviceSynchronize();
+        if (err != cudaSuccess) {
+            fprintf(stderr,
+                    DS4_GPU_LOG_PREFIX
+                    "Q4 attn_q_b transient scratch growth sync failed: %s\n",
+                    cudaGetErrorString(err));
+            (void)cudaGetLastError();
+            (void)cudaFree(scratch);
+            return 0;
+        }
+        err = cudaFree(g_rocm_q4_attn_q_b_transient_f16_scratch);
+        if (err != cudaSuccess) {
+            fprintf(stderr,
+                    DS4_GPU_LOG_PREFIX
+                    "Q4 attn_q_b transient scratch free failed: %s\n",
+                    cudaGetErrorString(err));
+            (void)cudaGetLastError();
+            (void)cudaFree(scratch);
+            return 0;
+        }
+    }
+    g_rocm_q4_attn_q_b_transient_f16_scratch = scratch;
+    g_rocm_q4_attn_q_b_transient_f16_scratch_bytes = required_bytes;
+    g_rocm_q4_attn_q_b_transient_f16_weight_bytes = weight_bytes;
+    if (allocated_out) *allocated_out = 1;
+    return 1;
+}
+
+/* Success leaves the transient mutex held through the caller's complete GPU
+ * enqueue sequence.  No allocation or synchronization is permitted here. */
+static int rocm_q4_attn_q_b_transient_f16_acquire(
+        uint64_t rows,
+        int include_output,
+        __half **weight_f16_out,
+        __half **x_f16_out,
+        __half **q_f16_out) {
+    uint64_t weight_bytes = 0;
+    uint64_t x_bytes = 0;
+    uint64_t total_bytes = 0;
+    if (!weight_f16_out || !x_f16_out || !q_f16_out ||
+        !rocm_q4_attn_q_b_transient_f16_layout(
+            rows, include_output, &weight_bytes, &x_bytes, NULL,
+            &total_bytes)) {
+        return 0;
+    }
+    pthread_mutex_lock(&g_rocm_q4_attn_q_b_transient_f16_mu);
+    if (!g_rocm_q4_attn_q_b_transient_f16_scratch ||
+        g_rocm_q4_attn_q_b_transient_f16_weight_bytes != weight_bytes ||
+        g_rocm_q4_attn_q_b_transient_f16_scratch_bytes < total_bytes) {
+        pthread_mutex_unlock(&g_rocm_q4_attn_q_b_transient_f16_mu);
+        return 0;
+    }
+    *weight_f16_out =
+        (__half *)g_rocm_q4_attn_q_b_transient_f16_scratch;
+    *x_f16_out = (__half *)(
+        (char *)g_rocm_q4_attn_q_b_transient_f16_scratch + weight_bytes);
+    *q_f16_out = (__half *)(
+        (char *)g_rocm_q4_attn_q_b_transient_f16_scratch +
+        weight_bytes + x_bytes);
+    return 1;
+}
+
+static void rocm_q4_attn_q_b_transient_f16_release_acquired(void) {
+    pthread_mutex_unlock(&g_rocm_q4_attn_q_b_transient_f16_mu);
+}
+
+static void rocm_q4_attn_q_b_f16_clear_locked(
+        int reset_stats, int reset_circuit) {
+    for (uint32_t i = 0;
+         i < DS4_ROCM_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES;
+         i++) {
+        g_rocm_q4_attn_q_b_f16_entries[i] = {};
+        g_rocm_q4_attn_q_b_f16_arenas[i] = {};
+    }
+    g_rocm_q4_attn_q_b_f16_entry_count = 0;
+    g_rocm_q4_attn_q_b_f16_arena_count = 0;
+    g_rocm_q4_attn_q_b_f16_bytes = 0;
+    if (reset_circuit) g_rocm_q4_attn_q_b_f16_hard_failure = 0;
+    g_rocm_q4_attn_q_b_f16_pending_evict = 0;
+    g_rocm_q4_attn_q_b_f16_generation++;
+    if (g_rocm_q4_attn_q_b_f16_generation == 0u) {
+        g_rocm_q4_attn_q_b_f16_generation = 1u;
+    }
+    if (reset_stats) {
+        g_rocm_q4_attn_q_b_f16_lookups = 0;
+        g_rocm_q4_attn_q_b_f16_hits = 0;
+        g_rocm_q4_attn_q_b_f16_misses = 0;
+        g_rocm_q4_attn_q_b_f16_builds = 0;
+        g_rocm_q4_attn_q_b_f16_build_failures = 0;
+        g_rocm_q4_attn_q_b_f16_candidate_calls = 0;
+        g_rocm_q4_attn_q_b_f16_fallbacks = 0;
+        g_rocm_q4_attn_q_b_f16_rejects = 0;
+    }
+}
+
+/* The caller owns build_mu. Keep both dispatch mutexes through synchronization
+ * and free: a persistent lookup cannot lose its arena, and a transient enqueue
+ * cannot lose the combined scratch while its final consumer is being queued. */
+static int rocm_q4_attn_q_b_f16_release_with_build_lock(
+        int reset_circuit) {
+    pthread_mutex_lock(&g_rocm_q4_attn_q_b_transient_f16_mu);
+    pthread_mutex_lock(&g_rocm_q4_attn_q_b_f16_cache_mu);
+
+    if (g_rocm_q4_attn_q_b_f16_arena_count != 0u ||
+        g_rocm_q4_attn_q_b_transient_f16_scratch) {
+        const cudaError_t sync_err = cudaDeviceSynchronize();
+        if (sync_err != cudaSuccess) {
+            fprintf(stderr,
+                    DS4_GPU_LOG_PREFIX
+                    "Q4 attn_q_b F16 cache release sync failed: %s\n",
+                    cudaGetErrorString(sync_err));
+            (void)cudaGetLastError();
+            g_rocm_q4_attn_q_b_f16_hard_failure = 1;
+            g_rocm_q4_attn_q_b_f16_pending_evict = 1;
+            pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_cache_mu);
+            pthread_mutex_unlock(&g_rocm_q4_attn_q_b_transient_f16_mu);
+            return 0;
+        }
+    }
+    int ok = 1;
+    for (uint32_t i = 0; i < g_rocm_q4_attn_q_b_f16_arena_count; i++) {
+        if (g_rocm_q4_attn_q_b_f16_arenas[i].device_ptr) {
+            const cudaError_t free_err =
+                cudaFree(g_rocm_q4_attn_q_b_f16_arenas[i].device_ptr);
+            if (free_err != cudaSuccess) {
+                fprintf(stderr,
+                        DS4_GPU_LOG_PREFIX
+                        "Q4 attn_q_b F16 cache free failed: %s\n",
+                        cudaGetErrorString(free_err));
+                (void)cudaGetLastError();
+                ok = 0;
+            } else {
+                /* A later retry must not double-free arenas already released
+                 * before another arena reported an error. */
+                g_rocm_q4_attn_q_b_f16_arenas[i].device_ptr = NULL;
+            }
+        }
+    }
+    if (g_rocm_q4_attn_q_b_transient_f16_scratch) {
+        const cudaError_t free_err =
+            cudaFree(g_rocm_q4_attn_q_b_transient_f16_scratch);
+        if (free_err != cudaSuccess) {
+            fprintf(stderr,
+                    DS4_GPU_LOG_PREFIX
+                    "Q4 attn_q_b transient scratch free failed: %s\n",
+                    cudaGetErrorString(free_err));
+            (void)cudaGetLastError();
+            ok = 0;
+        } else {
+            g_rocm_q4_attn_q_b_transient_f16_scratch = NULL;
+            g_rocm_q4_attn_q_b_transient_f16_scratch_bytes = 0;
+            g_rocm_q4_attn_q_b_transient_f16_weight_bytes = 0;
+        }
+    }
+    if (!ok) {
+        g_rocm_q4_attn_q_b_f16_hard_failure = 1;
+        g_rocm_q4_attn_q_b_f16_pending_evict = 1;
+        pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_cache_mu);
+        pthread_mutex_unlock(&g_rocm_q4_attn_q_b_transient_f16_mu);
+        return 0;
+    }
+    rocm_q4_attn_q_b_f16_clear_locked(0, reset_circuit);
+    pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_cache_mu);
+    pthread_mutex_unlock(&g_rocm_q4_attn_q_b_transient_f16_mu);
+    return 1;
+}
+
+static int rocm_q4_attn_q_b_f16_try_runtime_evict(void) {
+    pthread_mutex_lock(&g_rocm_q4_attn_q_b_f16_build_mu);
+    const int ok = rocm_q4_attn_q_b_f16_release_with_build_lock(0);
+    pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_build_mu);
+    return ok;
+}
+
+extern "C" int ds4_gpu_release_q4_attn_q_b_f16_sidecars(void) {
+    pthread_mutex_lock(&g_rocm_q4_attn_q_b_f16_build_mu);
+    const int ok = rocm_q4_attn_q_b_f16_release_with_build_lock(1);
+    pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_build_mu);
+    return ok;
+}
+
+extern "C" uint64_t ds4_gpu_q4_attn_q_b_f16_cache_generation(void) {
+    pthread_mutex_lock(&g_rocm_q4_attn_q_b_f16_cache_mu);
+    const uint64_t generation = g_rocm_q4_attn_q_b_f16_generation;
+    pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_cache_mu);
+    return generation;
+}
+
+extern "C" int ds4_gpu_make_room_for_q4_attn_q_b_f16_session(void) {
+    pthread_mutex_lock(&g_rocm_q4_attn_q_b_f16_build_mu);
+    pthread_mutex_lock(&g_rocm_q4_attn_q_b_transient_f16_mu);
+    pthread_mutex_lock(&g_rocm_q4_attn_q_b_f16_cache_mu);
+    const uint32_t entries = g_rocm_q4_attn_q_b_f16_entry_count;
+    const uint64_t bytes = g_rocm_q4_attn_q_b_f16_bytes;
+    const int needs_reset =
+        entries != 0u || g_rocm_q4_attn_q_b_f16_arena_count != 0u ||
+        g_rocm_q4_attn_q_b_transient_f16_scratch != NULL ||
+        g_rocm_q4_attn_q_b_f16_hard_failure ||
+        g_rocm_q4_attn_q_b_f16_pending_evict;
+    pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_cache_mu);
+    pthread_mutex_unlock(&g_rocm_q4_attn_q_b_transient_f16_mu);
+    if (!needs_reset) {
+        pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_build_mu);
+        return 1;
+    }
+    if (entries != 0u) {
+        fprintf(stderr,
+                DS4_GPU_LOG_PREFIX
+                "evicting %.2f GiB of resident Q4 attn_q_b F16 sidecars "
+                "before allocating another live session\n",
+                (double)bytes / 1073741824.0);
+    }
+    const int ok = rocm_q4_attn_q_b_f16_release_with_build_lock(1);
+    pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_build_mu);
+    return ok;
+}
+
+static int rocm_q4_attn_q_b_prepare_transient_f16(
+        const void *model_map,
+        uint64_t model_size,
+        const ds4_gpu_q4_attn_q_b_f16_sidecar_desc *descs,
+        uint32_t count,
+        uint32_t max_prefill_rows,
+        uint64_t working_set_reserve_bytes,
+        uint64_t *prepared_bytes) {
+    const uint64_t min_tokens =
+        rocm_q4_attn_q_b_transient_f16_min_tokens();
+    if ((uint64_t)max_prefill_rows < min_tokens ||
+        !rocm_q4_attn_q_b_transient_f16_policy_allowed() ||
+        rocm_q4_attn_q_b_f16_circuit_open() ||
+        !g_cublas_ready ||
+        model_map != g_model_host_base ||
+        model_size != g_model_registered_size) {
+        return 0;
+    }
+
+    /* Serialize prompt-aware preparation with cache construction, lifecycle
+     * release, and model-range teardown.  The automatic path is deliberately
+     * stricter than the explicit persistent experiment: every q_b source must
+     * already belong to a device image or device-backed resident range, so
+     * preflight never registers host pages or populates the mutable cache. */
+    pthread_mutex_lock(&g_rocm_q4_attn_q_b_f16_build_mu);
+    if (!rocm_q4_attn_q_b_transient_f16_policy_allowed() ||
+        rocm_q4_attn_q_b_f16_circuit_open() ||
+        !g_cublas_ready ||
+        model_map != g_model_host_base ||
+        model_size != g_model_registered_size) {
+        pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_build_mu);
+        return 0;
+    }
+
+    uint64_t weight_f16_bytes = 0;
+    for (uint32_t i = 0; i < count; i++) {
+        uint64_t desc_f16_bytes = 0;
+        if (!rocm_q4_attn_q_b_f16_desc_valid(
+                &descs[i], model_size, &desc_f16_bytes) ||
+            (weight_f16_bytes != 0u &&
+             weight_f16_bytes != desc_f16_bytes)) {
+            pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_build_mu);
+            return 0;
+        }
+        weight_f16_bytes = desc_f16_bytes;
+
+        if (!rocm_q4_attn_q_b_device_resident_source(
+                model_map, descs[i].weight_offset,
+                descs[i].weight_bytes)) {
+            pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_build_mu);
+            return 0;
+        }
+    }
+
+    uint64_t layout_weight_bytes = 0;
+    uint64_t total_bytes = 0;
+    const int include_output =
+        rocm_q4_attn_q_b_f16_output_enabled();
+    if (!rocm_q4_attn_q_b_transient_f16_layout(
+            max_prefill_rows, include_output,
+            &layout_weight_bytes, NULL, NULL,
+            &total_bytes) ||
+        layout_weight_bytes != weight_f16_bytes) {
+        pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_build_mu);
+        return 0;
+    }
+
+    int allocated = 0;
+    pthread_mutex_lock(&g_rocm_q4_attn_q_b_transient_f16_mu);
+    const int ready = rocm_q4_attn_q_b_transient_f16_ensure_locked(
+        total_bytes, layout_weight_bytes, working_set_reserve_bytes,
+        &allocated);
+    if (ready && allocated) {
+        pthread_mutex_lock(&g_rocm_q4_attn_q_b_f16_cache_mu);
+        if (++g_rocm_q4_attn_q_b_f16_generation == 0u) {
+            g_rocm_q4_attn_q_b_f16_generation = 1u;
+        }
+        pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_cache_mu);
+    }
+    pthread_mutex_unlock(&g_rocm_q4_attn_q_b_transient_f16_mu);
+    pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_build_mu);
+    if (!ready) return 0;
+
+    if (allocated) {
+        if (prepared_bytes) *prepared_bytes = total_bytes;
+        fprintf(stderr,
+                DS4_GPU_LOG_PREFIX
+                "prepared %.2f MiB Q4 attn_q_b transient F16 scratch "
+                "for up to %u rows (min batch %llu tokens)\n",
+                (double)total_bytes / 1048576.0,
+                max_prefill_rows,
+                (unsigned long long)min_tokens);
+    }
+    return 1;
+}
+
+extern "C" int ds4_gpu_prepare_q4_attn_q_b_f16_sidecars(
+        const void *model_map,
+        uint64_t model_size,
+        const ds4_gpu_q4_attn_q_b_f16_sidecar_desc *descs,
+        uint32_t count,
+        uint32_t max_prefill_rows,
+        uint64_t working_set_reserve_bytes,
+        uint64_t *prepared_bytes) {
+    if (prepared_bytes) *prepared_bytes = 0;
+    if (!model_map || !descs || count == 0u ||
+        count > DS4_ROCM_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES ||
+        max_prefill_rows < 32u) {
+        return 0;
+    }
+
+    const int required = rocm_q4_attn_q_b_f16_required();
+    const int persistent_requested =
+        required ||
+        (rocm_q4_attn_q_b_f16_enabled() &&
+         !rocm_q4_attn_q_b_f16_disabled());
+    /* ENABLE and REQUIRE deliberately select the multi-GiB persistent cache.
+     * DISABLE cancels a non-strict ENABLE back to transient, while REQUIRE
+     * still enters the persistent policy and reports DISABLE as a hard skip. */
+    if (!persistent_requested) {
+        return rocm_q4_attn_q_b_prepare_transient_f16(
+            model_map, model_size, descs, count, max_prefill_rows,
+            working_set_reserve_bytes, prepared_bytes);
+    }
+    const uint64_t min_tokens = rocm_q4_attn_q_b_f16_min_tokens();
+    if ((uint64_t)max_prefill_rows < min_tokens) return 0;
+    if (!rocm_q4_attn_q_b_f16_policy_allowed() || !g_cublas_ready ||
+        model_map != g_model_host_base ||
+        model_size != g_model_registered_size) {
+        return required ? -1 : 0;
+    }
+
+    uint64_t desc_f16_bytes[
+        DS4_ROCM_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES] = {0};
+    for (uint32_t i = 0; i < count; i++) {
+        if (!rocm_q4_attn_q_b_f16_desc_valid(
+                &descs[i], model_size, &desc_f16_bytes[i])) {
+            return required ? -1 : 0;
+        }
+    }
+
+    pthread_mutex_lock(&g_rocm_q4_attn_q_b_f16_build_mu);
+
+    pthread_mutex_lock(&g_rocm_q4_attn_q_b_f16_cache_mu);
+    const int pending_evict = g_rocm_q4_attn_q_b_f16_pending_evict;
+    pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_cache_mu);
+    /* Explicit prewarm is a quiescent session boundary. Retry an eviction
+     * that could not synchronize at the original runtime error, while
+     * preserving the hard circuit so this session cannot rebuild and repeat
+     * the failed specialization. */
+    if (pending_evict &&
+        !rocm_q4_attn_q_b_f16_release_with_build_lock(0)) {
+        pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_build_mu);
+        return required ? -1 : 0;
+    }
+    if (!rocm_q4_attn_q_b_f16_policy_allowed() || !g_cublas_ready ||
+        model_map != g_model_host_base ||
+        model_size != g_model_registered_size) {
+        pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_build_mu);
+        return required ? -1 : 0;
+    }
+    if (rocm_q4_attn_q_b_f16_circuit_open()) {
+        pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_build_mu);
+        return required ? -1 : 0;
+    }
+
+    /* The persistent sidecar still needs private X_F16 staging and, for the
+     * explicit output experiment, Q_F16. Reuse the dedicated combined arena
+     * instead of the backend-global cuda_tmp buffer; unlike Metal, the ROCm
+     * graph does not normally own a batch_q_half tensor. Dispatch holds
+     * transient_mu through conversion, GEMM, and epilogue. Keep the global
+     * lock order build -> transient -> cache. */
+    uint64_t scratch_weight_bytes = 0;
+    uint64_t scratch_bytes = 0;
+    const int include_output =
+        rocm_q4_attn_q_b_f16_output_enabled();
+    if (!rocm_q4_attn_q_b_transient_f16_layout(
+            max_prefill_rows, include_output,
+            &scratch_weight_bytes, NULL, NULL,
+            &scratch_bytes) ||
+        scratch_weight_bytes != desc_f16_bytes[0]) {
+        pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_build_mu);
+        return required ? -1 : 0;
+    }
+    int scratch_allocated = 0;
+    pthread_mutex_lock(&g_rocm_q4_attn_q_b_transient_f16_mu);
+    const int scratch_ready =
+        rocm_q4_attn_q_b_transient_f16_ensure_locked(
+            scratch_bytes, scratch_weight_bytes,
+            working_set_reserve_bytes, &scratch_allocated);
+    if (scratch_ready && scratch_allocated) {
+        pthread_mutex_lock(&g_rocm_q4_attn_q_b_f16_cache_mu);
+        if (++g_rocm_q4_attn_q_b_f16_generation == 0u) {
+            g_rocm_q4_attn_q_b_f16_generation = 1u;
+        }
+        pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_cache_mu);
+    }
+    pthread_mutex_unlock(&g_rocm_q4_attn_q_b_transient_f16_mu);
+    if (!scratch_ready) {
+        pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_build_mu);
+        return required ? -1 : 0;
+    }
+
+    uint32_t miss_indices[
+        DS4_ROCM_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES] = {0};
+    uint32_t free_slots[
+        DS4_ROCM_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES] = {0};
+    uint32_t miss_count = 0;
+    uint32_t free_count = 0;
+    uint64_t missing_bytes = 0;
+    uint64_t cached_bytes = 0;
+    int hard_failure = 0;
+    pthread_mutex_lock(&g_rocm_q4_attn_q_b_f16_cache_mu);
+    cached_bytes = g_rocm_q4_attn_q_b_f16_bytes;
+    hard_failure = g_rocm_q4_attn_q_b_f16_hard_failure;
+    for (uint32_t i = 0;
+         i < DS4_ROCM_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES;
+         i++) {
+        if (!g_rocm_q4_attn_q_b_f16_entries[i].valid) {
+            free_slots[free_count++] = i;
+        }
+    }
+    for (uint32_t i = 0; i < count; i++) {
+        if (rocm_q4_attn_q_b_f16_find_locked(
+                model_map, model_size, descs[i].weight_offset,
+                descs[i].weight_bytes, descs[i].in_dim,
+                descs[i].out_dim, descs[i].weight_type)) {
+            continue;
+        }
+        int duplicate_miss = 0;
+        for (uint32_t mi = 0; mi < miss_count; mi++) {
+            const ds4_gpu_q4_attn_q_b_f16_sidecar_desc *prior =
+                &descs[miss_indices[mi]];
+            if (prior->weight_offset == descs[i].weight_offset &&
+                prior->weight_bytes == descs[i].weight_bytes &&
+                prior->in_dim == descs[i].in_dim &&
+                prior->out_dim == descs[i].out_dim &&
+                prior->weight_type == descs[i].weight_type) {
+                duplicate_miss = 1;
+                break;
+            }
+        }
+        if (duplicate_miss) continue;
+        if (!cuda_u64_add_checked(missing_bytes, desc_f16_bytes[i],
+                                  &missing_bytes)) {
+            pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_cache_mu);
+            pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_build_mu);
+            return required ? -1 : 0;
+        }
+        miss_indices[miss_count++] = i;
+    }
+    const int entry_room =
+        miss_count <= free_count &&
+        g_rocm_q4_attn_q_b_f16_arena_count <
+            DS4_ROCM_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES;
+    pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_cache_mu);
+
+    if (hard_failure) {
+        pthread_mutex_lock(&g_rocm_q4_attn_q_b_f16_cache_mu);
+        g_rocm_q4_attn_q_b_f16_rejects++;
+        pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_cache_mu);
+        pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_build_mu);
+        return required ? -1 : 0;
+    }
+    if (miss_count == 0u) {
+        pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_build_mu);
+        return 1;
+    }
+    const uint64_t budget_bytes = rocm_q4_attn_q_b_f16_budget_bytes();
+    if (!entry_room ||
+        missing_bytes > (uint64_t)SIZE_MAX ||
+        missing_bytes > budget_bytes ||
+        cached_bytes > budget_bytes - missing_bytes) {
+        pthread_mutex_lock(&g_rocm_q4_attn_q_b_f16_cache_mu);
+        g_rocm_q4_attn_q_b_f16_rejects++;
+        pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_cache_mu);
+        pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_build_mu);
+        return required ? -1 : 0;
+    }
+
+    const char *weight_ptrs[
+        DS4_ROCM_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES] = {NULL};
+    for (uint32_t mi = 0; mi < miss_count; mi++) {
+        const ds4_gpu_q4_attn_q_b_f16_sidecar_desc *desc =
+            &descs[miss_indices[mi]];
+        weight_ptrs[mi] = cuda_model_range_ptr(
+            model_map, desc->weight_offset, desc->weight_bytes,
+            "Q4 attn_q_b sidecar source");
+        if (!weight_ptrs[mi]) {
+            pthread_mutex_lock(&g_rocm_q4_attn_q_b_f16_cache_mu);
+            g_rocm_q4_attn_q_b_f16_build_failures++;
+            g_rocm_q4_attn_q_b_f16_hard_failure = 1;
+            pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_cache_mu);
+            pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_build_mu);
+            return required ? -1 : 0;
+        }
+    }
+
+    uint64_t free_bytes = 0;
+    uint64_t total_bytes = 0;
+    uint64_t reserve_bytes = 0;
+    if (!rocm_q4_attn_q_b_f16_memory_has_room(
+            missing_bytes, working_set_reserve_bytes,
+            &free_bytes, &total_bytes, &reserve_bytes)) {
+        fprintf(stderr,
+                DS4_GPU_LOG_PREFIX
+                "Q4 attn_q_b F16 prewarm skipped for device headroom: "
+                "sidecars %.2f GiB + future sessions %.2f GiB + reserve "
+                "%.2f GiB, free %.2f GiB of %.2f GiB\n",
+                (double)missing_bytes / 1073741824.0,
+                (double)working_set_reserve_bytes / 1073741824.0,
+                (double)reserve_bytes / 1073741824.0,
+                (double)free_bytes / 1073741824.0,
+                (double)total_bytes / 1073741824.0);
+        pthread_mutex_lock(&g_rocm_q4_attn_q_b_f16_cache_mu);
+        g_rocm_q4_attn_q_b_f16_rejects++;
+        pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_cache_mu);
+        pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_build_mu);
+        return required ? -1 : 0;
+    }
+
+    void *arena_raw = NULL;
+    cudaError_t err = cudaMalloc(&arena_raw, (size_t)missing_bytes);
+    if (err != cudaSuccess || !arena_raw) {
+        fprintf(stderr,
+                DS4_GPU_LOG_PREFIX
+                "Q4 attn_q_b F16 sidecar allocation failed (%.2f GiB): %s\n",
+                (double)missing_bytes / 1073741824.0,
+                cudaGetErrorString(err));
+        (void)cudaGetLastError();
+        pthread_mutex_lock(&g_rocm_q4_attn_q_b_f16_cache_mu);
+        g_rocm_q4_attn_q_b_f16_build_failures++;
+        g_rocm_q4_attn_q_b_f16_hard_failure = 1;
+        pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_cache_mu);
+        pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_build_mu);
+        return required ? -1 : 0;
+    }
+
+    __half *arena = (__half *)arena_raw;
+    __half *sidecar_ptrs[
+        DS4_ROCM_Q4_ATTN_Q_B_F16_CACHE_MAX_ENTRIES] = {NULL};
+    uint64_t arena_offset = 0;
+    (void)cudaGetLastError();
+    int launch_ok = 1;
+    for (uint32_t mi = 0; mi < miss_count; mi++) {
+        const ds4_gpu_q4_attn_q_b_f16_sidecar_desc *desc =
+            &descs[miss_indices[mi]];
+        sidecar_ptrs[mi] = (__half *)((char *)arena + arena_offset);
+        const uint64_t blocks_per_row = desc->in_dim / CUDA_QK_K;
+        const uint64_t total_chunks =
+            desc->out_dim * (desc->in_dim / 16u);
+        rocm_dequant_q4_K_attn_q_b_f16_kernel<<<
+            (uint32_t)((total_chunks + 255u) / 256u), 256>>>(
+                sidecar_ptrs[mi],
+                (const cuda_block_q4_K *)weight_ptrs[mi],
+                desc->in_dim, desc->out_dim, blocks_per_row);
+        err = cudaGetLastError();
+        if (err != cudaSuccess) {
+            fprintf(stderr,
+                    DS4_GPU_LOG_PREFIX
+                    "Q4 attn_q_b F16 dequant launch failed at layer %u: %s\n",
+                    desc->layer, cudaGetErrorString(err));
+            launch_ok = 0;
+            break;
+        }
+        arena_offset += desc_f16_bytes[miss_indices[mi]];
+    }
+    if (launch_ok) {
+        err = cudaDeviceSynchronize();
+        if (err != cudaSuccess) {
+            fprintf(stderr,
+                    DS4_GPU_LOG_PREFIX
+                    "Q4 attn_q_b F16 dequant synchronization failed: %s\n",
+                    cudaGetErrorString(err));
+            (void)cudaGetLastError();
+            launch_ok = 0;
+        }
+    } else {
+        (void)cudaDeviceSynchronize();
+    }
+    if (!launch_ok) {
+        (void)cudaFree(arena);
+        pthread_mutex_lock(&g_rocm_q4_attn_q_b_f16_cache_mu);
+        g_rocm_q4_attn_q_b_f16_build_failures++;
+        g_rocm_q4_attn_q_b_f16_hard_failure = 1;
+        pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_cache_mu);
+        pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_build_mu);
+        return required ? -1 : 0;
+    }
+
+    /* Publish only after every dequantization completed.  Until this point no
+     * lookup can observe any part of the new batch. */
+    pthread_mutex_lock(&g_rocm_q4_attn_q_b_f16_cache_mu);
+    const uint32_t arena_slot = g_rocm_q4_attn_q_b_f16_arena_count++;
+    g_rocm_q4_attn_q_b_f16_arenas[arena_slot] = {arena, missing_bytes};
+    uint32_t published = 0;
+    for (uint32_t mi = 0; mi < miss_count; mi++) {
+        const uint32_t slot = free_slots[mi];
+        const ds4_gpu_q4_attn_q_b_f16_sidecar_desc *desc =
+            &descs[miss_indices[mi]];
+        g_rocm_q4_attn_q_b_f16_entries[slot] = {
+            model_map,
+            model_size,
+            desc->weight_offset,
+            desc->weight_bytes,
+            desc->in_dim,
+            desc->out_dim,
+            desc->weight_type,
+            sidecar_ptrs[mi],
+            desc_f16_bytes[miss_indices[mi]],
+            1,
+        };
+        published++;
+    }
+    g_rocm_q4_attn_q_b_f16_entry_count += published;
+    g_rocm_q4_attn_q_b_f16_bytes += missing_bytes;
+    g_rocm_q4_attn_q_b_f16_builds += published;
+    pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_cache_mu);
+    pthread_mutex_unlock(&g_rocm_q4_attn_q_b_f16_build_mu);
+
+    if (prepared_bytes) *prepared_bytes = missing_bytes;
+    fprintf(stderr,
+            DS4_GPU_LOG_PREFIX
+            "prewarmed %u resident Q4 attn_q_b F16 sidecars "
+            "(%.2f GiB; cache budget %llu MiB; min batch %llu tokens)\n",
+            published,
+            (double)missing_bytes / 1073741824.0,
+            (unsigned long long)(budget_bytes / 1048576u),
+            (unsigned long long)min_tokens);
+    return 1;
+}
diff --git a/rocm/ds4_rocm_q8.cuh b/rocm/ds4_rocm_q8.cuh
index bc0ffb2a96..f287d9318a 100644
--- a/rocm/ds4_rocm_q8.cuh
+++ b/rocm/ds4_rocm_q8.cuh
@@ -35,49 +35,6 @@ __device__ __forceinline__ static int32_t dot_i8_block(const int8_t *a, const in
     return dot;
 }
 
-__global__ static DS4_ROCM_UNUSED void matmul_q8_0_kernel(
-        float *out,
-        const unsigned char *w,
-        const float *x,
-        uint64_t in_dim,
-        uint64_t out_dim,
-        uint64_t n_tok) {
-    uint64_t row = (uint64_t)blockIdx.x;
-    uint64_t tok = (uint64_t)blockIdx.y;
-    if (row >= out_dim || tok >= n_tok) return;
-    const uint64_t blocks = (in_dim + 31) / 32;
-    const unsigned char *wr = w + row * blocks * 34;
-    const float *xr = x + tok * in_dim;
-    float acc = 0.0f;
-
-    for (uint64_t b = threadIdx.x; b < blocks; b += blockDim.x) {
-        uint64_t i0 = b * 32;
-        uint64_t bn = in_dim - i0 < 32 ? in_dim - i0 : 32;
-        float amax = 0.0f;
-        for (uint64_t i = 0; i < bn; i++) amax = fmaxf(amax, fabsf(xr[i0 + i]));
-        float d = amax / 127.0f;
-        float id = d != 0.0f ? 1.0f / d : 0.0f;
-        const __half *scale_h = (const __half *)(wr + b * 34);
-        const int8_t *qs = (const int8_t *)(wr + b * 34 + 2);
-        int dot = 0;
-        for (uint64_t i = 0; i < bn; i++) {
-            int q = (int)lrintf(xr[i0 + i] * id);
-            q = q > 127 ? 127 : (q < -128 ? -128 : q);
-            dot += (int)qs[i] * q;
-        }
-        acc += __half2float(*scale_h) * d * (float)dot;
-    }
-
-    __shared__ float partial[256];
-    partial[threadIdx.x] = acc;
-    __syncthreads();
-    for (uint32_t stride = blockDim.x >> 1; stride > 0; stride >>= 1) {
-        if (threadIdx.x < stride) partial[threadIdx.x] += partial[threadIdx.x + stride];
-        __syncthreads();
-    }
-    if (threadIdx.x == 0) out[tok * out_dim + row] = partial[0];
-}
-
 __global__ static void quantize_q8_0_f32_kernel(
         int8_t *xq,
         float *xscale,
@@ -147,33 +104,6 @@ __global__ static void matmul_q8_0_preq_kernel(
     if (threadIdx.x == 0) out[tok * out_dim + row] = partial[0];
 }
 
-__global__ static void matmul_q8_0_preq_warp8_kernel(
-        float *out,
-        const unsigned char *w,
-        const int8_t *xq,
-        const float *xscale,
-        uint64_t in_dim,
-        uint64_t out_dim,
-        uint64_t blocks,
-        int use_dp4a) {
-    uint64_t row = (uint64_t)blockIdx.x * 8u + (threadIdx.x >> 5u);
-    uint32_t lane = threadIdx.x & 31u;
-    if (row >= out_dim) return;
-    const unsigned char *wr = w + row * blocks * 34;
-    float acc = 0.0f;
-    for (uint64_t b = lane; b < blocks; b += 32u) {
-        uint64_t i0 = b * 32;
-        uint64_t bn = in_dim - i0 < 32 ? in_dim - i0 : 32;
-        const __half *scale_h = (const __half *)(wr + b * 34);
-        const int8_t *qs = (const int8_t *)(wr + b * 34 + 2);
-        const int8_t *xqb = xq + b * 32;
-        int dot = dot_i8_block(qs, xqb, bn, use_dp4a);
-        acc += __half2float(*scale_h) * xscale[b] * (float)dot;
-    }
-    acc = warp_sum_f32(acc);
-    if (lane == 0) out[row] = acc;
-}
-
 __global__ static void matmul_q8_0_preq_rows_w32_kernel(
         float *out,
         const unsigned char *w,
@@ -247,108 +177,6 @@ __global__ static void matmul_q8_0_pair_preq_warp8_kernel(
     }
 }
 
-__global__ static void shared_gate_up_swiglu_q8_0_pair_preq_warp8_kernel(
-        float *gate,
-        float *up,
-        float *mid,
-        const unsigned char *wg,
-        const unsigned char *wu,
-        const int8_t *xq,
-        const float *xscale,
-        uint64_t in_dim,
-        uint64_t out_dim,
-        uint64_t blocks,
-        int use_dp4a,
-        int store_gate_up,
-        float clamp) {
-    const uint64_t row = (uint64_t)blockIdx.x * 8u + (threadIdx.x >> 5u);
-    const uint32_t lane = threadIdx.x & 31u;
-    if (row >= out_dim) return;
-    const unsigned char *gr = wg + row * blocks * 34u;
-    const unsigned char *ur = wu + row * blocks * 34u;
-    float g = 0.0f;
-    float u = 0.0f;
-    for (uint64_t b = lane; b < blocks; b += 32u) {
-        const uint64_t i0 = b * 32u;
-        const uint64_t bn = in_dim - i0 < 32u ? in_dim - i0 : 32u;
-        const int8_t *xqb = xq + b * 32u;
-        const float xs = xscale[b];
-        const __half *gscale_h = (const __half *)(gr + b * 34u);
-        const int8_t *gqs = (const int8_t *)(gr + b * 34u + 2u);
-        const __half *uscale_h = (const __half *)(ur + b * 34u);
-        const int8_t *uqs = (const int8_t *)(ur + b * 34u + 2u);
-        const int gdot = dot_i8_block(gqs, xqb, bn, use_dp4a);
-        const int udot = dot_i8_block(uqs, xqb, bn, use_dp4a);
-        g += __half2float(*gscale_h) * xs * (float)gdot;
-        u += __half2float(*uscale_h) * xs * (float)udot;
-    }
-    g = warp_sum_f32(g);
-    u = warp_sum_f32(u);
-    if (lane == 0u) {
-        if (store_gate_up) {
-            gate[row] = g;
-            up[row] = u;
-        }
-        float sg = g;
-        float su = u;
-        if (clamp > 1.0e-6f) {
-            sg = fminf(sg, clamp);
-            su = fminf(fmaxf(su, -clamp), clamp);
-        }
-        mid[row] = (sg / (1.0f + expf(-sg))) * su;
-    }
-}
-
-__global__ static void matmul_q8_0_hc_expand_preq_warp8_kernel(
-        float *out_hc,
-        float *block_out,
-        const float *block_add,
-        const float *residual_hc,
-        const float *split,
-        const unsigned char *w,
-        const int8_t *xq,
-        const float *xscale,
-        uint64_t in_dim,
-        uint64_t out_dim,
-        uint32_t n_embd,
-        uint32_t n_hc,
-        uint64_t blocks,
-        int has_add,
-        int use_dp4a) {
-    const uint64_t row = (uint64_t)blockIdx.x * 8u + (threadIdx.x >> 5u);
-    const uint32_t lane = threadIdx.x & 31u;
-    if (row >= out_dim) return;
-    const unsigned char *wr = w + row * blocks * 34;
-    float acc = 0.0f;
-    for (uint64_t b = lane; b < blocks; b += 32u) {
-        const uint64_t i0 = b * 32;
-        const uint64_t bn = in_dim - i0 < 32 ? in_dim - i0 : 32;
-        const __half *scale_h = (const __half *)(wr + b * 34);
-        const int8_t *qs = (const int8_t *)(wr + b * 34 + 2);
-        const int8_t *xqb = xq + b * 32;
-        int dot = dot_i8_block(qs, xqb, bn, use_dp4a);
-        acc += __half2float(*scale_h) * xscale[b] * (float)dot;
-    }
-    acc = warp_sum_f32(acc);
-    if (lane == 0) {
-        const uint32_t d = (uint32_t)row;
-        block_out[d] = acc;
-        float block_v = acc;
-        if (has_add) block_v += block_add[d];
-        const float *post = split + n_hc;
-        const float *comb = split + 2u * n_hc;
-        for (uint32_t dst_hc = 0; dst_hc < n_hc; dst_hc++) {
-            float hc_acc = block_v * post[dst_hc];
-            for (uint32_t src_hc = 0; src_hc < n_hc; src_hc++) {
-                const float comb_v = comb[dst_hc + (uint64_t)src_hc * n_hc];
-                const float res_v = residual_hc[(uint64_t)src_hc * n_embd + d];
-                hc_acc += comb_v * res_v;
-            }
-            out_hc[(uint64_t)dst_hc * n_embd + d] = hc_acc;
-        }
-    }
-}
-
 __global__ static void matmul_q8_0_preq_batch_warp8_kernel(
         float *out,
         const unsigned char *w,
@@ -396,47 +224,6 @@ __device__ static float q8_0_scale_broadcast_w32(const unsigned char *blk) {
 #endif
 }
 
-__device__ static float q8_block_sum_w32(float v) {
-    __shared__ float sh[32];
-    const uint32_t tid = threadIdx.x;
-    const uint32_t lane = tid & 31u;
-    const uint32_t wid = tid >> 5u;
-    const uint32_t nwarp = (blockDim.x + 31u) >> 5u;
-    v = warp_sum_f32(v);
-    if (lane == 0u) sh[wid] = v;
-    __syncthreads();
-    v = (tid < nwarp) ? sh[lane] : 0.0f;
-    if (wid == 0u) v = warp_sum_f32(v);
-    if (tid == 0u) sh[0] = v;
-    __syncthreads();
-    return sh[0];
-}
-
-__global__ static void matmul_q8_0_f32_small_block_w32_kernel(
-        float *out,
-        const unsigned char *w,
-        const float *x,
-        uint32_t n_blocks,
-        uint64_t out_dim,
-        uint64_t row_bytes) {
-    const uint64_t row = (uint64_t)blockIdx.x;
-    if (row >= out_dim) return;
-    const uint32_t tid = threadIdx.x;
-    const uint32_t lane = tid & 31u;
-    const uint32_t wave = tid >> 5u;
-    const uint32_t waves_per_block = blockDim.x >> 5u;
-    const unsigned char *wr = w + row * row_bytes;
-    float acc = 0.0f;
-    for (uint32_t b = wave; b < n_blocks; b += waves_per_block) {
-        const unsigned char *blk = wr + (uint64_t)b * 34u;
-        const float d = q8_0_scale_broadcast_w32(blk);
-        const int8_t q = ((const int8_t *)(blk + 2u))[lane];
-        acc += d * (float)q * x[((uint64_t)b << 5u) + lane];
-    }
-    acc = q8_block_sum_w32(acc);
-    if (tid == 0u) out[row] = acc;
-}
-
 __global__ static void matmul_q8_0_f32_warp8_kernel(
         float *out,
         const unsigned char *w,
@@ -847,78 +634,6 @@ __global__ static void matmul_q8_0_f32_batch_wmma_rowtile_kernel(
     }
 }
 
-template <int TILES_N=8, int BM=16, int BN=16, int BK=16>
-__global__ static void matmul_q8_0_f32_batch_wmma_onthefly_kernel(
-        float *out,
-        const unsigned char *w,
-        const float *x,
-        uint32_t n_tokens,
-        uint32_t in_dim,
-        uint32_t out_dim,
-        uint64_t row_bytes) {
-    extern __shared__ unsigned char raw_sh[];
-    half *shA = reinterpret_cast<half *>(raw_sh);
-    half *shB = shA + BM * BK;
-    float *shC = reinterpret_cast<float *>(shB + TILES_N * BK * BN);
-    const uint32_t tid = threadIdx.x;
-    const uint32_t wave = tid >> 5u;
-    const uint32_t t0 = (uint32_t)blockIdx.y * BM;
-    const uint32_t row0 = (uint32_t)blockIdx.x * TILES_N * BN;
-
-    using frag_a = rocwmma::fragment<rocwmma::matrix_a, BM, BN, BK, half, rocwmma::row_major>;
-    using frag_b = rocwmma::fragment<rocwmma::matrix_b, BM, BN, BK, half, rocwmma::row_major>;
-    using frag_c = rocwmma::fragment<rocwmma::accumulator, BM, BN, BK, float>;
-    frag_a a;
-    frag_b b;
-    frag_c acc;
-    if (wave < TILES_N) rocwmma::fill_fragment(acc, 0.0f);
-
-    for (uint32_t k0 = 0; k0 < in_dim; k0 += BK) {
-        for (uint32_t j = tid; j < BM * BK; j += blockDim.x) {
-            const uint32_t m = j / BK;
-            const uint32_t kk = j - m * BK;
-            const uint32_t t = t0 + m;
-            shA[j] = (t < n_tokens && k0 + kk < in_dim)
-                ? __float2half(x[(uint64_t)t * in_dim + k0 + kk])
-                : __float2half(0.0f);
-        }
-        for (uint32_t j = tid; j < TILES_N * BK * BN; j += blockDim.x) {
-            const uint32_t tn = j / (BK * BN);
-            const uint32_t rem = j - tn * BK * BN;
-            const uint32_t kk = rem / BN;
-            const uint32_t nn = rem - kk * BN;
-            const uint32_t row = row0 + tn * BN + nn;
-            const uint32_t k = k0 + kk;
-            if (row < out_dim && k < in_dim) {
-                const unsigned char *blk = w + (uint64_t)row * row_bytes + (uint64_t)(k >> 5u) * 34u;
-                const float d = __half2float(*(const half *)blk);
-                const int8_t q = ((const int8_t *)(blk + 2u))[k & 31u];
-                shB[j] = __float2half(d * (float)q);
-            } else {
-                shB[j] = __float2half(0.0f);
-            }
-        }
-        __syncthreads();
-        if (wave < TILES_N) {
-            rocwmma::load_matrix_sync(a, shA, BK);
-            rocwmma::load_matrix_sync(b, shB + wave * BK * BN, BN);
-            rocwmma::mma_sync(acc, a, b, acc);
-        }
-        __syncthreads();
-    }
-
-    if (wave < TILES_N) rocwmma::store_matrix_sync(shC + wave * BM * BN, acc, BN, rocwmma::mem_row_major);
-    __syncthreads();
-    for (uint32_t j = tid; j < TILES_N * BM * BN; j += blockDim.x) {
-        const uint32_t tn = j / (BM * BN);
-        const uint32_t rem = j - tn * BM * BN;
-        const uint32_t m = rem / BN;
-        const uint32_t nn = rem - m * BN;
-        const uint32_t t = t0 + m;
-        const uint32_t row = row0 + tn * BN + nn;
-        if (t < n_tokens && row < out_dim) out[(uint64_t)t * out_dim + row] = shC[j];
-    }
-}
 #endif
 
 __global__ static void matmul_q8_0_pair_f32_warp8_kernel(
@@ -1232,182 +947,6 @@ __device__ static float q8_0_scale_broadcast_oldhip_w32(const unsigned char *blk
 #endif
 }
 
-__global__ static void matmul_q8_0_hc_partial16_w32_kernel(
-        float *partial,
-        const unsigned char *w,
-        const float *x,
-        uint32_t out_dim,
-        uint64_t row_bytes) {
-    extern __shared__ float shx[];
-    const uint32_t tid = threadIdx.x;
-    const uint32_t lane = tid & 31u;
-    const uint32_t wave = tid >> 5;
-    const uint32_t rows_per_block = blockDim.x >> 5;
-    const uint32_t split = blockIdx.y;
-    const uint32_t b0 = split << 4;
-    for (uint32_t i = tid; i < 512u; i += blockDim.x) shx[i] = x[((uint64_t)b0 << 5) + i];
-    __syncthreads();
-
-    const uint32_t row = blockIdx.x * rows_per_block + wave;
-    if (row >= out_dim) return;
-    const unsigned char *wr = w + (uint64_t)row * row_bytes;
-    float acc = 0.0f;
-#pragma unroll
-    for (uint32_t bb = 0; bb < 16u; bb++) {
-        const uint32_t b = b0 + bb;
-        const unsigned char *blk = wr + (uint64_t)b * 34u;
-        const float d = q8_0_scale_broadcast_oldhip_w32(blk);
-        const int8_t q = ((const int8_t *)(blk + 2u))[lane];
-        acc += d * (float)q * shx[(bb << 5) + lane];
-    }
-    acc = warp_sum_f32_oldhip_w32(acc);
-    if (lane == 0u) partial[(uint64_t)split * out_dim + row] = acc;
-}
-
-__global__ static void matmul_q8_0_hc_partial_w32_kernel(
-        float *partial,
-        const unsigned char *w,
-        const float *x,
-        uint32_t n_blocks,
-        uint32_t out_dim,
-        uint64_t row_bytes,
-        uint32_t n_splits) {
-    extern __shared__ float shx[];
-    const uint32_t tid = threadIdx.x;
-    const uint32_t lane = tid & 31u;
-    const uint32_t wave = tid >> 5;
-    const uint32_t rows_per_block = blockDim.x >> 5;
-    const uint32_t split = blockIdx.y;
-    const uint32_t chunk = (n_blocks + n_splits - 1u) / n_splits;
-    const uint32_t b0 = split * chunk;
-    const uint32_t b1 = min(n_blocks, b0 + chunk);
-    const uint32_t chunk_blocks = b1 > b0 ? b1 - b0 : 0u;
-    for (uint32_t i = tid; i < (chunk_blocks << 5); i += blockDim.x) shx[i] = x[((uint64_t)b0 << 5) + i];
-    __syncthreads();
-
-    const uint32_t row = blockIdx.x * rows_per_block + wave;
-    if (row >= out_dim) return;
-    const unsigned char *wr = w + (uint64_t)row * row_bytes;
-    float acc = 0.0f;
-    for (uint32_t bb = 0; bb < chunk_blocks; bb++) {
-        const uint32_t b = b0 + bb;
-        const unsigned char *blk = wr + (uint64_t)b * 34u;
-        const float d = q8_0_scale_broadcast_oldhip_w32(blk);
-        const int8_t q = ((const int8_t *)(blk + 2u))[lane];
-        acc += d * (float)q * shx[(bb << 5) + lane];
-    }
-    acc = warp_sum_f32_oldhip_w32(acc);
-    if (lane == 0u) partial[(uint64_t)split * out_dim + row] = acc;
-}
-
-__global__ static void hc_expand_partial_kernel(
-        float *out_hc,
-        float *block_out,
-        const float *partial,
-        const float *residual_hc,
-        const float *split,
-        uint32_t out_dim,
-        uint32_t n_hc,
-        uint32_t n_splits,
-        int store_block_out) {
-    const uint32_t row = blockIdx.x * blockDim.x + threadIdx.x;
-    if (row >= out_dim) return;
-    float acc = 0.0f;
-    for (uint32_t s = 0; s < n_splits; s++) acc += partial[(uint64_t)s * out_dim + row];
-    if (store_block_out) block_out[row] = acc;
-    const float *post = split + n_hc;
-    const float *comb = split + 2u * n_hc;
-    for (uint32_t dst = 0; dst < n_hc; dst++) {
-        float v = acc * post[dst];
-        for (uint32_t src = 0; src < n_hc; src++) {
-            v += comb[dst + (uint64_t)src * n_hc] * residual_hc[(uint64_t)src * out_dim + row];
-        }
-        out_hc[(uint64_t)dst * out_dim + row] = v;
-    }
-}
-
-__global__ static void hc_expand_add_partial_kernel(
-        float *out_hc,
-        float *block_out,
-        const float *partial,
-        const float *block_add,
-        const float *residual_hc,
-        const float *split,
-        uint32_t out_dim,
-        uint32_t n_hc,
-        uint32_t n_splits,
-        int store_block_out) {
-    const uint32_t row = blockIdx.x * blockDim.x + threadIdx.x;
-    if (row >= out_dim) return;
-    float acc = 0.0f;
-    for (uint32_t s = 0; s < n_splits; s++) acc += partial[(uint64_t)s * out_dim + row];
-    if (store_block_out) block_out[row] = acc;
-    const float block = acc + block_add[row];
-    const float *post = split + n_hc;
-    const float *comb = split + 2u * n_hc;
-    for (uint32_t dst = 0; dst < n_hc; dst++) {
-        float v = block * post[dst];
-        for (uint32_t src = 0; src < n_hc; src++) {
-            v += comb[dst + (uint64_t)src * n_hc] * residual_hc[(uint64_t)src * out_dim + row];
-        }
-        out_hc[(uint64_t)dst * out_dim + row] = v;
-    }
-}
-
-__global__ static void hc_expand_add_partial4_kernel(
-        float *out_hc,
-        float *block_out,
-        const float *partial,
-        const float *block_add,
-        const float *residual_hc,
-        const float *split,
-        uint32_t out_dim,
-        uint32_t n_hc,
-        int store_block_out) {
-    const uint32_t row = blockIdx.x * blockDim.x + threadIdx.x;
-    if (row >= out_dim) return;
-    float acc = 0.0f;
-#pragma unroll
-    for (uint32_t s = 0; s < 4u; s++) acc += partial[(uint64_t)s * out_dim + row];
-    if (store_block_out) block_out[row] = acc;
-    const float block = acc + block_add[row];
-    const float *post = split + n_hc;
-    const float *comb = split + 2u * n_hc;
-    for (uint32_t dst = 0; dst < n_hc; dst++) {
-        float v = block * post[dst];
-        for (uint32_t src = 0; src < n_hc; src++) {
-            v += comb[dst + (uint64_t)src * n_hc] * residual_hc[(uint64_t)src * out_dim + row];
-        }
-        out_hc[(uint64_t)dst * out_dim + row] = v;
-    }
-}
-
-__global__ static void hc_expand_partial16_kernel(
-        float *out_hc,
-        float *block_out,
-        const float *partial,
-        const float *residual_hc,
-        const float *split,
-        uint32_t out_dim,
-        uint32_t n_hc,
-        int store_block_out) {
-    const uint32_t row = blockIdx.x * blockDim.x + threadIdx.x;
-    if (row >= out_dim) return;
-    float acc = 0.0f;
-#pragma unroll
-    for (uint32_t s = 0; s < 16u; s++) acc += partial[(uint64_t)s * out_dim + row];
-    if (store_block_out) block_out[row] = acc;
-    const float *post = split + n_hc;
-    const float *comb = split + 2u * n_hc;
-    for (uint32_t dst = 0; dst < n_hc; dst++) {
-        float v = acc * post[dst];
-        for (uint32_t src = 0; src < n_hc; src++) {
-            v += comb[dst + (uint64_t)src * n_hc] * residual_hc[(uint64_t)src * out_dim + row];
-        }
-        out_hc[(uint64_t)dst * out_dim + row] = v;
-    }
-}
-
 __global__ static void grouped_q8_0_a_f32_warp8_kernel(
         float *low,
         const unsigned char *w,
@@ -1719,87 +1258,6 @@ __global__ static void grouped_q8_0_a_f32_batch_sharedx_chunked_strided_w32_kern
     }
 }
 
-#if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__)
-template <int TILES_N=8, int BM=16, int BN=16, int BK=16>
-__global__ static void grouped_q8_0_a_f32_batch_wmma_onthefly_kernel(
-        float *low,
-        const unsigned char *w,
-        const float *heads,
-        uint32_t n_tokens,
-        uint32_t n_groups,
-        uint32_t group_dim,
-        uint32_t rank,
-        uint64_t row_bytes) {
-    extern __shared__ unsigned char raw_sh[];
-    half *shA = reinterpret_cast<half *>(raw_sh);
-    half *shB = shA + BM * BK;
-    float *shC = reinterpret_cast<float *>(shB + TILES_N * BK * BN);
-    const uint32_t tid = threadIdx.x;
-    const uint32_t wave = tid >> 5u;
-    const uint32_t row_tiles_per_group = (rank + TILES_N * BN - 1u) / (TILES_N * BN);
-    const uint32_t g = (uint32_t)blockIdx.x / row_tiles_per_group;
-    const uint32_t row_tile = (uint32_t)blockIdx.x - g * row_tiles_per_group;
-    const uint32_t row0 = row_tile * TILES_N * BN;
-    const uint32_t t0 = (uint32_t)blockIdx.y * BM;
-    if (g >= n_groups) return;
-
-    using frag_a = rocwmma::fragment<rocwmma::matrix_a, BM, BN, BK, half, rocwmma::row_major>;
-    using frag_b = rocwmma::fragment<rocwmma::matrix_b, BM, BN, BK, half, rocwmma::row_major>;
-    using frag_c = rocwmma::fragment<rocwmma::accumulator, BM, BN, BK, float>;
-    frag_a a;
-    frag_b b;
-    frag_c acc;
-    if (wave < TILES_N) rocwmma::fill_fragment(acc, 0.0f);
-
-    for (uint32_t k0 = 0; k0 < group_dim; k0 += BK) {
-        for (uint32_t j = tid; j < BM * BK; j += blockDim.x) {
-            const uint32_t m = j / BK;
-            const uint32_t kk = j - m * BK;
-            const uint32_t t = t0 + m;
-            const uint32_t k = k0 + kk;
-            shA[j] = (t < n_tokens && k < group_dim)
-                ? __float2half(heads[((uint64_t)t * n_groups + g) * group_dim + k])
-                : __float2half(0.0f);
-        }
-        for (uint32_t j = tid; j < TILES_N * BK * BN; j += blockDim.x) {
-            const uint32_t tn = j / (BK * BN);
-            const uint32_t rem = j - tn * BK * BN;
-            const uint32_t kk = rem / BN;
-            const uint32_t nn = rem - kk * BN;
-            const uint32_t row = row0 + tn * BN + nn;
-            const uint32_t k = k0 + kk;
-            if (row < rank && k < group_dim) {
-                const unsigned char *blk = w + ((uint64_t)g * rank + row) * row_bytes + (uint64_t)(k >> 5u) * 34u;
-                const float d = __half2float(*(const half *)blk);
-                const int8_t q = ((const int8_t *)(blk + 2u))[k & 31u];
-                shB[j] = __float2half(d * (float)q);
-            } else {
-                shB[j] = __float2half(0.0f);
-            }
-        }
-        __syncthreads();
-        if (wave < TILES_N) {
-            rocwmma::load_matrix_sync(a, shA, BK);
-            rocwmma::load_matrix_sync(b, shB + wave * BK * BN, BN);
-            rocwmma::mma_sync(acc, a, b, acc);
-        }
-        __syncthreads();
-    }
-
-    if (wave < TILES_N) rocwmma::store_matrix_sync(shC + wave * BM * BN, acc, BN, rocwmma::mem_row_major);
-    __syncthreads();
-    for (uint32_t j = tid; j < TILES_N * BM * BN; j += blockDim.x) {
-        const uint32_t tn = j / (BM * BN);
-        const uint32_t rem = j - tn * BM * BN;
-        const uint32_t m = rem / BN;
-        const uint32_t nn = rem - m * BN;
-        const uint32_t t = t0 + m;
-        const uint32_t row = row0 + tn * BN + nn;
-        if (t < n_tokens && row < rank) low[((uint64_t)t * n_groups + g) * rank + row] = shC[j];
-    }
-}
-#endif
-
 __global__ static void dequant_q8_0_to_f16_kernel(
         __half *out,
         const unsigned char *w,
diff --git a/rocm/ds4_rocm_runtime.cuh b/rocm/ds4_rocm_runtime.cuh
index 00d4424877..845417f0c2 100644
--- a/rocm/ds4_rocm_runtime.cuh
+++ b/rocm/ds4_rocm_runtime.cuh
@@ -30,7 +30,6 @@ enum {
 static int g_rocblas_f16_solution_set;
 static int g_rocblas_f16_solutions_disabled;
 static int g_rocblas_attention_b_solution_disabled;
-#include "ds4_rocm_hipblaslt.cuh"
 #endif
 static int g_quality_mode;
 static int g_glm_model;
@@ -535,14 +534,6 @@ static int cuda_tensor_has_i32(const ds4_gpu_tensor *t, uint64_t elems) {
     return cuda_tensor_has_elems(t, elems, sizeof(int32_t));
 }
 
-static int cuda_tensor_has_f16(const ds4_gpu_tensor *t, uint64_t elems) {
-    return cuda_tensor_has_elems(t, elems, sizeof(__half));
-}
-
-static int cuda_tensor_has_u16(const ds4_gpu_tensor *t, uint64_t elems) {
-    return cuda_tensor_has_elems(t, elems, sizeof(uint16_t));
-}
-
 static const char *cuda_model_range_ptr_from_fd(
         const void *model_map,
         uint64_t offset,
@@ -576,8 +567,6 @@ __global__ static void dequant_q8_0_to_f16_transpose_kernel(
         uint64_t out_dim,
         uint64_t blocks);
 
-static void cuda_shared_gate_up_async_cleanup(void);
-
 static void *cuda_tmp_alloc(uint64_t bytes, const char *what) {
     if (bytes == 0) return NULL;
     if (g_cuda_tmp_bytes >= bytes) return g_cuda_tmp;
@@ -3647,191 +3636,6 @@ static int cuda_stream_batch_selected_prepare(
     return ok;
 }
 
-static int cuda_stream_layer_expert_cache_prepare_batch(
-        const void *model_map,
-        uint32_t layer,
-        const ds4_gpu_tensor *selected,
-        uint32_t n_tokens,
-        uint32_t n_total_expert,
-        uint32_t n_selected,
-        uint64_t gate_offset,
-        uint64_t up_offset,
-        uint64_t down_offset,
-        uint64_t gate_expert_bytes,
-        uint64_t down_expert_bytes,
-        const ds4_gpu_tensor **selected_exec,
-        const char ***gate_ptrs,
-        const char ***up_ptrs,
-        const char ***down_ptrs,
-        uint32_t *unique_out) {
-    if (!selected ||
-        !selected_exec ||
-        !gate_ptrs ||
-        !up_ptrs ||
-        !down_ptrs ||
-        !unique_out ||
-        !cuda_tensor_has_elems2(selected, n_tokens, n_selected, sizeof(int32_t)) ||
-        n_tokens <= 1 ||
-        n_total_expert == 0 ||
-        n_total_expert > DS4_ROCM_MAX_N_EXPERT ||
-        n_selected == 0 ||
-        n_selected > DS4_ROCM_N_EXPERT_USED ||
-        gate_expert_bytes == 0 ||
-        down_expert_bytes == 0) {
-        return 0;
-    }
-    const char *layer_gate = NULL;
-    const char *layer_up = NULL;
-    const char *layer_down = NULL;
-    if (!cuda_stream_layer_expert_cache_apply(model_map,
-                                              layer,
-                                              n_total_expert,
-                                              gate_offset,
-                                              up_offset,
-                                              down_offset,
-                                              gate_expert_bytes,
-                                              down_expert_bytes,
-                                              &layer_gate,
-                                              &layer_up,
-                                              &layer_down)) {
-        return 0;
-    }
-
-    uint64_t n_ids64 = 0;
-    if (!cuda_u64_mul_checked(n_tokens, n_selected, &n_ids64) ||
-        n_ids64 > SIZE_MAX / sizeof(int32_t)) {
-        return 0;
-    }
-    int32_t *ids = (int32_t *)malloc((size_t)n_ids64 * sizeof(ids[0]));
-    int32_t *compact_ids =
-        (int32_t *)malloc((size_t)n_ids64 * sizeof(compact_ids[0]));
-    if (!ids || !compact_ids) {
-        free(ids);
-        free(compact_ids);
-        return 0;
-    }
-
-    int ok = cuda_ok(cudaMemcpy(ids,
-                                selected->ptr,
-                                (size_t)n_ids64 * sizeof(ids[0]),
-                                cudaMemcpyDeviceToHost),
-                     "streaming full-layer selected ids copy");
-
-    int32_t map[DS4_ROCM_MAX_N_EXPERT];
-    int32_t unique_ids[DS4_ROCM_MAX_N_EXPERT];
-    for (uint32_t i = 0; i < DS4_ROCM_MAX_N_EXPERT; i++) map[i] = -1;
-    uint32_t unique_count = 0;
-    for (uint64_t i = 0; ok && i < n_ids64; i++) {
-        const int32_t expert = ids[i];
-        if (expert < 0 || (uint32_t)expert >= n_total_expert) {
-            fprintf(stderr,
-                    DS4_GPU_LOG_PREFIX "streaming full-layer selected expert id %d "
-                    "outside 0..%u (layer=%u)\n",
-                    expert,
-                    n_total_expert,
-                    layer);
-            ok = 0;
-            break;
-        }
-        int32_t slot = map[(uint32_t)expert];
-        if (slot < 0) {
-            if (unique_count >= DS4_ROCM_MAX_N_EXPERT) {
-                ok = 0;
-                break;
-            }
-            slot = (int32_t)unique_count;
-            map[(uint32_t)expert] = slot;
-            unique_ids[unique_count++] = expert;
-        }
-        compact_ids[i] = slot;
-    }
-    if (ok && unique_count == 0) ok = 0;
-    if (ok && !cuda_stream_batch_selected_ensure_buffers(n_ids64, unique_count)) {
-        ok = 0;
-    }
-    if (ok && !cuda_stream_selected_ensure_stream()) ok = 0;
-
-    const char *gate_host[DS4_ROCM_MAX_N_EXPERT] = {0};
-    const char *up_host[DS4_ROCM_MAX_N_EXPERT] = {0};
-    const char *down_host[DS4_ROCM_MAX_N_EXPERT] = {0};
-    for (uint32_t u = 0; ok && u < unique_count; u++) {
-        const uint64_t expert = (uint64_t)(uint32_t)unique_ids[u];
-        uint64_t gate_rel = 0;
-        uint64_t down_rel = 0;
-        if (!cuda_u64_mul_checked(expert, gate_expert_bytes, &gate_rel) ||
-            !cuda_u64_mul_checked(expert, down_expert_bytes, &down_rel)) {
-            ok = 0;
-            break;
-        }
-        gate_host[u] = layer_gate + gate_rel;
-        up_host[u] = layer_up + gate_rel;
-        down_host[u] = layer_down + down_rel;
-    }
-
-    if (ok) {
-        cudaError_t err = cudaMemcpyAsync(g_stream_batch_selected_cache.selected_ids,
-                                          compact_ids,
-                                          (size_t)n_ids64 * sizeof(compact_ids[0]),
-                                          cudaMemcpyHostToDevice,
-                                          g_stream_selected_upload_stream);
-        if (err == cudaSuccess) {
-            err = cudaMemcpyAsync(g_stream_batch_selected_cache.gate_ptrs,
-                                  gate_host,
-                                  unique_count * sizeof(gate_host[0]),
-                                  cudaMemcpyHostToDevice,
-                                  g_stream_selected_upload_stream);
-        }
-        if (err == cudaSuccess) {
-            err = cudaMemcpyAsync(g_stream_batch_selected_cache.up_ptrs,
-                                  up_host,
-                                  unique_count * sizeof(up_host[0]),
-                                  cudaMemcpyHostToDevice,
-                                  g_stream_selected_upload_stream);
-        }
-        if (err == cudaSuccess) {
-            err = cudaMemcpyAsync(g_stream_batch_selected_cache.down_ptrs,
-                                  down_host,
-                                  unique_count * sizeof(down_host[0]),
-                                  cudaMemcpyHostToDevice,
-                                  g_stream_selected_upload_stream);
-        }
-        if (err == cudaSuccess) err = cudaStreamSynchronize(g_stream_selected_upload_stream);
-        if (err != cudaSuccess) {
-            fprintf(stderr,
-                    DS4_GPU_LOG_PREFIX "streaming full-layer selected table upload failed: %s\n",
-                    cudaGetErrorString(err));
-            (void)cudaGetLastError();
-            ok = 0;
-        }
-    }
-
-    if (ok) {
-        g_stream_batch_selected_cache.loaded = 0;
-        g_stream_batch_selected_cache.model_map = model_map;
-        g_stream_batch_selected_cache.layer = layer;
-        g_stream_batch_selected_cache.n_total_expert = n_total_expert;
-        g_stream_batch_selected_cache.n_selected = n_selected;
-        g_stream_batch_selected_cache.n_tokens = n_tokens;
-        g_stream_batch_selected_cache.n_unique = unique_count;
-        g_stream_batch_selected_cache.gate_offset = gate_offset;
-        g_stream_batch_selected_cache.up_offset = up_offset;
-        g_stream_batch_selected_cache.down_offset = down_offset;
-        g_stream_batch_selected_cache.gate_expert_bytes = gate_expert_bytes;
-        g_stream_batch_selected_cache.down_expert_bytes = down_expert_bytes;
-        *selected_exec = &g_stream_batch_selected_cache.selected_tensor;
-        *gate_ptrs = g_stream_batch_selected_cache.gate_ptrs;
-        *up_ptrs = g_stream_batch_selected_cache.up_ptrs;
-        *down_ptrs = g_stream_batch_selected_cache.down_ptrs;
-        *unique_out = unique_count;
-    } else {
-        g_stream_batch_selected_cache.loaded = 0;
-    }
-
-    free(ids);
-    free(compact_ids);
-    return ok;
-}
-
 static int cuda_stream_layer_expert_cache_seed_selected(
         const void *model_map,
         uint64_t model_size,
@@ -5874,22 +5678,14 @@ extern "C" int ds4_gpu_init(void) {
         __atomic_store_n(&g_rocblas_attention_b_solution_disabled, 0, __ATOMIC_RELAXED);
         g_rocblas_ready = 1;
     }
-    if (!g_hipblaslt_ready) {
-        if (hipblaslt_ok(hipblasLtCreate(&g_hipblaslt), "create handle")) {
-            g_hipblaslt_ready = 1;
-        }
-    }
 #endif
     return 1;
 }
 
 extern "C" void ds4_gpu_cleanup(void) {
     (void)cudaDeviceSynchronize();
+    (void)ds4_gpu_release_q4_attn_q_b_f16_sidecars();
     cuda_stream_cache_stats_print("cleanup");
-    cuda_shared_gate_up_async_cleanup();
-#ifdef __HIP_PLATFORM_AMD__
-    hipblaslt_gemm_plan_clear();
-#endif
     if (g_cublas_ready) {
         (void)cublasDestroy(g_cublas);
         g_cublas_ready = 0;
@@ -5904,11 +5700,6 @@ extern "C" void ds4_gpu_cleanup(void) {
         __atomic_store_n(&g_rocblas_f16_solutions_disabled, 0, __ATOMIC_RELAXED);
         __atomic_store_n(&g_rocblas_attention_b_solution_disabled, 0, __ATOMIC_RELAXED);
     }
-    if (g_hipblaslt_ready) {
-        (void)hipblasLtDestroy(g_hipblaslt);
-        g_hipblaslt_ready = 0;
-        g_hipblaslt = NULL;
-    }
 #endif
     cuda_model_range_release_all();
     cuda_q8_f16_cache_release_all();
@@ -5957,6 +5748,7 @@ extern "C" void ds4_gpu_cleanup(void) {
     g_model_device_owned = 0;
     g_model_range_mapping_supported = 1;
     g_model_fd = -1;
+    g_model_fd_host_base = NULL;
     if (g_model_direct_fd >= 0) {
         (void)close(g_model_direct_fd);
         g_model_direct_fd = -1;
@@ -5964,6 +5756,8 @@ extern "C" void ds4_gpu_cleanup(void) {
     g_model_direct_align = 1;
     g_model_file_size = 0;
     g_model_cache_full = 0;
+    g_ssd_streaming_mode = 0;
+    g_stream_expert_cache_budget = 0;
 }
 
 __global__ static void fill_f32_kernel(float *x, uint64_t n, float v);
@@ -6140,6 +5934,7 @@ extern "C" int ds4_gpu_set_model_map(const void *model_map, uint64_t model_size)
     const int multi_model =
         g_model_host_base != NULL &&
         (g_model_host_base != model_map || g_model_registered_size != model_size);
+    if (!ds4_gpu_release_q4_attn_q_b_f16_sidecars()) return 0;
     cuda_model_range_release_all();
     cuda_q8_f16_cache_release_all();
     g_q8_f16_disabled_after_oom = 0;
@@ -6234,6 +6029,46 @@ extern "C" int ds4_gpu_set_aux_model_map_range(
     return 1;
 }
 
+extern "C" int ds4_gpu_prepare_support_model(
+        const void *model_map,
+        uint64_t model_size,
+        uint64_t map_offset,
+        uint64_t map_size,
+        uint64_t max_tensor_bytes) {
+    (void)max_tensor_bytes;
+    if (!model_map || model_size == 0 || map_offset > model_size ||
+        map_size == 0 || map_size > model_size - map_offset) {
+        return 0;
+    }
+    if (g_model_fd < 0 || g_model_fd_host_base != model_map) {
+        fprintf(stderr,
+                DS4_GPU_LOG_PREFIX "support model fd does not match its mmap\n");
+        return 0;
+    }
+
+    /* ROCm's streaming range cache is intentionally replaced at every target
+     * layer. Store the small DSpark GGUF in the persistent, mmap-keyed image
+     * registry instead, while leaving the target map as the active mapping. */
+    const void *saved_host_base = g_model_host_base;
+    const char *saved_device_base = g_model_device_base;
+    const uint64_t saved_registered_size = g_model_registered_size;
+    const int saved_device_owned = g_model_device_owned;
+
+    if (!ds4_gpu_release_q4_attn_q_b_f16_sidecars()) return 0;
+    cuda_q8_f16_cache_release_all();
+    g_q8_f16_disabled_for_multi_model = 1;
+    const int ok = cuda_model_copy_chunked(model_map,
+                                           model_size,
+                                           map_offset,
+                                           map_size);
+
+    g_model_host_base = saved_host_base;
+    g_model_device_base = saved_device_base;
+    g_model_registered_size = saved_registered_size;
+    g_model_device_owned = saved_device_owned;
+    return ok;
+}
+
 extern "C" int ds4_gpu_set_model_map_spans(
         const void *model_map,
         uint64_t model_size,
@@ -6437,6 +6272,9 @@ extern "C" void ds4_gpu_print_memory_report(const char *label) {
 
 extern "C" void ds4_gpu_set_quality(bool quality) {
     const int new_quality_mode = quality ? 1 : 0;
+    if (new_quality_mode && !g_quality_mode) {
+        (void)ds4_gpu_release_q4_attn_q_b_f16_sidecars();
+    }
     if (g_quality_mode != new_quality_mode) {
         g_rocm_cfg.initialized = 0;
     }
diff --git a/rocm/ds4_rocm_shared_expert.cuh b/rocm/ds4_rocm_shared_expert.cuh
index 52a5598e38..9dc07a1f4a 100644
--- a/rocm/ds4_rocm_shared_expert.cuh
+++ b/rocm/ds4_rocm_shared_expert.cuh
@@ -211,198 +211,6 @@ extern "C" int ds4_gpu_shared_gate_up_swiglu_q8_0_rows_tensor(
                                                            n_tok);
 }
 
-static cudaStream_t g_shared_gate_up_stream = NULL;
-static cudaEvent_t g_shared_gate_up_ready_event = NULL;
-static void *g_shared_gate_up_tmp = NULL;
-static uint64_t g_shared_gate_up_tmp_bytes = 0;
-static int g_shared_gate_up_pending = 0;
-
-static int cuda_shared_gate_up_async_wait_internal(void) {
-    if (!g_shared_gate_up_pending) return 1;
-    cudaError_t err = cudaStreamSynchronize(g_shared_gate_up_stream);
-    g_shared_gate_up_pending = 0;
-    if (err != cudaSuccess) {
-        fprintf(stderr, DS4_GPU_LOG_PREFIX "shared gate/up async wait failed: %s\n", cudaGetErrorString(err));
-        (void)cudaGetLastError();
-        return 0;
-    }
-    return 1;
-}
-
-static void *cuda_shared_gate_up_async_tmp_alloc(uint64_t bytes) {
-    if (bytes == 0) return NULL;
-    if (g_shared_gate_up_tmp_bytes >= bytes) return g_shared_gate_up_tmp;
-    if (g_shared_gate_up_tmp) {
-        (void)cuda_shared_gate_up_async_wait_internal();
-        (void)cudaFree(g_shared_gate_up_tmp);
-        g_shared_gate_up_tmp = NULL;
-        g_shared_gate_up_tmp_bytes = 0;
-    }
-    void *ptr = NULL;
-    cudaError_t err = cudaMalloc(&ptr, (size_t)bytes);
-    if (err != cudaSuccess) {
-        fprintf(stderr, DS4_GPU_LOG_PREFIX "shared gate/up async temp alloc failed (%.2f MiB): %s\n",
-                (double)bytes / 1048576.0, cudaGetErrorString(err));
-        (void)cudaGetLastError();
-        return NULL;
-    }
-    g_shared_gate_up_tmp = ptr;
-    g_shared_gate_up_tmp_bytes = bytes;
-    return g_shared_gate_up_tmp;
-}
-
-static void cuda_shared_gate_up_async_cleanup(void) {
-    if (g_shared_gate_up_stream) {
-        (void)cuda_shared_gate_up_async_wait_internal();
-    }
-    if (g_shared_gate_up_tmp) {
-        (void)cudaFree(g_shared_gate_up_tmp);
-        g_shared_gate_up_tmp = NULL;
-        g_shared_gate_up_tmp_bytes = 0;
-    }
-    if (g_shared_gate_up_ready_event) {
-        (void)cudaEventDestroy(g_shared_gate_up_ready_event);
-        g_shared_gate_up_ready_event = NULL;
-    }
-    if (g_shared_gate_up_stream) {
-        (void)cudaStreamDestroy(g_shared_gate_up_stream);
-        g_shared_gate_up_stream = NULL;
-    }
-}
-
-extern "C" int ds4_gpu_shared_gate_up_swiglu_q8_0_async_tensor(
-        ds4_gpu_tensor       *gate,
-        ds4_gpu_tensor       *up,
-        ds4_gpu_tensor       *mid,
-        const void             *model_map,
-        uint64_t                model_size,
-        uint64_t                gate_offset,
-        uint64_t                up_offset,
-        uint64_t                in_dim,
-        uint64_t                out_dim,
-        const ds4_gpu_tensor *x,
-        float                   clamp) {
-    if (g_quality_mode || cuda_runtime_config()->graph_dump) return 0;
-    if (g_shared_gate_up_pending && !cuda_shared_gate_up_async_wait_internal()) return 0;
-    if (!gate || !up || !mid || !model_map || !x ||
-        in_dim == 0u || out_dim == 0u || in_dim > UINT32_MAX || out_dim > UINT32_MAX) {
-        return 0;
-    }
-    const uint64_t blocks = (in_dim + 31u) / 32u;
-    uint64_t row_bytes = 0;
-    uint64_t weight_bytes = 0;
-    if (!cuda_u64_mul_checked(blocks, 34u, &row_bytes) ||
-        !cuda_u64_mul_checked(out_dim, row_bytes, &weight_bytes)) {
-        return 0;
-    }
-    if (g_quality_mode ||
-        !gate || !up || !mid || !model_map || !x ||
-        in_dim == 0u || out_dim == 0u || in_dim > UINT32_MAX || out_dim > UINT32_MAX ||
-        gate_offset > model_size || up_offset > model_size ||
-        weight_bytes > model_size - gate_offset ||
-        weight_bytes > model_size - up_offset ||
-        x->bytes < in_dim * sizeof(float) ||
-        gate->bytes < out_dim * sizeof(float) ||
-        up->bytes < out_dim * sizeof(float) ||
-        mid->bytes < out_dim * sizeof(float)) {
-        return 0;
-    }
-    const char *wg = cuda_model_range_ptr(model_map, gate_offset, weight_bytes, "shared_gate_q8_pair_async");
-    const char *wu = cuda_model_range_ptr(model_map, up_offset, weight_bytes, "shared_up_q8_pair_async");
-    if (!wg || !wu) return 0;
-    if (!g_shared_gate_up_stream) {
-        int least_priority = 0;
-        int greatest_priority = 0;
-#ifdef __HIP_PLATFORM_AMD__
-        hipError_t err = hipDeviceGetStreamPriorityRange(&least_priority, &greatest_priority);
-        if (err == hipSuccess) {
-            err = hipStreamCreateWithPriority(&g_shared_gate_up_stream, cudaStreamNonBlocking, least_priority);
-        } else {
-            (void)cudaGetLastError();
-            err = hipStreamCreateWithFlags(&g_shared_gate_up_stream, cudaStreamNonBlocking);
-        }
-        if (err != hipSuccess) return 0;
-#else
-        cudaError_t err = cudaDeviceGetStreamPriorityRange(&least_priority, &greatest_priority);
-        if (err == cudaSuccess) {
-            err = cudaStreamCreateWithPriority(&g_shared_gate_up_stream, cudaStreamNonBlocking, least_priority);
-        } else {
-            (void)cudaGetLastError();
-            err = cudaStreamCreateWithFlags(&g_shared_gate_up_stream, cudaStreamNonBlocking);
-        }
-        if (err != cudaSuccess) return 0;
-#endif
-    }
-    if (!g_shared_gate_up_ready_event) {
-        cudaError_t err = cudaEventCreateWithFlags(&g_shared_gate_up_ready_event, cudaEventDisableTiming);
-        if (err != cudaSuccess) {
-            fprintf(stderr, DS4_GPU_LOG_PREFIX "shared gate/up async event create failed: %s\n", cudaGetErrorString(err));
-            (void)cudaGetLastError();
-            return 0;
-        }
-    }
-    /*
-     * This stream is intentionally non-blocking so it can overlap routed MoE.
-     * Non-blocking streams do not inherit default-stream ordering, so explicitly
-     * wait until the default-stream producer of x (ffn_norm) has completed before
-     * quantizing it here.
-     */
-    cudaError_t dep_err = cudaEventRecord(g_shared_gate_up_ready_event, 0);
-    if (dep_err != cudaSuccess) {
-        fprintf(stderr, DS4_GPU_LOG_PREFIX "shared gate/up async dependency record failed: %s\n", cudaGetErrorString(dep_err));
-        (void)cudaGetLastError();
-        return 0;
-    }
-#ifdef __HIP_PLATFORM_AMD__
-    dep_err = hipStreamWaitEvent(g_shared_gate_up_stream, g_shared_gate_up_ready_event, 0);
-#else
-    dep_err = cudaStreamWaitEvent(g_shared_gate_up_stream, g_shared_gate_up_ready_event, 0);
-#endif
-    if (dep_err != cudaSuccess) {
-        fprintf(stderr, DS4_GPU_LOG_PREFIX "shared gate/up async dependency wait failed: %s\n", cudaGetErrorString(dep_err));
-        (void)cudaGetLastError();
-        return 0;
-    }
-    const uint64_t xq_bytes = blocks * 32u;
-    const uint64_t scale_offset = (xq_bytes + 15u) & ~15ull;
-    const uint64_t tmp_bytes = scale_offset + blocks * sizeof(float);
-    void *tmp = cuda_shared_gate_up_async_tmp_alloc(tmp_bytes);
-    if (!tmp) return 0;
-    int8_t *xq = (int8_t *)tmp;
-    float *xscale = (float *)((char *)tmp + scale_offset);
-    const int use_dp4a = 1;
-    dim3 qgrid((unsigned)blocks, 1, 1);
-    quantize_q8_0_f32_kernel<<<qgrid, 32, 0, g_shared_gate_up_stream>>>(xq, xscale, (const float *)x->ptr, in_dim, blocks);
-    if (!cuda_ok(cudaGetLastError(), "shared gate/up async quantize launch")) return 0;
-    matmul_q8_0_pair_preq_warp8_kernel<<<((unsigned)out_dim + 7u) / 8u, 256, 0, g_shared_gate_up_stream>>>(
-            (float *)gate->ptr,
-            (float *)up->ptr,
-            reinterpret_cast<const unsigned char *>(wg),
-            reinterpret_cast<const unsigned char *>(wu),
-            xq,
-            xscale,
-            in_dim,
-            out_dim,
-            out_dim,
-            blocks,
-            use_dp4a);
-    if (!cuda_ok(cudaGetLastError(), "shared gate/up async pair launch")) return 0;
-    swiglu_kernel<<<((unsigned)out_dim + 255u) / 256u, 256, 0, g_shared_gate_up_stream>>>(
-            (float *)mid->ptr,
-            (const float *)gate->ptr,
-            (const float *)up->ptr,
-            (uint32_t)out_dim,
-            clamp,
-            1.0f);
-    if (!cuda_ok(cudaGetLastError(), "shared gate/up async swiglu launch")) return 0;
-    g_shared_gate_up_pending = 1;
-    return 1;
-}
-
-extern "C" int ds4_gpu_shared_gate_up_async_wait(void) {
-    return cuda_shared_gate_up_async_wait_internal();
-}
-
 extern "C" int ds4_gpu_shared_gate_up_swiglu_q8_0_batch_tensor(
         ds4_gpu_tensor       *gate,
         ds4_gpu_tensor       *up,
diff --git a/scripts/environment_variables.tsv b/scripts/environment_variables.tsv
new file mode 100644
index 0000000000..225ef25d60
--- /dev/null
+++ b/scripts/environment_variables.tsv
@@ -0,0 +1,1283 @@
+SCOPE	NAME	VALUE_DEFAULT_SEMANTICS	PURPOSE	SOURCE
+external/system	GGML_CUDA_DISABLE_GRAPHS	Pure presence flag cached on first is_enabled() call; any defined value, including empty or "0", disables. Unset permits CUDA graphs unless the GPU architecture independently disables them; relevant only when compiled with USE_CUDA_GRAPH.	Disable vendored GGML CUDA graph capture/replay and execute through the ordinary CUDA path.	cuda/mmq/common.cuh:1208
+external/system	HOME	Filesystem directory string. In ds4-agent, unset or empty falls back to "." for the default cache and history roots; the web helper applies the same fallback for its browser profile. A nonempty value roots .ds4/kvcache, .ds4_agent_history and .ds4/browser.	Choose the user's persistent ds4-agent cache, line-history and Chrome-profile base directory.	ds4_agent.c:4023
+external/system	LINENOISE_ASSUME_TTY	Pure presence test flag: any defined value, including empty or "0", treats non-TTY input as interactive while skipping real termios raw-mode setup/restoration. Unset follows isatty and uses plain blocking line input for pipes.	Exercise the live linenoise/agent editor, prompt and status layout in automated pipe-based tests without a real terminal.	linenoise.c:593
+external/system	LINENOISE_COLS	If defined, its value is returned directly through atoi with no validation: empty/nonnumeric becomes 0 and signed values are accepted. If unset, linenoise uses TIOCGWINSZ, then a cursor-position query, then fallback width 80.	Force a deterministic terminal column count for linenoise wrapping/layout tests.	linenoise.c:684
+external/system	PATH	Colon-separated executable search directories consulted only after DS4_CHROME, macOS app paths and fixed Chrome/Chromium paths fail. The first executable google-chrome, google-chrome-stable, chromium or chromium-browser wins; unset/empty/no match falls back to the literal command google-chrome, which execlp may search again.	Locate a Chrome/Chromium executable for the ds4 web/CDP tool.	ds4_web.c:992
+external/system	TERM	Case-insensitive terminal name. Values dumb, cons25, or emacs select linenoise's simple prompt plus blocking line reader; unset, empty, or any other value selects the normal interactive editor when stdin is a TTY.	Avoid ANSI/raw interactive editing on terminal types known not to support the required escape sequences.	linenoise.c:559
+runtime/bench	DS4_BENCH_DISABLE_SNAPSHOT	presence flag; unset allows snapshots for eligible frontiers	Disable benchmark state snapshots.	ds4_bench.c:720
+runtime/bench	DS4_BENCH_FORCE_SNAPSHOT	presence flag; unset obeys the normal size/eligibility checks	Force benchmark state snapshots despite the normal limit.	ds4_bench.c:723
+runtime/bench	DS4_BENCH_SNAPSHOT_MAX_BYTES	unsigned bytes or unlimited/inf; default DS4_BENCH_DEFAULT_SNAPSHOT_MAX_BYTES	Limit session snapshot size during benchmark sweeps.	ds4_bench.c:70
+runtime/cli	DS4_CLI_FORCE_SESSION	Pure presence flag: any defined value, including empty or "0", forces the session path. Unset uses the session path only for distributed coordinators, TP leaders, temperature>0, or MTP depth>1; otherwise the CLI calls direct argmax generation.	Force ordinary CLI generation through run_sampled_generation/session APIs so single-node validation follows the same stateful path as TP/distributed runs.	ds4_cli.c:1226
+runtime/core	DS4_BATCHED_FFN	Pure presence flag: any defined value, including empty or "0", enables. Unset leaves the default shared-expert-batched FFN path (or its configured fallback). It is read only by CPU layer-major prefill and takes precedence over shared-batch and token-parallel FFN choices.	Run the complete CPU prefill FFN in chunks through layer_ffn_batch instead of the default shared-expert-only batched path.	ds4.c:14411
+runtime/core	DS4_BATCHED_ROPE_MAX	Nonempty value parsed by strtol without full-string validation; integers 0..65536 are accepted, otherwise default 4096. Zero disables batched RoPE for every nonempty prompt. Effective only when prefix batch attention is selected and DS4_NO_BATCHED_ROPE is absent.	Set the largest CPU prefix-prefill token batch that applies RoPE and inverse RoPE with the batched kernels.	ds4.c:13911
+runtime/core	DS4_DECODE_PROFILE_DETAIL	presence flag; unset=off	Print per-stage timing for the single-token CPU FFN path.	ds4.c:12209
+runtime/core	DS4_DISABLE_GREEDY_TOP1_READBACK	presence rollback; unset uses device top-1 plus a 4-byte readback for eligible single-tier greedy generation, including SSD streaming; any defined value including empty or 0 restores full-logits readback and CPU argmax	Restore the legacy per-token full-logits host readback for greedy generation A/B and emergency rollback.	ds4.c:51964
+runtime/core	DS4_EXPERT_HOTLIST	nonempty filesystem path; unset=off; currently Metal-only	Load an expert hotlist for Metal expert profiling/streaming.	ds4.c:60637
+runtime/core	DS4_EXPERT_PROFILE	presence diagnostic flag; unset=off	Collect timing/profile diagnostics for expert profile.	ds4.c:60635
+runtime/core	DS4_LOCK_FILE	path string; default /tmp/ds4.lock	Override the single-instance lock file.	ds4.c:52260
+runtime/core	DS4_NO_BATCHED_ATTN	presence rollback flag; unset keeps default/optimized path	Disable/roll back no batched attn.	ds4.c:14410
+runtime/core	DS4_NO_BATCHED_ROPE	presence rollback flag; unset keeps default/optimized path	Disable/roll back no batched rope.	ds4.c:13918
+runtime/core	DS4_NO_PARALLEL_ATTN_ROWS	presence rollback flag; unset keeps default/optimized path	Disable/roll back no parallel attn rows.	ds4.c:13904
+runtime/core	DS4_NO_ROUTED_TOKEN_PARALLEL	presence rollback flag; unset keeps default/optimized path	Disable/roll back no routed token parallel.	ds4.c:12588
+runtime/core	DS4_NO_SHARED_BATCH_FFN	presence rollback flag; unset keeps default/optimized path	Disable/roll back no shared batch ffn.	ds4.c:14413
+runtime/core	DS4_ORACLE_LOGITS	filesystem path; unset disables read/write	Load reference logits for graph correctness comparison.	ds4.c:39056
+runtime/core	DS4_PARALLEL_ATTN_ROWS	Pure presence opt-in; any defined value enables the preference, but DS4_NO_PARALLEL_ATTN_ROWS overrides it. The path is eligible only for prefix prefill with cache n_raw==0 and pos0==0; unset uses per-token attention rows.	Batch/parallelize CPU prefix attention-row evaluation after cache/index preparation.	ds4.c:13901
+runtime/core	DS4_PARALLEL_FFN	Pure presence opt-in. It is effective only in CPU prefill when batched attention is enabled, DS4_BATCHED_FFN is absent, and the default shared-batch path has been disabled with DS4_NO_SHARED_BATCH_FFN; otherwise higher-priority paths win.	Run independent prompt-token FFNs through layer_ffn_tokens_parallel as the fallback after disabling shared batching.	ds4.c:14412
+runtime/core	DS4_PREFILL_BATCH	Nonempty value parsed by strtol without full-string validation; accepted range 1..4095, default 128 for unset/invalid/out-of-range values. It is used only when DS4_BATCHED_FFN selects full batched CPU FFN.	Set the token chunk size for layer_ffn_batch during CPU layer-major prefill.	ds4.c:14414
+runtime/core	DS4_PREFILL_PROFILE_DETAIL	presence flag; unset=off	Print detailed per-stage CPU prefill timing.	ds4.c:12555
+runtime/core	DS4_PREFILL_PROFILE_TOKEN	presence flag; effective within detailed prefill profiling	Print token-loop substage timings during CPU prefill.	ds4.c:14130
+runtime/core	DS4_ROUTED_TOKEN_PARALLEL	Pure presence flag that forces token-parallel routed MoE, even if DS4_NO_ROUTED_TOKEN_PARALLEL is also set. When unset, token parallelism is automatic for n_tok>=64 unless the NO flag is present; smaller batches use per-token routed MoE.	Choose token-parallel CPU routed-expert evaluation inside the default shared-batch FFN prefill path.	ds4.c:12587
+runtime/core	DS4_THREADS	positive integer; default min(online CPUs,12), capped by DS4_MAX_THREADS; CLI thread request overrides env	Set CPU worker-pool size.	ds4.c:1874
+runtime/core	DS4_TOKEN_TIMING	Pure presence flag; default off. Any defined value times and prints each CPU token decode evaluation; sampling, emission and callbacks are outside the measured interval.	Report per-token CPU model-evaluation latency during direct argmax generation.	ds4.c:41619
+runtime/core	DS4_TRACE_TOP	presence flag; unset=off	Print top-logit/token trace data during CPU generation.	ds4.c:41588
+runtime/cpu	DS4_CPU_DISABLE_UNROLLED_ARGMAX	presence rollback flag; unset keeps optimized/default path	Disable/roll back cpu disable unrolled argmax.	ds4.c:41075
+runtime/cpu	DS4_CPU_DUMP_LOGITS	filesystem path; unset disables read/write	Dump or select diagnostic data for cpu dump logits.	ds4.c:39086
+runtime/cpu	DS4_CPU_DUMP_PREFILL_LOGITS	filesystem path; unset disables read/write	Dump or select diagnostic data for cpu dump prefill logits.	ds4.c:41606
+runtime/cuda	DS4_CUDA_ATTENTION_OUTPUT_A_CUBLAS_MIN	integer tokens; default 2; accepted range 2..4095, otherwise 2	Set the token-count threshold for using cuBLAS on attention output-A.	ds4_cuda.cu:23925
+runtime/cuda	DS4_CUDA_ATTENTION_OUTPUT_PRELOAD	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Allow attention-output Q8 weights to be preloaded into the selective F16 cache.	ds4_cuda.cu:2402
+runtime/cuda	DS4_CUDA_ATTN_OUTPUT_PROFILE	presence diagnostic flag; default off; any defined value including 0 enables	Measure and print CUDA attention-output stage timings.	ds4_cuda.cu:23910
+runtime/cuda	DS4_CUDA_ATTN_Q_B_F32_CACHE	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Enable an F32-derived-weight cache for attention Q-B weights.	ds4_cuda.cu:2414
+runtime/cuda	DS4_CUDA_BUILD_ARTIFACTS	boolean-ish, default on for eligible derived artifacts; only exact 0 disables	Control construction of eligible CUDA derived/repacked weight artifacts.	ds4_cuda.cu:8440
+runtime/cuda	DS4_CUDA_COPY_MODEL	nonempty-string opt-in (but mere presence also suppresses prefetch); default off; value 0 is nonempty and requests a full copy	Copy the complete mapped model image into device memory.	ds4_cuda.cu:2740
+runtime/cuda	DS4_CUDA_COPY_MODEL_CHUNKED	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Use range-by-range model prefetch/copy preparation instead of the normal bulk preparation.	ds4_cuda.cu:37740
+runtime/cuda	DS4_CUDA_DECODE_GRAPHS	boolean, default on; any value starting with 0, or exact off/no/false in listed case variants, disables; oracle flags force off; effective only on one GPU	Control CUDA Graph capture and replay for decode.	ds4_cuda.cu:1468
+runtime/cuda	DS4_CUDA_DECODE_GRAPH_LOG	presence diagnostic flag; default off; any defined value including 0 enables	Log CUDA decode-graph cache misses, capture failures, and lifecycle events.	ds4_cuda.cu:1580
+runtime/cuda	DS4_CUDA_DECODE_HEADS8_ONLINE	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Force the eight-head online CUDA decode-attention kernel when eligible.	ds4_cuda.cu:371
+runtime/cuda	DS4_CUDA_DECODE_SCORE4	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Select four score lanes in the CUDA decode-attention fallback kernel.	ds4_cuda.cu:372
+runtime/cuda	DS4_CUDA_DECODE_SCORE8	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Select eight score lanes in the CUDA decode-attention fallback kernel.	ds4_cuda.cu:373
+runtime/cuda	DS4_CUDA_DIRECT_MODEL	mixed presence/nonempty flag, default off; any defined value bypasses host caching, while backend direct lookup requires nonempty; value 0 therefore still changes behavior	Use the mapped model directly and bypass selective CUDA weight caching.	ds4.c:3058; ds4_cuda.cu:1250
+runtime/cuda	DS4_CUDA_DISABLE_BATCH_INDEXER_QUERY_PRUNE	presence rollback; unset prunes unused zero-prefix indexer query work on eligible resident CUDA prefills; any value including 0 disables	Restore transient indexer Q projection, RoPE, QAT, and weight projection before compressed rows exceed top-k.	ds4.c:30137
+runtime/cuda	DS4_CUDA_DISABLE_DSPARK_EXACTN	value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables	Disable the CUDA DSpark exactn optimization.	ds4.c:52534
+runtime/cuda	DS4_CUDA_DISABLE_DSPARK_EXACTN_BATCH_HEAD	value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables	Disable the CUDA DSpark exactn batch head optimization.	ds4.c:37300
+runtime/cuda	DS4_CUDA_DISABLE_DSPARK_EXACTN_GRAPHS	value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables	Disable the CUDA DSpark exactn graphs optimization.	ds4.c:37266
+runtime/cuda	DS4_CUDA_DISABLE_DSPARK_NONCAUSAL_ONLINE	value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on	Disable the noncausal online-attention DSpark experiment.	ds4_cuda.cu:21691
+runtime/cuda	DS4_CUDA_DISABLE_HC_NORM_MIX_FUSE	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable fused HC RMSNorm-plus-mix.	ds4_cuda.cu:20905
+runtime/cuda	DS4_CUDA_DISABLE_HC_SPLIT_NORM_FUSED	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the fused HC split/weighted-sum/norm kernel.	ds4_cuda.cu:31785
+runtime/cuda	DS4_CUDA_DISABLE_IQ2_XXS_SSD_PREFILL_MMQ	false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate	Disable the CUDA IQ2 XXS SSD prefill MMQ optimization/path.	ds4_cuda.cu:4627
+runtime/cuda	DS4_CUDA_DISABLE_Q4_ATTN_OUT_HC_FUSE	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable fused Q4 attention-output/HC expansion.	ds4_cuda.cu:37357
+runtime/cuda	DS4_CUDA_DISABLE_Q4_ATTN_Q_B_F16_CACHE	value-aware rollback; unset/empty/0/false/no/off keeps the experiment available, any other nonempty value disables it	Disable the resident Q4_K attn_q_b-to-F16 prefill cache even when ENABLE or REQUIRE is set.	ds4_cuda.cu:2502
+runtime/cuda	DS4_CUDA_DISABLE_Q4_ATTN_Q_B_TRANSIENT_F16	value-aware rollback; unset/empty/0/false/no/off keeps the automatic transient path eligible, any other nonempty value disables it	Disable per-layer transient Q4_K attn_q_b-to-F16 scratch for physically device-resident single-GPU model images; the existing resident-cache controls remain independent and otherwise eligible calls use native Q4.	ds4_cuda.cu:2514
+runtime/cuda	DS4_CUDA_DISABLE_Q4_DENSE_PAIR	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the Q4 dense pair CUDA Q4 optimization.	ds4_cuda.cu:20402
+runtime/cuda	DS4_CUDA_DISABLE_Q4_GROUPED_ATTN_A_SINGLE_GRID	value-aware authoritative rollback; unset/empty/exact 0 permits the opt-in, every other nonempty value disables it and makes REQUIRE fail closed	Restore the established one-MMQ-launch-per-group grouped attention-A prefill path instead of mapping the groups to grid.z.	ds4_cuda.cu:41943
+runtime/cuda	DS4_CUDA_DISABLE_Q8_HC_EXPAND_FUSED	false-like-aware flag, default off; 0/false/no/off is off, other nonempty values request split; force-fused wins	Request the split Q8 shared-down/HC path when safe.	ds4_cuda.cu:2086
+runtime/cuda	DS4_CUDA_DISABLE_QKV_RMS_FUSED	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the CUDA QKV RMS fused optimization/path.	ds4_cuda.cu:369; ds4.c:17428
+runtime/cuda	DS4_CUDA_DISABLE_SHARED_GATE_UP_PAIR	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the CUDA shared gate up pair optimization/path.	ds4_cuda.cu:24293
+runtime/cuda	DS4_CUDA_DISABLE_STREAMING_EXPERT_PERSISTENT_CACHE	false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate	Disable streaming expert persistent cache in CUDA SSD streaming.	ds4_cuda.cu:4113
+runtime/cuda	DS4_CUDA_DISABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable streaming prefill batch selected addr in CUDA SSD streaming.	ds4.c:18592
+runtime/cuda	DS4_CUDA_DISABLE_STREAMING_PREFILL_BATCH_SELECTED_LOAD	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable streaming prefill batch selected load in CUDA SSD streaming.	ds4.c:21917
+runtime/cuda	DS4_CUDA_DISABLE_STREAMING_SELECTED_BATCH_IO	false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate	Disable streaming selected batch I/O in CUDA SSD streaming.	ds4_cuda.cu:4734
+runtime/cuda	DS4_CUDA_DISABLE_STREAMING_SELECTED_EVENT_PIPELINE	false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate	Disable streaming selected event pipeline in CUDA SSD streaming.	ds4_cuda.cu:4866
+runtime/cuda	DS4_CUDA_DISABLE_STREAMING_SELECTED_SHARED_OVERLAP	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable streaming selected shared overlap in CUDA SSD streaming.	ds4.c:20969
+runtime/cuda	DS4_CUDA_DSPARK_DEVICE_PROPOSER	value-aware opt-in, default off; 0/off/no/false (lowercase only) disable; other nonempty enables unless rollback set	Enable the CUDA-resident DSpark proposer.	ds4.c:34889; ds4_cuda.cu:19035
+runtime/cuda	DS4_CUDA_DSPARK_EXACT2	value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables	Enable the exact two-draft CUDA DSpark support path.	ds4.c:52247
+runtime/cuda	DS4_CUDA_DSPARK_EXACTN	value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables	Enable the exact multi-draft CUDA DSpark support path.	ds4.c:52268
+runtime/cuda	DS4_CUDA_DSPARK_EXACTN_BATCH_HEAD	value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables	Enable the batched output-head stage for exact-N DSpark verification.	ds4.c:37283
+runtime/cuda	DS4_CUDA_DSPARK_EXACTN_GRAPHS	value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables	Enable CUDA Graph capture for stable exact-N DSpark islands.	ds4.c:37249
+runtime/cuda	DS4_CUDA_DSPARK_NO_DEVICE_PROPOSER	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Disable the CUDA-resident DSpark proposer.	ds4.c:34899; ds4_cuda.cu:19040
+runtime/cuda	DS4_CUDA_DSPARK_NO_PADDED_HEAD	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Disable the padded CUDA output-head optimization used by DSpark.	ds4.c:34247
+runtime/cuda	DS4_CUDA_DSPARK_NO_Q_NORM_ROPE_FUSION	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Disable fused Q RMSNorm plus RoPE in DSpark support stages.	ds4.c:33623
+runtime/cuda	DS4_CUDA_DSPARK_PROPOSER_BLOCK_MAX	integer 0..UINT32_MAX; 0/invalid keeps native size; unset uses auto caps for exact-N/exact2; positive values cap the block and are limited by DS4_DSPARK_MAX_BLOCK_SIZE	Cap the CUDA DSpark proposal block length.	ds4.c:52354
+runtime/cuda	DS4_CUDA_DSPARK_TINY_ALIGNED_VEC	value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on	Use aligned routed-MoE vector kernels for tiny DSpark batches.	ds4_cuda.cu:29523
+runtime/cuda	DS4_CUDA_ENABLE_DSPARK_NONCAUSAL_ONLINE	value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on	Enable the small-batch noncausal online-attention DSpark experiment.	ds4_cuda.cu:21690
+runtime/cuda	DS4_CUDA_ENABLE_HC_NORM_MIX_FUSE	nonempty opt-in, default off; only exact 0 disables; the F32/F16 activation mode follows the selected standalone matmul path; disable/serial/alternate flags can veto	Enable and select the fused HC RMSNorm-plus-mix one-token implementation.	ds4_cuda.cu:20902
+runtime/cuda	DS4_CUDA_ENABLE_IQ2_XXS_SSD_PREFILL_MMQ	false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate	Enable the CUDA IQ2 XXS SSD prefill MMQ experimental path.	ds4_cuda.cu:4625
+runtime/cuda	DS4_CUDA_ENABLE_Q4_ATTN_OUT_HC_FUSE	value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on	Opt in to the fused Q4 attention-output/HC expansion path.	ds4_cuda.cu:37375
+runtime/cuda	DS4_CUDA_ENABLE_Q4_ATTN_Q_B_F16_CACHE	value-aware persistent-cache opt-in, default off; unset/empty/0/false/no/off is off; DISABLE cancels an optional persistent request but leaves the automatic transient path independent; REQUIRE plus DISABLE fails closed	Prewarm and use persistent resident F16 sidecars for eligible single-GPU Q4_K attn_q_b prefills.	ds4_cuda.cu:2490
+runtime/cuda	DS4_CUDA_ENABLE_Q4_ATTN_Q_B_F16_OUTPUT	value-aware experimental opt-in; unset/empty/0/false/no/off keeps the release F32 projection boundary; other nonempty values enable	Write eligible resident Q4_K attn_q_b GEMM output in F16 and run the half-input norm/RoPE epilogue; SSD remains excluded.	ds4_cuda.cu:2523
+runtime/cuda	DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_BATCH	value-aware opt-in, default off; nonempty value other than exact 0 enables; rollback wins	Enable flattened grouped attention-A MMQ for two-to-eight-token GB10 batches.	cuda/mmq/ds4_mmq.cu:4303
+runtime/cuda	DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_PREFILL	value-aware compatibility switch, default on; unset/empty uses the default, exact 0 opts out, any other nonempty value requests the path; REQUIRE also requests it; local/global rollback wins	Control direct-strided grouped Q4_K attention-A MMQ for GB10 prefill widths above eight tokens, removing per-group pack/unpack copies while preserving the per-group reduction tree.	ds4_cuda.cu:41928
+runtime/cuda	DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_SINGLE_GRID	value-aware opt-in, default off; unset/empty/exact 0 is off, every other nonempty value requests the candidate; REQUIRE also requests it; DISABLE wins	Submit all eligible GB10 grouped Q4_K attention-A prefill projections in one grid.z launch while isolating each group's stream-K coordinates and fixup storage.	ds4_cuda.cu:41941
+runtime/cuda	DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT	presence flag, default off; any defined value including 0 requests the path; rollback wins	Enable the GB10 persistent-CTA kernel for M=32768, N=1, K=1024 Q4.	cuda/mmq/ds4_mmq.cu:3905
+runtime/cuda	DS4_CUDA_ENABLE_Q8_FOLD	strict flag, default off; only exact value 1 enables; overridden by DS4_CUDA_NO_Q8_FOLD	Enable one-shot producer-to-consumer reuse of freshly quantized Q8_1 data.	ds4_cuda.cu:785
+runtime/cuda	DS4_CUDA_ENABLE_STREAMING_EXPERT_PERSISTENT_CACHE	false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate	Enable streaming expert persistent cache in CUDA SSD streaming.	ds4_cuda.cu:4111
+runtime/cuda	DS4_CUDA_ENABLE_STREAMING_SELECTED_BATCH_IO	false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate	Enable streaming selected batch I/O in CUDA SSD streaming.	ds4_cuda.cu:4732
+runtime/cuda	DS4_CUDA_ENABLE_STREAMING_SELECTED_EVENT_PIPELINE	false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate	Enable streaming selected event pipeline in CUDA SSD streaming.	ds4_cuda.cu:4864
+runtime/cuda	DS4_CUDA_END_STREAM_SYNC	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Synchronize only CUDA stream 0 at command-batch end instead of synchronizing the whole device.	ds4_cuda.cu:376
+runtime/cuda	DS4_CUDA_EXACT_SCORE_SPLIT_CHUNK	integer scores/chunk; default 512; clamped 1..8192	Tune exact score split chunk for exact score-split CUDA decode attention.	ds4_cuda.cu:13669
+runtime/cuda	DS4_CUDA_EXACT_SCORE_SPLIT_DECODE	value-aware boolean, default on; exact 0 disables; a nonzero explicit setting also takes precedence over split-KV selection	Control the exact score-split decode-attention implementation.	ds4_cuda.cu:13631
+runtime/cuda	DS4_CUDA_EXACT_SCORE_SPLIT_DIM2	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Tune exact score split dim2 for exact score-split CUDA decode attention.	ds4_cuda.cu:387
+runtime/cuda	DS4_CUDA_EXACT_SCORE_SPLIT_FUSE_INV_ROPE	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Tune exact score split fuse inv rope for exact score-split CUDA decode attention.	ds4_cuda.cu:390
+runtime/cuda	DS4_CUDA_EXACT_SCORE_SPLIT_GRAPH	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Tune exact score split graph for exact score-split CUDA decode attention.	ds4_cuda.cu:379
+runtime/cuda	DS4_CUDA_EXACT_SCORE_SPLIT_LDG	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Tune exact score split ldg for exact score-split CUDA decode attention.	ds4_cuda.cu:381
+runtime/cuda	DS4_CUDA_EXACT_SCORE_SPLIT_MIN_SCORE	integer score count; default 1; clamped 0..8192	Set the minimum visible-score count for exact score-split decode.	ds4_cuda.cu:13665
+runtime/cuda	DS4_CUDA_EXACT_SCORE_SPLIT_S	integer exact split count; unset/invalid = automatic; valid value clamped 1..16	Tune exact score split s for exact score-split CUDA decode attention.	ds4_cuda.cu:13679
+runtime/cuda	DS4_CUDA_EXACT_SCORE_SPLIT_S_FLOOR	integer split count; default 6; clamped 1..16	Tune exact score split s floor for exact score-split CUDA decode attention.	ds4_cuda.cu:13672
+runtime/cuda	DS4_CUDA_EXACT_SCORE_SPLIT_S_MAX	integer split count; default 16; clamped 1..16	Tune exact score split s max for exact score-split CUDA decode attention.	ds4_cuda.cu:13675
+runtime/cuda	DS4_CUDA_EXACT_SCORE_SPLIT_VEC4	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Tune exact score split vec4 for exact score-split CUDA decode attention.	ds4_cuda.cu:383
+runtime/cuda	DS4_CUDA_EXACT_SCORE_SPLIT_VEC4_PLAIN	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Tune exact score split vec4 plain for exact score-split CUDA decode attention.	ds4_cuda.cu:385
+runtime/cuda	DS4_CUDA_F16_CUBLAS_ONE	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Control the F16 cuBLAS one CUDA F16 matmul path.	ds4_cuda.cu:20852
+runtime/cuda	DS4_CUDA_F16_SMALL_BATCH	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Control the F16 small batch CUDA F16 matmul path.	ds4_cuda.cu:20837
+runtime/cuda	DS4_CUDA_F16_SMALL_OUT	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Control the F16 small out CUDA F16 matmul path.	ds4_cuda.cu:20819
+runtime/cuda	DS4_CUDA_GLM_VERIFY_NO_Q8_TOK2	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Control or tune the CUDA glm verify no Q8 tok2 path.	ds4_cuda.cu:19824
+runtime/cuda	DS4_CUDA_GREEDY_SPLITKV	value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables	Enable greedy split-KV fast attention.	ds4.c:17235
+runtime/cuda	DS4_CUDA_GREEDY_SPLITKV_FALLBACK_LOG	presence diagnostic flag; default off; any defined value including 0 enables	Control greedy splitkv fallback log in CUDA greedy fast decode.	ds4.c:55663
+runtime/cuda	DS4_CUDA_GREEDY_SPLITKV_MARGIN	nonnegative finite float; default 0.25; invalid value warns and uses 0.25; 0 disables margin fallback	Control greedy splitkv margin in CUDA greedy fast decode.	ds4.c:17313
+runtime/cuda	DS4_CUDA_GREEDY_SPLITKV_MAX_SEGMENT	integer 0..INT32_MAX; default/invalid 0 (segment cap disabled)	Control greedy splitkv max segment in CUDA greedy fast decode.	ds4.c:17389
+runtime/cuda	DS4_CUDA_GREEDY_SPLITKV_PAIR_REPLAY	value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables	Replay greedy split-KV tokens in pairs.	ds4.c:17363
+runtime/cuda	DS4_CUDA_GREEDY_SPLITKV_TOP2	value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables	Use top-2 output margins with greedy split-KV.	ds4.c:17345
+runtime/cuda	DS4_CUDA_GREEDY_SPLITKV_TRACE	presence diagnostic flag; default off; any defined value including 0 enables	Control greedy splitkv trace in CUDA greedy fast decode.	ds4.c:55698
+runtime/cuda	DS4_CUDA_GREEDY_SPLITKV_TRUST_REPLAY	value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables	Trust replayed greedy split-KV results without the normal confirmation policy.	ds4.c:17353
+runtime/cuda	DS4_CUDA_GREEDY_SPLIT_TOP1	value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables	Enable split top-1 selection in greedy CUDA decode.	ds4.c:17207
+runtime/cuda	DS4_CUDA_GREEDY_TOP1	value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables	Control greedy top1 in CUDA greedy fast decode.	ds4.c:56126
+runtime/cuda	DS4_CUDA_GREEDY_VEC4	value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables	Enable greedy vec4 fast attention.	ds4.c:17245
+runtime/cuda	DS4_CUDA_GREEDY_VEC4_FALLBACK_LOG	presence diagnostic flag; default off; any defined value including 0 enables	Control greedy vec4 fallback log in CUDA greedy fast decode.	ds4.c:55664
+runtime/cuda	DS4_CUDA_GREEDY_VEC4_MARGIN	nonnegative finite float; default 0.25; invalid value warns and uses 0.25; 0 disables margin fallback	Control greedy vec4 margin in CUDA greedy fast decode.	ds4.c:17283
+runtime/cuda	DS4_CUDA_GREEDY_VEC4_MAX_SEGMENT	integer 0..INT32_MAX; default/invalid 0 (segment cap disabled)	Control greedy vec4 max segment in CUDA greedy fast decode.	ds4.c:17397
+runtime/cuda	DS4_CUDA_GREEDY_VEC4_TRACE	presence diagnostic flag; default off; any defined value including 0 enables	Control greedy vec4 trace in CUDA greedy fast decode.	ds4.c:55727
+runtime/cuda	DS4_CUDA_INDEXED_TWOPASS	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Force the two-pass indexed-attention path instead of the fused heads8 online kernel.	ds4_cuda.cu:23587
+runtime/cuda	DS4_CUDA_IQ2_XXS_SSD_PREFILL_MMQ_STATS	false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate	Print CUDA IQ2 XXS SSD prefill MMQ counters.	ds4_cuda.cu:4633
+runtime/cuda	DS4_CUDA_KEEP_MODEL_PAGES	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Keep source model pages resident instead of advising the OS to discard copied pages.	ds4_cuda.cu:2840
+runtime/cuda	DS4_CUDA_MIXED_PREFILL_DECODE	value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables	Control native mixed prefill/decode scheduling.	ds4.c:70701
+runtime/cuda	DS4_CUDA_MIXED_ROUTED_MAX_PREFILL	integer rows 0..UINT32_MAX; default/invalid 512	Set the maximum prefill rows admitted to the mixed routed-MoE path.	ds4.c:70190
+runtime/cuda	DS4_CUDA_MIXED_ROUTED_SCATTER	value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables	Use scattered row handling in mixed routed-MoE execution.	ds4.c:69651
+runtime/cuda	DS4_CUDA_MMQ	boolean-ish, default on; any value beginning with 0 disables; quality mode and multi-GPU disable normal MMQ tier (MXFP4 path differs)	Control the vendored CUDA MMQ prefill tier.	ds4_cuda.cu:1722
+runtime/cuda	DS4_CUDA_MMQ_Q81_PERSISTENT	strict boolean, default off; accepts 1/on/true/yes and 0/off/false/no in listed lower/upper-case forms; unknown values are off	Reuse a persistent Q8_1 MMQ scratch arena on supported GB10 devices.	cuda/mmq/ds4_mmq.cu:152
+runtime/cuda	DS4_CUDA_MMQ_X_MAX	integer >=8; rounded down to multiple of 8 and only lowers the hardware base; invalid/unset = hardware base	Cap the MMQ X tile-width selector for architecture tuning.	cuda/mmq/mmq.cuh:127
+runtime/cuda	DS4_CUDA_MODEL_COPY_CHUNK_MB	positive integer MiB; default 64; clamped 16..4096	Set the chunk size used for CUDA model copying.	ds4_cuda.cu:2827
+runtime/cuda	DS4_CUDA_MODEL_COPY_VERBOSE	presence diagnostic flag; default off; any defined value including 0 enables	Print periodic progress while copying the model to device memory.	ds4_cuda.cu:6537
+runtime/cuda	DS4_CUDA_MODEL_PREFETCH_SYNC	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Synchronize after each CUDA model prefetch range for diagnostics.	ds4_cuda.cu:2808
+runtime/cuda	DS4_CUDA_MOE_ATOMIC_DOWN	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Select or tune the atomic down variant in CUDA routed-MoE dispatch.	ds4_cuda.cu:30086
+runtime/cuda	DS4_CUDA_MOE_DECODE_GRAPH	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Select or tune the decode graph variant in CUDA routed-MoE dispatch.	ds4_cuda.cu:391
+runtime/cuda	DS4_CUDA_MOE_DIRECT_MIDQ	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Select or tune the direct midq variant in CUDA routed-MoE dispatch.	ds4_cuda.cu:30140
+runtime/cuda	DS4_CUDA_MOE_DOWN_ROW1024	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Select or tune the down row1024 variant in CUDA routed-MoE dispatch.	ds4_cuda.cu:30109
+runtime/cuda	DS4_CUDA_MOE_DOWN_ROW128	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Select or tune the down row128 variant in CUDA routed-MoE dispatch.	ds4_cuda.cu:30128
+runtime/cuda	DS4_CUDA_MOE_DOWN_ROW2048	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Select or tune the down row2048 variant in CUDA routed-MoE dispatch.	ds4_cuda.cu:30110
+runtime/cuda	DS4_CUDA_MOE_DOWN_ROW256	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Select or tune the down row256 variant in CUDA routed-MoE dispatch.	ds4_cuda.cu:30127
+runtime/cuda	DS4_CUDA_MOE_DOWN_ROW512	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Select or tune the down row512 variant in CUDA routed-MoE dispatch.	ds4_cuda.cu:30108
+runtime/cuda	DS4_CUDA_MOE_DOWN_ROW64	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Select or tune the down row64 variant in CUDA routed-MoE dispatch.	ds4_cuda.cu:30129
+runtime/cuda	DS4_CUDA_MOE_GATE_ROW1024	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Select or tune the gate row1024 variant in CUDA routed-MoE dispatch.	ds4_cuda.cu:30120
+runtime/cuda	DS4_CUDA_MOE_GATE_ROW128	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Select or tune the gate row128 variant in CUDA routed-MoE dispatch.	ds4_cuda.cu:30093
+runtime/cuda	DS4_CUDA_MOE_GATE_ROW2048	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Select or tune the gate row2048 variant in CUDA routed-MoE dispatch.	ds4_cuda.cu:30091
+runtime/cuda	DS4_CUDA_MOE_GATE_ROW256	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Select or tune the gate row256 variant in CUDA routed-MoE dispatch.	ds4_cuda.cu:30092
+runtime/cuda	DS4_CUDA_MOE_MIDQ_SIDECAR	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Select or tune the midq sidecar variant in CUDA routed-MoE dispatch.	ds4_cuda.cu:30169
+runtime/cuda	DS4_CUDA_MOE_NO_ATOMIC_DOWN	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Disable the atomic down variant in CUDA routed-MoE dispatch.	ds4_cuda.cu:30087
+runtime/cuda	DS4_CUDA_MOE_NO_DECODE_LUT_GATE	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Disable the decode lut gate variant in CUDA routed-MoE dispatch.	ds4_cuda.cu:30117
+runtime/cuda	DS4_CUDA_MOE_NO_DIRECT_DOWN_SUM6	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Disable the direct down sum6 variant in CUDA routed-MoE dispatch.	ds4_cuda.cu:30137
+runtime/cuda	DS4_CUDA_MOE_NO_DIRECT_MIDQ	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Disable the direct midq variant in CUDA routed-MoE dispatch.	ds4_cuda.cu:30141
+runtime/cuda	DS4_CUDA_MOE_NO_DOWN_ROW128	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Disable the down row128 variant in CUDA routed-MoE dispatch.	ds4_cuda.cu:30133
+runtime/cuda	DS4_CUDA_MOE_NO_DOWN_ROW2048	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Disable the down row2048 variant in CUDA routed-MoE dispatch.	ds4_cuda.cu:30131
+runtime/cuda	DS4_CUDA_MOE_NO_DOWN_ROW256	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Disable the down row256 variant in CUDA routed-MoE dispatch.	ds4_cuda.cu:30132
+runtime/cuda	DS4_CUDA_MOE_NO_DOWN_ROW64	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Disable the down row64 variant in CUDA routed-MoE dispatch.	ds4_cuda.cu:30134
+runtime/cuda	DS4_CUDA_MOE_NO_DOWN_TILE16	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Disable the down tile16 variant in CUDA routed-MoE dispatch.	ds4_cuda.cu:30102
+runtime/cuda	DS4_CUDA_MOE_NO_EXPERT_TILES	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Disable the expert tiles variant in CUDA routed-MoE dispatch.	ds4_cuda.cu:30062
+runtime/cuda	DS4_CUDA_MOE_NO_GATE_ROW128	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Disable the gate row128 variant in CUDA routed-MoE dispatch.	ds4_cuda.cu:30097
+runtime/cuda	DS4_CUDA_MOE_NO_GATE_ROW2048	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Disable the gate row2048 variant in CUDA routed-MoE dispatch.	ds4_cuda.cu:30095
+runtime/cuda	DS4_CUDA_MOE_NO_GATE_ROW256	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Disable the gate row256 variant in CUDA routed-MoE dispatch.	ds4_cuda.cu:30096
+runtime/cuda	DS4_CUDA_MOE_NO_IQ2_ALIGNED	value-aware kill switch, default off; nonempty value other than exact 0 disables aligned IQ2 path	Disable the IQ2 aligned variant in CUDA routed-MoE dispatch.	ds4_cuda.cu:1011
+runtime/cuda	DS4_CUDA_MOE_NO_MIDQ_SIDECAR	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Disable the midq sidecar variant in CUDA routed-MoE dispatch.	ds4_cuda.cu:30170
+runtime/cuda	DS4_CUDA_MOE_NO_OWNED_SPARSE_BUFFERS	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Disable the owned sparse buffers variant in CUDA routed-MoE dispatch.	ds4_cuda.cu:30089
+runtime/cuda	DS4_CUDA_MOE_NO_P2	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Disable the p2 variant in CUDA routed-MoE dispatch.	ds4_cuda.cu:30084
+runtime/cuda	DS4_CUDA_MOE_NO_Q2K_ALIGNED	value-aware kill switch, default off; nonempty value other than exact 0 disables aligned Q2_K path	Disable the q2k aligned variant in CUDA routed-MoE dispatch.	ds4_cuda.cu:1016
+runtime/cuda	DS4_CUDA_MOE_NO_Q4_DOWN_ROWSPAN	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Disable the Q4 down rowspan variant in CUDA routed-MoE dispatch.	ds4_cuda.cu:30114
+runtime/cuda	DS4_CUDA_MOE_NO_Q4_DOWN_SLOT3	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Disable the Q4 down slot3 variant in CUDA routed-MoE dispatch.	ds4_cuda.cu:30164
+runtime/cuda	DS4_CUDA_MOE_NO_Q4_GATE_H16	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Disable the Q4 gate H16 variant in CUDA routed-MoE dispatch.	ds4_cuda.cu:30149
+runtime/cuda	DS4_CUDA_MOE_NO_Q4_GATE_H16R8	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Disable the Q4 gate H16R8 variant in CUDA routed-MoE dispatch.	ds4_cuda.cu:30145
+runtime/cuda	DS4_CUDA_MOE_NO_Q4_GATE_W32	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Disable the Q4 gate W32 variant in CUDA routed-MoE dispatch.	ds4_cuda.cu:30157
+runtime/cuda	DS4_CUDA_MOE_NO_Q4_GATE_W32R16	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Disable the Q4 gate W32R16 variant in CUDA routed-MoE dispatch.	ds4_cuda.cu:30153
+runtime/cuda	DS4_CUDA_MOE_NO_Q4_GATE_W32_NOAUX	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Disable the Q4 gate W32 no-aux variant in CUDA routed-MoE dispatch.	ds4_cuda.cu:30160
+runtime/cuda	DS4_CUDA_MOE_NO_Q4_MMA	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Disable the Q4 MMA variant in CUDA routed-MoE dispatch.	ds4_cuda.cu:312
+runtime/cuda	DS4_CUDA_MOE_NO_Q4_MMA_TILE16	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Disable the Q4 MMA tile16 variant in CUDA routed-MoE dispatch.	ds4_cuda.cu:30100
+runtime/cuda	DS4_CUDA_MOE_NO_Q4_SORTED	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Disable the Q4 sorted variant in CUDA routed-MoE dispatch.	ds4_cuda.cu:30061
+runtime/cuda	DS4_CUDA_MOE_NO_SMALL_SORTED_PREP	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Disable the small sorted prep variant in CUDA routed-MoE dispatch.	ds4_cuda.cu:30106
+runtime/cuda	DS4_CUDA_MOE_PROFILE	presence diagnostic flag; default off; any defined value including 0 enables	Measure and print routed-MoE CUDA kernel-stage timings.	ds4_cuda.cu:30045
+runtime/cuda	DS4_CUDA_MOE_Q4_DOWN_SLOT3	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Select or tune the Q4 down slot3 variant in CUDA routed-MoE dispatch.	ds4_cuda.cu:30163
+runtime/cuda	DS4_CUDA_MOE_Q4_GATE_H16	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Select or tune the Q4 gate H16 variant in CUDA routed-MoE dispatch.	ds4_cuda.cu:30148
+runtime/cuda	DS4_CUDA_MOE_Q4_GATE_H16R8	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Select or tune the Q4 gate H16R8 variant in CUDA routed-MoE dispatch.	ds4_cuda.cu:30144
+runtime/cuda	DS4_CUDA_MOE_Q4_GATE_W32R16	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Select or tune the Q4 gate W32R16 variant in CUDA routed-MoE dispatch.	ds4_cuda.cu:30152
+runtime/cuda	DS4_CUDA_MOE_TILE4	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Select or tune the tile4 variant in CUDA routed-MoE dispatch.	ds4_cuda.cu:30063
+runtime/cuda	DS4_CUDA_MOE_TILE8	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Select or tune the tile8 variant in CUDA routed-MoE dispatch.	ds4_cuda.cu:30079
+runtime/cuda	DS4_CUDA_MOE_WRITE_GATE_UP	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Select or tune the write gate up variant in CUDA routed-MoE dispatch.	ds4_cuda.cu:30081
+runtime/cuda	DS4_CUDA_NO_ATTENTION_OUTPUT_F16_CACHE	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the attention output F16 cache CUDA F16 path.	ds4_cuda.cu:2364
+runtime/cuda	DS4_CUDA_NO_ATTN_A_TOK2	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the CUDA attn a tok2 optimization/path.	ds4_cuda.cu:24024
+runtime/cuda	DS4_CUDA_NO_ATTN_Q_B_F16_CACHE	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the attn q b F16 cache CUDA F16 path.	ds4_cuda.cu:2367
+runtime/cuda	DS4_CUDA_NO_COMPRESSOR_PREFILL_BATCH	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the CUDA compressor prefill batch optimization/path.	ds4.c:29890
+runtime/cuda	DS4_CUDA_NO_CUBLAS_ATTENTION	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the CUDA cuBLAS attention optimization/path.	ds4_cuda.cu:23067
+runtime/cuda	DS4_CUDA_NO_CUBLAS_ATTENTION_OUTPUT_A	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the CUDA cuBLAS attention output a optimization/path.	ds4_cuda.cu:23934
+runtime/cuda	DS4_CUDA_NO_DECODE_VALUE512	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the 512-thread CUDA decode value/finalize specialization.	ds4_cuda.cu:374
+runtime/cuda	DS4_CUDA_NO_DERIVED_WEIGHTS	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the CUDA derived weights optimization/path.	ds4_cuda.cu:1034
+runtime/cuda	DS4_CUDA_NO_DIRECT_IO	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the CUDA direct I/O optimization/path.	cuda/mmq/ds4_repack.cu:68
+runtime/cuda	DS4_CUDA_NO_DIRECT_Q2_PREFILL	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the CUDA direct q2 prefill optimization/path.	ds4_cuda.cu:395
+runtime/cuda	DS4_CUDA_NO_EXACT_SCORE_SPLIT_DECODE	value-aware kill switch, default off; exact 0 is off, other nonempty values disable	Disable exact score split decode for exact score-split CUDA decode attention.	ds4_cuda.cu:13629; ds4.c:68110
+runtime/cuda	DS4_CUDA_NO_EXACT_SCORE_SPLIT_DIM2	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable exact score split dim2 for exact score-split CUDA decode attention.	ds4_cuda.cu:388
+runtime/cuda	DS4_CUDA_NO_F16_CUBLAS_BATCH	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the F16 cuBLAS batch CUDA F16 path.	ds4_cuda.cu:20854
+runtime/cuda	DS4_CUDA_NO_F16_CUBLAS_ONE	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the F16 cuBLAS one CUDA F16 path.	ds4_cuda.cu:20851
+runtime/cuda	DS4_CUDA_NO_F16_PAIR_COMPRESSOR_STORE	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the F16 pair compressor store CUDA F16 path.	ds4_cuda.cu:399
+runtime/cuda	DS4_CUDA_NO_F16_PAIR_COMPRESSOR_TRANSPOSE	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the F16 pair compressor transpose CUDA F16 path.	ds4_cuda.cu:21343
+runtime/cuda	DS4_CUDA_NO_F16_PAIR_COMPRESSOR_TRANSPOSE_PREFETCH8	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the F16 pair compressor transpose prefetch8 CUDA F16 path.	ds4_cuda.cu:21349
+runtime/cuda	DS4_CUDA_NO_F16_PAIR_MATMUL	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the F16 pair matmul CUDA F16 path.	ds4_cuda.cu:21130
+runtime/cuda	DS4_CUDA_NO_F16_SMALL_BATCH	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the F16 small batch CUDA F16 path.	ds4_cuda.cu:20838
+runtime/cuda	DS4_CUDA_NO_F16_SMALL_OUT	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the F16 small out CUDA F16 path.	ds4_cuda.cu:20821
+runtime/cuda	DS4_CUDA_NO_FD_CACHE	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the CUDA fd cache optimization/path.	ds4_cuda.cu:1274
+runtime/cuda	DS4_CUDA_NO_GREEDY_SPLITKV	value-aware kill switch; default off; nonempty value other than exact 0 disables	Disable greedy splitkv in CUDA greedy fast decode.	ds4.c:17233
+runtime/cuda	DS4_CUDA_NO_GREEDY_SPLITKV_FALLBACK	value-aware kill switch; default off; nonempty value other than exact 0 disables margin fallback	Disable greedy splitkv fallback in CUDA greedy fast decode.	ds4.c:17333
+runtime/cuda	DS4_CUDA_NO_GREEDY_SPLITKV_PAIR_REPLAY	value-aware kill switch; default off; nonempty value other than exact 0 disables	Disable greedy splitkv pair replay in CUDA greedy fast decode.	ds4.c:17361
+runtime/cuda	DS4_CUDA_NO_GREEDY_SPLITKV_TOP2	value-aware kill switch; default off; nonempty value other than exact 0 disables	Disable greedy splitkv top2 in CUDA greedy fast decode.	ds4.c:17343
+runtime/cuda	DS4_CUDA_NO_GREEDY_SPLIT_TOP1	value-aware kill switch; default off; nonempty value other than exact 0 disables	Disable greedy split top1 in CUDA greedy fast decode.	ds4.c:17205
+runtime/cuda	DS4_CUDA_NO_GREEDY_VEC4	value-aware kill switch; default off; nonempty value other than exact 0 disables	Disable greedy vec4 in CUDA greedy fast decode.	ds4.c:17243
+runtime/cuda	DS4_CUDA_NO_GREEDY_VEC4_FALLBACK	value-aware kill switch; default off; nonempty value other than exact 0 disables margin fallback	Disable greedy vec4 fallback in CUDA greedy fast decode.	ds4.c:17303
+runtime/cuda	DS4_CUDA_NO_HC_SPLIT_NORM_SPLIT4096	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the split partial-reduction specialization for one-row, 4096-wide HC normalization.	ds4_cuda.cu:31826
+runtime/cuda	DS4_CUDA_NO_INDEXED_HEADS8	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the CUDA indexed heads8 optimization/path.	ds4_cuda.cu:23586
+runtime/cuda	DS4_CUDA_NO_INDEXED_TOPK_SORT	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the CUDA indexed topk sort optimization/path.	ds4_cuda.cu:23577
+runtime/cuda	DS4_CUDA_NO_INDEXER_DIRECT_ONE	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the indexer direct one CUDA indexer kernel/path.	ds4_cuda.cu:18881
+runtime/cuda	DS4_CUDA_NO_INDEXER_MXF4	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the indexer MXF4 CUDA indexer kernel/path.	ds4_cuda.cu:17774
+runtime/cuda	DS4_CUDA_NO_INDEXER_WMMA	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the indexer WMMA CUDA indexer kernel/path.	ds4_cuda.cu:18891
+runtime/cuda	DS4_CUDA_NO_INDEXER_WMMA128	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the indexer wmma128 CUDA indexer kernel/path.	ds4_cuda.cu:18892
+runtime/cuda	DS4_CUDA_NO_INDEXER_WMMA32	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the indexer wmma32 CUDA indexer kernel/path.	ds4_cuda.cu:18910
+runtime/cuda	DS4_CUDA_NO_INDEXER_WMMA64	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the indexer wmma64 CUDA indexer kernel/path.	ds4_cuda.cu:18901
+runtime/cuda	DS4_CUDA_NO_IQ2_XXS_SSD_PREFILL_MMQ	false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate	Disable the CUDA IQ2 XXS SSD prefill MMQ optimization/path.	ds4_cuda.cu:4629
+runtime/cuda	DS4_CUDA_NO_MODEL_COPY	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the CUDA model copy optimization/path.	ds4_cuda.cu:6472
+runtime/cuda	DS4_CUDA_NO_MODEL_PREFETCH	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the CUDA model prefetch optimization/path.	ds4_cuda.cu:2739
+runtime/cuda	DS4_CUDA_NO_MOE_DEDUP	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the CUDA MoE dedup optimization/path.	cuda/mmq/ds4_mmq.cu:6172
+runtime/cuda	DS4_CUDA_NO_ORDERED_F16_MATMUL	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the ordered F16 matmul CUDA F16 path.	ds4_cuda.cu:20811
+runtime/cuda	DS4_CUDA_NO_PARALLEL_ROUTER_SELECT	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the CUDA parallel router select optimization/path.	ds4_cuda.cu:24491
+runtime/cuda	DS4_CUDA_NO_Q4_DENSE_SCRATCH	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the Q4 dense scratch CUDA Q4 optimization.	cuda/mmq/ds4_mmq.cu:3986
+runtime/cuda	DS4_CUDA_NO_Q4_GB10_FAST	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the GB10-specific Q4 fast-path family.	cuda/mmq/ds4_mmq.cu:3908
+runtime/cuda	DS4_CUDA_NO_Q4_GROUPED_ATTN_A	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the Q4 grouped attn a CUDA Q4 optimization.	cuda/mmq/ds4_mmq.cu:4295
+runtime/cuda	DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the Q4 grouped attn a batch CUDA Q4 optimization.	cuda/mmq/ds4_mmq.cu:4305
+runtime/cuda	DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL	presence kill switch for the default-on GB10 path; default unset; any defined value including empty or 0 disables and dominates ENABLE/REQUIRE	Restore the eight pack/MMQ/unpack Q4 attention-A prefill projections.	ds4_cuda.cu:41930
+runtime/cuda	DS4_CUDA_NO_Q4_GROUPED_ATTN_A_Q81	presence rollback for the default-on fixed-shape quantizer; default unset; any defined value including empty or 0 disables; REQUIRE then fails closed	Restore the canonical strided Q8_1 producer while retaining grouped Q4 attention-A prefill and its eight MMQ grids.	cuda/mmq/ds4_mmq.cu:1762; ds4_cuda.cu:44143
+runtime/cuda	DS4_CUDA_NO_Q4_K1024_PERSISTENT	presence kill switch, default off; any defined value including 0 disables	Disable the Q4 K1024 persistent CUDA Q4 optimization.	cuda/mmq/ds4_mmq.cu:3907
+runtime/cuda	DS4_CUDA_NO_Q4_MMQ_16WARP	value-aware rollback, default off; unset/empty/exact 0 permits the experiment, every other nonempty value disables it and overrides REQUEST/REQUIRE	Disable the experimental Stream-K-compatible CUDA Q4_K m128n128 16-warp prefill kernel.	cuda/mmq/ds4_mmq.cu:1133
+runtime/cuda	DS4_CUDA_NO_Q8_ALIGNED_DENSE_SCRATCH	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the Q8 aligned dense scratch CUDA Q8 optimization.	cuda/mmq/ds4_mmq.cu:5578
+runtime/cuda	DS4_CUDA_NO_Q8_ALIGNED_PERSISTENT	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the Q8 aligned persistent CUDA Q8 optimization.	cuda/mmq/ds4_mmq.cu:5410
+runtime/cuda	DS4_CUDA_NO_Q8_BATCH_EXACT_TOK2	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the Q8 batch exact tok2 CUDA Q8 optimization.	ds4_cuda.cu:19852
+runtime/cuda	DS4_CUDA_NO_Q8_BATCH_TOK4	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the Q8 batch tok4 CUDA Q8 optimization.	ds4_cuda.cu:19805
+runtime/cuda	DS4_CUDA_NO_Q8_BATCH_TOK8	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the Q8 batch tok8 CUDA Q8 optimization.	ds4_cuda.cu:19788
+runtime/cuda	DS4_CUDA_NO_Q8_BATCH_WARP	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the Q8 batch warp CUDA Q8 optimization.	ds4_cuda.cu:19787
+runtime/cuda	DS4_CUDA_NO_Q8_DP4A	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the Q8 DP4A CUDA Q8 optimization.	ds4_cuda.cu:2391
+runtime/cuda	DS4_CUDA_NO_Q8_F16_CACHE	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the Q8 F16 cache CUDA Q8 optimization.	ds4_cuda.cu:2356
+runtime/cuda	DS4_CUDA_NO_Q8_F32_CACHE	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the Q8 F32 cache CUDA Q8 optimization.	ds4_cuda.cu:2411
+runtime/cuda	DS4_CUDA_NO_Q8_FOLD	value-aware kill switch, default off; nonempty value other than exact 0 disables and wins over enable	Disable the Q8_1 producer-to-consumer fold.	ds4_cuda.cu:786
+runtime/cuda	DS4_CUDA_NO_Q8_FUSED_ALIGNED	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the Q8 fused aligned CUDA Q8 optimization.	ds4_cuda.cu:20141
+runtime/cuda	DS4_CUDA_NO_Q8_MMA	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the Q8 MMA CUDA Q8 optimization.	ds4_cuda.cu:10781
+runtime/cuda	DS4_CUDA_NO_Q8_PAIR_BATCH_EXACT	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the Q8 pair batch exact CUDA Q8 optimization.	ds4_cuda.cu:20302
+runtime/cuda	DS4_CUDA_NO_Q8_PAIR_BATCH_EXACT_TOK2	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the Q8 pair batch exact tok2 CUDA Q8 optimization.	ds4_cuda.cu:20305
+runtime/cuda	DS4_CUDA_NO_QKV_KV_ROPE_FUSE	value-aware kill switch; default off; nonempty value other than exact 0 disables	Disable the CUDA QKV KV rope fuse optimization/path.	ds4.c:17426
+runtime/cuda	DS4_CUDA_NO_QKV_PAIR	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the CUDA QKV pair optimization/path.	ds4.c:17562
+runtime/cuda	DS4_CUDA_NO_SCORE_TILE	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the CUDA score tile optimization/path.	ds4_cuda.cu:13734; ds4.c:68122
+runtime/cuda	DS4_CUDA_NO_SETDEVICE_CACHE	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the cached current-tier shortcut and call cudaSetDevice for every tier selection.	ds4_cuda.cu:377
+runtime/cuda	DS4_CUDA_NO_SPLITKV_DECODE	value-aware kill switch, default off; exact 0/empty is off, other nonempty values disable	Disable splitkv decode in CUDA split-KV attention/speculation.	ds4_cuda.cu:2211
+runtime/cuda	DS4_CUDA_NO_SPLITKV_SPEC	value-aware kill switch; default off; nonempty value other than exact 0 disables	Disable splitkv spec in CUDA split-KV attention/speculation.	ds4.c:17253
+runtime/cuda	DS4_CUDA_NO_SPLITKV_SPEC_BATCH_VERIFY	value-aware kill switch; default off; nonempty value other than exact 0 disables	Disable splitkv spec batch verify in CUDA split-KV attention/speculation.	ds4.c:17273
+runtime/cuda	DS4_CUDA_NO_SPLITKV_SPEC_TOPONLY_ROW0	value-aware kill switch; default off; nonempty value other than exact 0 disables	Disable splitkv spec toponly row0 in CUDA split-KV attention/speculation.	ds4.c:17263
+runtime/cuda	DS4_CUDA_NO_STREAMING_EXPERT_PERSISTENT_CACHE	false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate	Disable streaming expert persistent cache in CUDA SSD streaming.	ds4_cuda.cu:4115
+runtime/cuda	DS4_CUDA_NO_STREAMING_SELECTED_BATCH_IO	false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate	Disable streaming selected batch I/O in CUDA SSD streaming.	ds4_cuda.cu:4736
+runtime/cuda	DS4_CUDA_NO_STREAMING_SELECTED_EVENT_PIPELINE	false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate	Disable streaming selected event pipeline in CUDA SSD streaming.	ds4_cuda.cu:4868
+runtime/cuda	DS4_CUDA_NO_TF32	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Use default cuBLAS math instead of TF32 tensor operations.	ds4_cuda.cu:6657
+runtime/cuda	DS4_CUDA_NO_TOP1	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the dedicated CUDA indexer top-1 kernel.	ds4_cuda.cu:375
+runtime/cuda	DS4_CUDA_NO_TOPK1024	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the topk1024 CUDA indexer kernel/path.	ds4_cuda.cu:19290
+runtime/cuda	DS4_CUDA_NO_TOPK2048	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the topk2048 CUDA indexer kernel/path.	ds4_cuda.cu:19297
+runtime/cuda	DS4_CUDA_NO_TOPK2048_WIDE	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the topk2048 wide CUDA indexer kernel/path.	ds4_cuda.cu:19212
+runtime/cuda	DS4_CUDA_NO_TOPK8192	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the topk8192 CUDA indexer kernel/path.	ds4_cuda.cu:19335
+runtime/cuda	DS4_CUDA_NO_TOPK_CHUNKED	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the topk chunked CUDA indexer kernel/path.	ds4_cuda.cu:19374
+runtime/cuda	DS4_CUDA_NO_TOPK_STREAM	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the topk stream CUDA indexer kernel/path.	ds4_cuda.cu:19366
+runtime/cuda	DS4_CUDA_NO_TP_ATTN_OUT_HC_FUSE	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the CUDA TP attn out HC fuse optimization/path.	ds4.c:17565
+runtime/cuda	DS4_CUDA_NO_VERIFY_DECODE2_SPLIT_TOP1	value-aware kill switch; default off; nonempty value other than exact 0 disables	Disable the CUDA verify decode2 split top1 optimization/path.	ds4.c:17223
+runtime/cuda	DS4_CUDA_NO_WARP_ROUTER_SELECT	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the CUDA warp router select optimization/path.	ds4_cuda.cu:24490
+runtime/cuda	DS4_CUDA_NO_WINDOW_ATTENTION	presence kill switch; default unset (eligible path remains available); any defined value including 0 disables	Disable the CUDA window attention optimization/path.	ds4_cuda.cu:23050
+runtime/cuda	DS4_CUDA_NSYS_PREFILL_START_POS	nonempty-string flag, default off; any nonempty value enables MMQ NVTX ranges (the value is not parsed as a position)	Enable MMQ NVTX annotations intended for Nsight Systems prefill capture.	cuda/mmq/ds4_mmq.cu:49
+runtime/cuda	DS4_CUDA_NVTX	strict flag, default off; only exact value 1 enables (a nonempty NSYS variable also enables ranges)	Enable NVTX ranges around MMQ work.	cuda/mmq/ds4_mmq.cu:48
+runtime/cuda	DS4_CUDA_OUTPUT_FUSED_TOP1	value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables	Fuse output projection with top-1 selection in greedy decode.	ds4.c:17215
+runtime/cuda	DS4_CUDA_PREFILL_PIPELINE	boolean, default follows CUDA TP decode; nonempty exact 0 disables, any other nonempty value enables	Control the CUDA multi-tier prefill pipeline.	ds4.c:17454
+runtime/cuda	DS4_CUDA_PREFILL_PIPELINE_MB	positive integer rows; default/invalid 512	Set prefill-pipeline microbatch rows.	ds4.c:17469
+runtime/cuda	DS4_CUDA_PREFILL_PIPELINE_Q8_CACHE	value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables	Keep selective Q8 caches enabled while running the prefill pipeline.	ds4.c:17464
+runtime/cuda	DS4_CUDA_PREFILL_PIPELINE_SEQUENTIAL	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Execute prefill pipeline stages sequentially for diagnosis.	ds4.c:35525
+runtime/cuda	DS4_CUDA_PREFILL_PIPELINE_SYNC_BOUNDARY	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Synchronize CUDA at every prefill pipeline tier boundary.	ds4.c:35572
+runtime/cuda	DS4_CUDA_Q4_ATTN_OUT_HC_ORACLE	value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on	Compare fused Q4 attention-output/HC expansion with the canonical path and retain canonical output.	ds4_cuda.cu:1475
+runtime/cuda	DS4_CUDA_Q4_ATTN_OUT_HC_Q8K_EXPERIMENT	value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on	Enable the experimental Q8_K-based Q4 attention-output/HC fusion.	ds4_cuda.cu:37377
+runtime/cuda	DS4_CUDA_Q4_ATTN_Q_B_F16_CACHE_MB	unsigned integer MiB with a full-string parse; default 3072; 0 prevents admission; invalid text restores the default; overflow saturates	Cap device memory used by resident Q4_K attn_q_b F16 sidecars.	ds4_cuda.cu:2477
+runtime/cuda	DS4_CUDA_Q4_ATTN_Q_B_F16_CACHE_MIN_TOKENS	integer token count; default 512; valid values clamp to 32..UINT32_MAX; empty or invalid text restores 512	Set the minimum prefill batch eligible to prepare or use the CUDA Q4 attn_q_b F16 sidecars.	ds4_cuda.cu:2484
+runtime/cuda	DS4_CUDA_Q4_ATTN_Q_B_TRANSIENT_F16_MIN_TOKENS	full-string unsigned token count; default 4096; empty or invalid text restores 4096; parsed values clamp to 32..UINT32_MAX	Set the minimum physically device-image-resident, single-GPU, non-SSD prefill batch eligible for per-layer transient CUDA Q4_K attn_q_b-to-F16 expansion.	ds4_cuda.cu:2508
+runtime/cuda	DS4_CUDA_Q4_GROUPED_ATTN_A_ORACLE	value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on	Compare grouped attention-A against the canonical per-group result.	ds4_cuda.cu:1477
+runtime/cuda	DS4_CUDA_Q4_K1024_PERSISTENT_ORACLE	value-aware flag, default off; nonempty value other than exact 0 enables and implies candidate admission	Bitwise-compare the exact-shape persistent Q4 K1024 kernel with canonical MMVQ and retain canonical output.	cuda/mmq/ds4_mmq.cu:3738
+runtime/cuda	DS4_CUDA_Q4_K1024_PERSISTENT_STATS	value-aware flag, default off; nonempty value other than exact 0 enables	Print exact-shape persistent Q4 K1024 dispatch counters at exit.	cuda/mmq/ds4_mmq.cu:3737
+runtime/cuda	DS4_CUDA_Q4_MMQ_16WARP	value-aware opt-in cached on the first Q4_K dense or dense-pair MMQ call; unset/empty/exact 0 is off, every other nonempty value requests the candidate; rollback wins; standalone dense requires M>=1024 and admits K<=8192 including attention output-B, while dense-pair admits legs down to M=512, remains bounded to K<=4096, and shares one Q8_1 activation; grids require at least 80% whole-tile SM-wave efficiency and use the canonical Stream-K partition/fixup below its 90% cutoff; ineligible optional shapes fall back	Enable the experimental exact-integer CUDA Q4_K m128n128 16-warp kernel for eligible dense and dense-pair prefills without changing the canonical FP32 reduction tree.	cuda/mmq/ds4_mmq.cu:1126
+runtime/cuda	DS4_CUDA_Q8_F16_ALL	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Control the Q8 F16 all CUDA quantized-matmul/cache optimization.	ds4_cuda.cu:2358
+runtime/cuda	DS4_CUDA_Q8_F16_CACHE_MB	unsigned integer MiB, full-string parse; default unlimited; 0 disables this cache	Limit the selective Q8-to-F16 derived-weight cache.	ds4_cuda.cu:2218
+runtime/cuda	DS4_CUDA_Q8_F16_CACHE_RESERVE_MB	unsigned integer MiB, full-string parse; default is VRAM-dependent (>=112 GiB: 512; >=40 GiB: max(768,1%); smaller: max(4096,5%))	Reserve free VRAM when growing the selective Q8-to-F16 cache.	ds4_cuda.cu:2224
+runtime/cuda	DS4_CUDA_Q8_F32_ALL	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Control the Q8 F32 all CUDA quantized-matmul/cache optimization.	ds4_cuda.cu:2412
+runtime/cuda	DS4_CUDA_Q8_F32_LARGE	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Control the Q8 F32 large CUDA quantized-matmul/cache optimization.	ds4_cuda.cu:2416
+runtime/cuda	DS4_CUDA_Q8_F32_PRELOAD	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Control the Q8 F32 preload CUDA quantized-matmul/cache optimization.	ds4_cuda.cu:8597
+runtime/cuda	DS4_CUDA_Q8_FOLD_ORACLE	strict flag, default off; only exact value 1 enables	Compare folded Q8_1 bytes and consumer outputs against canonical work while retaining canonical results.	cuda/mmq/ds4_mmq.cu:381
+runtime/cuda	DS4_CUDA_Q8_HC_EXPAND_FUSED	false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, other nonempty values force fused	Force the fused Q8 shared-down/HC expansion path.	ds4_cuda.cu:2084
+runtime/cuda	DS4_CUDA_Q8_HC_EXPAND_STATS	false-like-aware flag, default off; 0/false/no/off is off, other nonempty values print report	Print Q8 shared-down/HC policy and dispatch counters at exit.	ds4_cuda.cu:2090
+runtime/cuda	DS4_CUDA_Q8_NO_ALIGNED	value-aware kill switch, default off; nonempty value other than exact 0 disables aligned Q8 kernels	Disable aligned Q8 CUDA matmul kernels.	ds4_cuda.cu:1021
+runtime/cuda	DS4_CUDA_Q8_PAIR_BATCH	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Control the Q8 pair batch CUDA quantized-matmul/cache optimization.	ds4_cuda.cu:20167
+runtime/cuda	DS4_CUDA_QKV_KV_ROPE_FUSE	value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables	Control or tune the CUDA QKV KV rope fuse path.	ds4.c:17429
+runtime/cuda	DS4_CUDA_Q_NORM_ROPE_FUSE	value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables	Control or tune the CUDA q norm rope fuse path.	ds4.c:17418
+runtime/cuda	DS4_CUDA_REQUIRE_IQ2_XXS_SSD_PREFILL_MMQ	false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate	Require the CUDA IQ2 XXS SSD prefill MMQ path; fail closed when unavailable.	ds4_cuda.cu:4631
+runtime/cuda	DS4_CUDA_REQUIRE_Q4_ATTN_Q_B_F16_CACHE	value-aware strict opt-in, default off; unset/empty/0/false/no/off is off, any other nonempty value requires eligible batches to use the cache; DISABLE wins	Fail an eligible CUDA prefill instead of falling back when the resident Q4_K attn_q_b F16 specialization cannot be prepared or dispatched.	ds4_cuda.cu:2492; ds4_cuda.cu:2497
+runtime/cuda	DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_BATCH	value-aware flag, default off; unset/empty/exact 0 is off, any other nonempty value is on	Fail if grouped batched attention-A cannot be used.	ds4_cuda.cu:40198
+runtime/cuda	DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_PREFILL	value-aware fail-closed assertion, default off; unset/empty/exact 0 is off, any other nonempty value requests the candidate and rejects ineligibility before enqueue	Require the GB10 grouped Q4_K attention-A prefill path instead of silently using pack/MMQ/unpack.	ds4_cuda.cu:41889
+runtime/cuda	DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_Q81	value-aware fail-closed assertion, default off; unset/empty/exact 0 is off, any other nonempty value requests grouped prefill and the fixed K=4096, groups=8, rank=1024 Q8_1 producer; NO wins	Require the eight-warp K4096/G8x2 Q8_1 producer instead of silently using the generic strided quantizer.	cuda/mmq/ds4_mmq.cu:1767; ds4_cuda.cu:44089
+runtime/cuda	DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_SINGLE_GRID	value-aware fail-closed assertion, default off; unset/empty/exact 0 is off, every other nonempty value requests the candidate; DISABLE or ineligibility fails before enqueue	Require one grid.z MMQ submission for eligible GB10 grouped Q4_K attention-A prefill instead of falling back to one launch per group.	ds4_cuda.cu:41896
+runtime/cuda	DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT	presence flag, default off; any defined value including 0 makes ineligible candidate fail closed	Fail when the exact Q4 K1024 persistent candidate is unavailable instead of using MMVQ.	cuda/mmq/ds4_mmq.cu:3929
+runtime/cuda	DS4_CUDA_REQUIRE_Q4_MMQ_16WARP	value-aware fail-closed prefill opt-in cached on the first Q4_K dense or dense-pair MMQ call; unset/empty/exact 0 is off, every other nonempty value requests and requires the candidate for N>8; a dense-pair is rejected before allocation unless both legs are eligible; rollback, disabled MMQ, ineligibility, or preflight failure prevents fallback; decode/speculative N<=8 remains on MMVQ	Require the experimental CUDA Q4_K 16-warp prefill kernel so benchmark runs cannot silently measure another path.	cuda/mmq/ds4_mmq.cu:1130; ds4_cuda.cu:38208; ds4_cuda.cu:38358
+runtime/cuda	DS4_CUDA_REQUIRE_STREAMING_EXPERT_PERSISTENT_CACHE	false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate	Require streaming expert persistent cache in CUDA SSD streaming; fail closed when unavailable.	ds4_cuda.cu:4117
+runtime/cuda	DS4_CUDA_REQUIRE_STREAMING_SELECTED_BATCH_IO	false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate	Require streaming selected batch I/O in CUDA SSD streaming; fail closed when unavailable.	ds4_cuda.cu:4738
+runtime/cuda	DS4_CUDA_REQUIRE_STREAMING_SELECTED_EVENT_PIPELINE	false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate	Require streaming selected event pipeline in CUDA SSD streaming; fail closed when unavailable.	ds4_cuda.cu:4870
+runtime/cuda	DS4_CUDA_SERIAL_F16_MATMUL	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Control the serial F16 matmul CUDA F16 matmul path.	ds4_cuda.cu:20801
+runtime/cuda	DS4_CUDA_SERIAL_ROUTER	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Control or tune the CUDA serial router path.	ds4_cuda.cu:20806
+runtime/cuda	DS4_CUDA_SESSION_BATCH_ATTN_ALIAS	value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables	Control the grouped multi-session CUDA attn alias stage.	ds4.c:69902
+runtime/cuda	DS4_CUDA_SESSION_BATCH_ATTN_CORE	value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables	Control the grouped multi-session CUDA attn core stage.	ds4.c:69905
+runtime/cuda	DS4_CUDA_SESSION_BATCH_ATTN_POST	value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables	Control the grouped multi-session CUDA attn post stage.	ds4.c:69917
+runtime/cuda	DS4_CUDA_SESSION_BATCH_ATTN_PRE	value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables	Control the grouped multi-session CUDA attn pre stage.	ds4.c:69898
+runtime/cuda	DS4_CUDA_SESSION_BATCH_FFN_PRE	value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables	Control the grouped multi-session CUDA ffn pre stage.	ds4.c:69894
+runtime/cuda	DS4_CUDA_SESSION_BATCH_INTERLEAVE	boolean, default on; unset/empty/nonzero enables pipeline interleaving; exact 0 disables	Control the grouped multi-session CUDA interleave stage.	ds4.c:70578
+runtime/cuda	DS4_CUDA_SESSION_BATCH_KV_STORE	value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables	Control the grouped multi-session CUDA KV store stage.	ds4.c:69913
+runtime/cuda	DS4_CUDA_SESSION_BATCH_MOE	value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables	Control the grouped multi-session CUDA MoE stage.	ds4.c:69649
+runtime/cuda	DS4_CUDA_SESSION_BATCH_MOE_COMBINE_ROWS	value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables	Control the grouped multi-session CUDA MoE combine rows stage.	ds4.c:69431
+runtime/cuda	DS4_CUDA_SESSION_BATCH_QKV	value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables	Control the grouped multi-session CUDA QKV stage.	ds4.c:69909
+runtime/cuda	DS4_CUDA_SESSION_BATCH_SHARED	value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables	Control the grouped multi-session CUDA shared stage.	ds4.c:69890
+runtime/cuda	DS4_CUDA_SPLITKV_CHUNK	integer scores/chunk; default 512; clamped 1..512	Control splitkv chunk in CUDA split-KV attention/speculation.	ds4_cuda.cu:22568
+runtime/cuda	DS4_CUDA_SPLITKV_DECODE	value-aware boolean, default off; exact 0/empty is off, other nonempty values enable; mere presence also excludes one session-batch path	Enable split-KV decode attention.	ds4_cuda.cu:2213; ds4.c:68111
+runtime/cuda	DS4_CUDA_SPLITKV_GLOBAL_SOFTMAX	value-aware opt-in, default off; exact 0/empty is off, other nonempty values enable	Use the global-softmax variant of split-KV attention.	ds4_cuda.cu:22589
+runtime/cuda	DS4_CUDA_SPLITKV_MIN_SCORE	integer score count 0..UINT32_MAX; default 0 when explicitly enabled, otherwise 512; CUDA kernel clamps to 0..8192	Set the minimum visible-score count for split-KV attention.	ds4_cuda.cu:22557; ds4.c:17402
+runtime/cuda	DS4_CUDA_SPLITKV_S	integer exact split count; unset/invalid = automatic; valid value clamped 1..16	Control splitkv s in CUDA split-KV attention/speculation.	ds4_cuda.cu:22578
+runtime/cuda	DS4_CUDA_SPLITKV_SPEC	value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables	Enable split-KV speculative decoding.	ds4.c:17255
+runtime/cuda	DS4_CUDA_SPLITKV_SPEC_BATCH_VERIFY	value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables	Use batched verification for split-KV speculation.	ds4.c:17275
+runtime/cuda	DS4_CUDA_SPLITKV_SPEC_LOG	presence diagnostic flag; default off; any defined value including 0 enables	Log split-KV speculative-decode admission and fallback decisions.	ds4.c:55812
+runtime/cuda	DS4_CUDA_SPLITKV_SPEC_TIMING	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Print timing for split-KV speculative-decode stages.	ds4.c:55851
+runtime/cuda	DS4_CUDA_SPLITKV_SPEC_TOPONLY_ROW0	value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables	Compute only the top result for row zero in split-KV speculation.	ds4.c:17265
+runtime/cuda	DS4_CUDA_SPLITKV_S_FLOOR	integer split count; default 4; clamped 1..16	Control splitkv s floor in CUDA split-KV attention/speculation.	ds4_cuda.cu:22571
+runtime/cuda	DS4_CUDA_SPLITKV_S_MAX	integer split count; default 16; clamped 1..16	Control splitkv s max in CUDA split-KV attention/speculation.	ds4_cuda.cu:22574
+runtime/cuda	DS4_CUDA_STREAMING_EXPERT_CACHE_PROFILE	presence diagnostic flag; default off; any defined value including 0 enables	Profile CUDA SSD-streaming streaming expert cache.	ds4.c:21849
+runtime/cuda	DS4_CUDA_STREAMING_EXPERT_PERSISTENT_CACHE_ORACLE	false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate	Run the diagnostic oracle for CUDA SSD-streaming streaming expert persistent cache.	ds4_cuda.cu:4121
+runtime/cuda	DS4_CUDA_STREAMING_EXPERT_PERSISTENT_CACHE_STATS	false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate	Print counters for CUDA SSD-streaming streaming expert persistent cache.	ds4_cuda.cu:4119
+runtime/cuda	DS4_CUDA_STREAMING_PREFILL_BATCH_SELECTED_PROFILE	presence diagnostic flag; default off; any defined value including 0 enables	Profile CUDA SSD-streaming streaming prefill batch selected.	ds4.c:21932
+runtime/cuda	DS4_CUDA_STREAMING_SELECTED_BATCH_IO_ORACLE	false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate	Run the diagnostic oracle for CUDA SSD-streaming streaming selected batch I/O.	ds4_cuda.cu:4740
+runtime/cuda	DS4_CUDA_STREAMING_SELECTED_BATCH_IO_PROFILE	false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate	Profile CUDA SSD-streaming streaming selected batch I/O.	ds4_cuda.cu:5711
+runtime/cuda	DS4_CUDA_STREAMING_SELECTED_EVENT_PIPELINE_ORACLE	false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate	Run the diagnostic oracle for CUDA SSD-streaming streaming selected event pipeline.	ds4_cuda.cu:4872
+runtime/cuda	DS4_CUDA_STREAMING_SELECTED_EVENT_PIPELINE_STATS	false-like-aware flag, default off; 0/false/no/off (case-insensitive) is off, any other nonempty value is on; disable flags dominate	Print counters for CUDA SSD-streaming streaming selected event pipeline.	ds4_cuda.cu:4874
+runtime/cuda	DS4_CUDA_STRICT_WEIGHT_CACHE	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Fail a weight lookup when cache allocation fails instead of falling back to mapped model memory.	ds4_cuda.cu:6388
+runtime/cuda	DS4_CUDA_SYNC_XDEV	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Synchronize cross-device CUDA copies for debugging and error localization.	ds4_cuda.cu:363
+runtime/cuda	DS4_CUDA_TP_ATTN	value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables	Control CUDA tensor/expert-parallel attn execution.	ds4.c:17035
+runtime/cuda	DS4_CUDA_TP_ATTN_CACHE_DUP	value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables	Control CUDA tensor/expert-parallel attn cache dup execution.	ds4.c:17059
+runtime/cuda	DS4_CUDA_TP_ATTN_HEADS	value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables	Control CUDA tensor/expert-parallel attn heads execution.	ds4.c:17051
+runtime/cuda	DS4_CUDA_TP_ATTN_OUT_HC_FUSE	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Control CUDA tensor/expert-parallel attn out HC fuse execution.	ds4.c:17564
+runtime/cuda	DS4_CUDA_TP_ATTN_PEER_READ	value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables	Control CUDA tensor/expert-parallel attn peer read execution.	ds4.c:17043
+runtime/cuda	DS4_CUDA_TP_EP_BALANCED_SHARED_MID	value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables	Control CUDA tensor/expert-parallel EP balanced shared mid execution.	ds4.c:17116
+runtime/cuda	DS4_CUDA_TP_EP_DELAY_REDUCE	value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables	Control CUDA tensor/expert-parallel EP delay reduce execution.	ds4.c:17091
+runtime/cuda	DS4_CUDA_TP_EP_DIRECT_RETURN	value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables	Control CUDA tensor/expert-parallel EP direct return execution.	ds4.c:17083
+runtime/cuda	DS4_CUDA_TP_EP_DUAL_PREQUANT	value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables	Control CUDA tensor/expert-parallel EP dual prequant execution.	ds4.c:17125
+runtime/cuda	DS4_CUDA_TP_EP_FUSED_HC_REDUCE	value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables	Control CUDA tensor/expert-parallel EP fused HC reduce execution.	ds4.c:17099
+runtime/cuda	DS4_CUDA_TP_EP_FUSED_SHARED_MID	value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables	Control CUDA tensor/expert-parallel EP fused shared mid execution.	ds4.c:17107
+runtime/cuda	DS4_CUDA_TP_EP_PACK_EXACT	value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables	Control CUDA tensor/expert-parallel EP pack exact execution.	ds4.c:17075
+runtime/cuda	DS4_CUDA_TP_MOE	value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables	Control CUDA tensor/expert-parallel MoE execution.	ds4.c:17067
+runtime/cuda	DS4_CUDA_TP_MOE_COPY3	value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables	Control CUDA tensor/expert-parallel MoE copy3 execution.	ds4.c:17149
+runtime/cuda	DS4_CUDA_TP_MOE_DELAY_REDUCE	value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables	Control CUDA tensor/expert-parallel MoE delay reduce execution.	ds4.c:17133
+runtime/cuda	DS4_CUDA_TP_MOE_PACK	value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables	Control CUDA tensor/expert-parallel MoE pack execution.	ds4.c:17141
+runtime/cuda	DS4_CUDA_TP_MOE_PEER_READ	value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables	Control CUDA tensor/expert-parallel MoE peer read execution.	ds4.c:17157
+runtime/cuda	DS4_CUDA_TP_MOE_PEER_ROUTER	value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables	Control CUDA tensor/expert-parallel MoE peer router execution.	ds4.c:17165
+runtime/cuda	DS4_CUDA_TP_OUTPUT	boolean, default on; empty/unset/nonzero enables, exact 0 disables	Control CUDA tensor/expert-parallel output execution.	ds4.c:51713
+runtime/cuda	DS4_CUDA_TP_OUTPUT_WAYS	integer 2..DS4_MAX_GPUS (16); default 8; invalid value falls back to 2; capped by available GPUs	Set the number of GPU ways used to shard CUDA tensor-parallel output projection.	ds4.c:58; ds4.c:51720
+runtime/cuda	DS4_CUDA_TP_PREFILL_ATTN_OUTPUT	value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables	Control CUDA tensor/expert-parallel prefill attn output execution.	ds4.c:17445
+runtime/cuda	DS4_CUDA_TP_PREFILL_FFN	value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables	Control CUDA tensor/expert-parallel prefill ffn execution.	ds4.c:17437
+runtime/cuda	DS4_CUDA_TP_Q	value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables	Control CUDA tensor/expert-parallel q execution.	ds4.c:17189
+runtime/cuda	DS4_CUDA_TP_SHARED	value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables	Control CUDA tensor/expert-parallel shared execution.	ds4.c:17173
+runtime/cuda	DS4_CUDA_TP_SHARED_FOLD	value-aware boolean; default on; unset/empty uses default, exact 0 disables, any other nonempty value enables	Control CUDA tensor/expert-parallel shared fold execution.	ds4.c:17181
+runtime/cuda	DS4_CUDA_VERIFY_DECODE2_SPLIT_TOP1	value-aware boolean; default off; unset/empty uses default, exact 0 disables, any other nonempty value enables	Enable the split top-1 path for two-row verification decode.	ds4.c:17225
+runtime/cuda	DS4_CUDA_WEIGHT_ARENA_CHUNK_MB	positive integer MiB; default 1792; clamped 256..8192 and raised/aligned when one allocation needs more	Set the CUDA selective-weight arena allocation chunk.	ds4_cuda.cu:6311
+runtime/cuda	DS4_CUDA_WEIGHT_CACHE	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Force selective CUDA weight caching instead of direct mapped access.	ds4_cuda.cu:1246
+runtime/cuda	DS4_CUDA_WEIGHT_CACHE_LIMIT_GB	unsigned integer GiB; default/0 = unlimited; parser accepts a numeric prefix even with trailing text	Limit total CUDA selective-weight cache allocation.	ds4_cuda.cu:6299
+runtime/cuda	DS4_CUDA_WEIGHT_CACHE_VERBOSE	presence diagnostic flag; default off; any defined value including 0 enables	Print CUDA weight mapping, caching, and preload diagnostics.	ds4_cuda.cu:1297
+runtime/cuda	DS4_CUDA_WEIGHT_PRELOAD	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Request proactive CUDA weight caching/preloading.	ds4_cuda.cu:1247
+runtime/cuda	DS4_CUDA_WEIGHT_PRELOAD_SPAN_MB	positive integer MiB; default 1024; clamped 64..4096	Set the maximum span size used by CUDA weight preload.	ds4.c:2880
+runtime/cuda	DS4_CUDA_WINDOW_ATTENTION	presence flag; unset does not force the path; any defined value including 0 is true; eligibility/auto-policy still applies	Control or tune the CUDA window attention path.	ds4_cuda.cu:23051
+runtime/cuda-mmq	DS4_MMID_CASE1	boolean-ish cached flag; default on; a value starting with 0 disables	Disable the single-expert MM-IDs specialized fast path for comparison.	cuda/mmq/mmid.cu:290
+runtime/cuda-mmq	DS4_MMID_LARGE	boolean-ish cached flag; default on; a value starting with 0 disables	Control the large-N global-memory MM-IDs path used beyond shared-memory capacity.	cuda/mmq/mmid.cu:245
+runtime/cuda-mmq	DS4_MMQ_D2R	boolean-ish cached flag; default on; a value starting with 0 disables	Control the direct-to-register Q2_K MoE down path.	cuda/mmq/ds4_mmq.cu:525
+runtime/cuda-mmq	DS4_MMQ_D2R_IQ2	boolean-ish cached flag; default on; a value starting with 0 disables	Control the direct-to-register IQ2 MoE gate/up path.	cuda/mmq/ds4_mmq.cu:534
+runtime/cuda-mmq	DS4_MMQ_D2R_MIN_COLS	positive integer; default 1024; invalid or nonpositive input restores the default	Set the minimum output-column count for the MMQ direct-to-register path.	cuda/mmq/ds4_mmq.cu:629
+runtime/cuda-mmq	DS4_MMQ_D2R_STATS	exact 1 enables; unset or every other value disables; cached and synchronizes the stream	Print partial-tile fill telemetry for the direct-to-register MMQ kernels.	cuda/mmq/ds4_mmq_d2r.cu:33
+runtime/cuda-mmq	DS4_MMQ_DENSE_D2R	boolean-ish flag; default on; exact 0 disables	Control the eligible aligned-Q8 dense prefill direct-to-register path.	ds4_cuda.cu:19567
+runtime/cuda-mmq	DS4_MMQ_NO_YIND	presence rollback; unset keeps Y-indirect staging; any defined value including 0 disables it	Restore slot-gathered MoE gate/up activation quantization.	cuda/mmq/ds4_mmq.cu:609
+runtime/cuda-mmq	DS4_MMQ_OUT_MEMSET	exact 1 enables; unset or every other value disables; cached	Restore blanket MMQ output-buffer zeroing for diagnostics.	cuda/mmq/ds4_mmq.cu:552
+runtime/cuda-mmq	DS4_MMQ_YBUF_MEMSET	unset or 0 disables; 1 zero-fills; a value starting with p or P poison-fills with 0xFF	Control MMQ Q8_1 activation-staging initialization and its poison oracle.	cuda/mmq/ds4_mmq.cu:578
+runtime/cuda-mmq	DS4_MMQ_YIND_VERIFY	presence diagnostic; unset is off; any defined value including 0 enables	Byte-compare Y-indirect and slot-gathered MoE activation buffers.	cuda/mmq/ds4_mmq.cu:620
+runtime/cuda-mmq	DS4_Q8_FOLD_SELFTEST	positive call budget; unset/empty disables; a nonempty value parsing to 1 or less selects 512 calls	Byte-check folded Q8_1 activations against a fresh quantization; synchronizes eager streams.	cuda/mmq/ds4_mmq.cu:5953
+runtime/cuda-shared	DS4_FORCE_CUDA_PEER	presence flag read once at CUDA init; unset uses automatic transfer selection; any defined value including 0 enables	Force cross-device transfers through cudaMemcpyPeerAsync for diagnostics.	ds4_cuda.cu:364
+runtime/cuda-shared	DS4_FORCE_HOST_BOUNCE	presence flag read once at CUDA init; unset uses automatic transfer selection; any defined value including 0 enables	Force cross-device transfers through pinned host bounce buffers for diagnostics.	ds4_cuda.cu:365
+runtime/cuda-tools	DS4_WS_REPACK_HASH	exact 1 enables; unset or every other value disables unless overridden by CLI; cached	Print a per-artifact FNV-1a hash for workspace repack identity checks.	cuda/mmq/ds4_repack.cu:530
+runtime/cuda-tools	DS4_WS_REPACK_THREADS	positive integer; default min(6, hardware threads), capped at 16 and the job count	Set the CPU worker count for CUDA workspace artifact repacking.	cuda/mmq/ds4_repack.cu:539
+runtime/distributed	DS4_DIST_CONNECT_BIND_HOST	non-empty string; unset/empty means no local bind constraint	Bind outgoing distributed connections to a local host/address.	ds4_distributed.c:1332
+runtime/distributed	DS4_DIST_CONNECT_BIND_IF	non-empty string; unset/empty means no local bind constraint	Bind outgoing distributed connections to a network interface.	ds4_distributed.c:1334
+runtime/distributed	DS4_DIST_CONNECT_TRACE	presence flag; unset=off, any set value=on	Emit trace diagnostics for dist connect trace.	ds4_distributed.c:1112
+runtime/distributed	DS4_DIST_DECODE_PROFILE	presence flag; unset=off, any set value=on	Collect timing/profile diagnostics for dist decode profile.	ds4_distributed.c:753
+runtime/distributed	DS4_DIST_DISABLE_PREFILL_ACK_ONLY	presence flag; unset keeps optimized/default behavior, any set value disables it	Disable/roll back dist disable prefill ack only.	ds4_distributed.c:3692
+runtime/distributed	DS4_DIST_DISABLE_PREFILL_PIPELINE	presence flag; unset keeps optimized/default behavior, any set value disables it	Disable/roll back dist disable prefill pipeline.	ds4_distributed.c:3427
+runtime/distributed	DS4_DIST_DISABLE_WORKER_PREFETCH	presence flag; unset keeps optimized/default behavior, any set value disables it	Disable/roll back dist disable worker prefetch.	ds4_distributed.c:7878
+runtime/distributed	DS4_DIST_PREFILL_CHUNK	positive integer; unset/0 uses session prefill capacity; explicit value may not exceed capacity	Set distributed prefill chunk size.	ds4_distributed.c:3455
+runtime/distributed	DS4_DIST_PREFILL_SEND_DEPTH	integer 1..8; default 2; capped to chunk count	Set coordinator prefill sender queue depth.	ds4_distributed.c:470
+runtime/distributed	DS4_DIST_PREFILL_WINDOW	positive integer <=64; auto default remote stages+2 clamped 2..8 and chunk count	Set maximum distributed prefill chunks in flight.	ds4_distributed.c:3485
+runtime/distributed	DS4_DIST_SOCKET_BUFFER_MB	integer 0..512 MiB; default 128; 0 disables socket buffer override	Set TCP send/receive buffer sizes.	ds4_distributed.c:712
+runtime/distributed	DS4_DIST_SOCKET_RECV_TIMEOUT_SEC	Nonempty base-10 integer parsed completely; valid range 1..3600 seconds. Unset, empty, partially parsed, or out-of-range values install no SO_RCVTIMEO at all.	Optionally bound blocking receives on distributed TCP sockets; the default deliberately permits indefinitely idle control connections during separate KV transfers.	ds4_distributed.c:1049
+runtime/distributed	DS4_DIST_SOCKET_TIMEOUT_SEC	Nonempty base-10 integer parsed completely; valid range 1..3600 seconds. Default 60 seconds for unset, empty, partially parsed, or out-of-range values.	Set SO_SNDTIMEO on distributed TCP sockets so blocked coordinator/worker sends eventually fail.	ds4_distributed.c:1032
+runtime/distributed	DS4_DIST_WORKER_FORWARD_WINDOW	integer 1..64; default 4	Set worker forward-results window.	ds4_distributed.c:740
+runtime/distributed	DS4_DIST_WORKER_PREFETCH_DEPTH	integer 1..8; default 2	Set worker input-prefetch queue depth.	ds4_distributed.c:726
+runtime/dspark	DS4_DSPARK_CACHE_RESERVE_GB	integer GiB via atoi; default 4.5 GiB; values 1..32 replace it, all other values fall back; decimal/trailing text is truncated/accepted by atoi	Reserve VRAM on DSpark support-cache tiers before packing support-model tensors.	ds4.c:59770
+runtime/dspark	DS4_DSPARK_DISABLE_FINAL_OUTPUT_ALIAS	presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it	Disable aliasing the final DSpark stage output to the next-stage buffer and use an explicit copy.	ds4.c:33784
+runtime/dspark	DS4_DSPARK_DISABLE_FUSED_CPU_MARKOV_ARGMAX	cached value-aware kill switch; default off; nonempty value other than exact 0 disables; false/off also disable because only 0 is recognized as false	Disable the fused CPU Markov-bias plus argmax implementation.	ds4.c:34487
+runtime/dspark	DS4_DSPARK_DISABLE_REUSE_CONFIDENCE0_MARKOV	cached value-aware kill switch; default off; nonempty value other than exact 0 disables; false/off also disable because only 0 is recognized as false	Disable reuse of the first confidence score during Markov proposal.	ds4.c:34496
+runtime/dspark	DS4_DSPARK_DISABLE_VERIFY_SELECTED_PROFILE	presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it	Override and disable the selected-expert verifier profiler.	ds4.c:36660
+runtime/dspark	DS4_DSPARK_EXEC_TIER	integer tier via atoi; default is placement/TP free-VRAM heuristic; valid 0..n_gpus-1 overrides; invalid numeric range falls back, but nonnumeric text becomes tier 0	Choose the GPU tier that executes and primarily caches the DSpark support model.	ds4.c:59743
+runtime/dspark	DS4_DSPARK_FAKE_ARGMAX_PROPOSAL	nonempty boolean; unset/empty or exact 0: off; every other nonempty value enables, but only while DSpark itself is enabled	If the real DSpark proposer produced no draft, installs a one-token fallback proposal equal to the argmax of the current target logits; debug/test mode also selects the non-fused stage-0 setup path.	ds4.c:64278
+runtime/dspark	DS4_DSPARK_LOW_MEMORY_PREFILL_CHUNK	unsigned integer rows; default 128; 0 disables the low-memory policy; invalid/overflow falls back, numeric prefixes are accepted; only consulted for Metal SSD+DSpark on <=24 GiB hosts without an explicit chunk	Set the automatic low-memory Metal prefill chunk for SSD-streamed DSpark.	ds4.c:60714
+runtime/dspark	DS4_DSPARK_NO_GPU_MARKOV	presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it	Disable GPU Markov bias/argmax and the fully device-resident proposal path.	ds4.c:34711
+runtime/dspark	DS4_DSPARK_NO_MARKOV	cached value-aware kill switch; default off; nonempty value other than exact 0 disables Markov bias; false/off also disable	Disable Markov bias in DSpark proposal generation.	ds4.c:34478
+runtime/dspark	DS4_DSPARK_PROBE	nonempty-string diagnostic; unset/empty is off, any nonempty value including 0 is on	Log DSpark proposal/probe diagnostics.	ds4.c:64275
+runtime/dspark	DS4_DSPARK_PROP_PROFILE	presence flag; unset is off; any defined value, including empty or 0, is on; normal eligibility still applies	Print fine-grained timings for DSpark proposal setup.	ds4.c:32959
+runtime/dspark	DS4_DSPARK_SPEC_LOG	presence flag; unset is off; any defined value, including empty or 0, is on; normal eligibility still applies	Log speculative proposal, verification, acceptance, and fallback decisions.	ds4.c:66596
+runtime/dspark	DS4_DSPARK_SSD_VERIFY_BLOCK_MAX	unsigned integer rows; default/fallback 0 means automatic policy; numeric prefixes accepted; used both as verifier cap and as an exact-2 proposer-policy discriminator	Cap speculative rows verified from SSD and influence exact-2 proposal sizing.	ds4.c:52380
+runtime/dspark	DS4_DSPARK_STAGE_PROFILE	presence flag; unset is off; any defined value, including empty or 0, is on; DS4_DSPARK_STAGE_PROFILE_STAGE must also match	Profile DSpark support stages with command-boundary timings.	ds4.c:33416
+runtime/dspark	DS4_DSPARK_STAGE_PROFILE_STAGE	selector subordinate to DS4_DSPARK_STAGE_PROFILE; unset/empty: match every stage; otherwise strtoul base 10 must consume the whole value, fit uint32_t, and equal the current stage; invalid/out-of-range values match no stage	Restricts DSpark stage-boundary timing output to one stage; it does not enable profiling by itself.	ds4.c:33417
+runtime/dspark	DS4_DSPARK_STATS	value-aware flag; default off; nonempty value other than exact 0 enables; false/off are treated as enabled	Collect and print aggregate DSpark runtime statistics.	ds4.c:61607
+runtime/dspark	DS4_DSPARK_VERIFY_CACHE	presence diagnostic; unset: off; any presence including empty or 0 enables on each support-cache installation	CUDA only: copies every installed nonempty DSpark/support-cache range back to the host, byte-compares it with its source, and logs each mismatch plus a bad-count summary without changing the install result.	ds4_cuda.cu:8257
+runtime/dspark	DS4_DSPARK_VERIFY_HEAD_NO_TP	presence rollback; unset: allow eligible CUDA output tensor parallelism; any presence including empty or 0 removes the TP path from eligibility	CUDA only: forces the DSpark speculative batched vocabulary head away from output-TP for correctness isolation; under CUDA TP+EP the attempt fails instead of using unavailable full output weights.	ds4.c:26641
+runtime/dspark	DS4_DSPARK_VERIFY_NONCAUSAL	presence diagnostic sampled once after the first successfully submitted CUDA noncausal-attention kernel; unset: verify 0 calls; any presence including empty or 0: verify that call and the next 2	CUDA only: synchronizes and reads back Q/KV/output, computes the DSpark noncausal attention CPU reference, and logs max absolute/relative error; it reports only and does not fail the operation.	ds4_cuda.cu:21736
+runtime/dspark	DS4_DSPARK_VERIFY_PROFILE	cached presence diagnostic; unset is off; any defined value including empty/0 profiles only the first eligible verifier invocation	Profile one full DSpark target-verifier invocation layer by layer.	ds4.c:36757
+runtime/dspark	DS4_DSPARK_VERIFY_SELECTED_PROFILE	presence flag; unset is off; any defined value, including empty or 0, enables unless DS4_DSPARK_DISABLE_VERIFY_SELECTED_PROFILE is also present (disable wins)	Profile selected-expert streaming inside the DSpark verifier.	ds4.c:36659
+runtime/dspark	DS4_DSPARK_VERIFY_SPLIT_HEAD	nonempty boolean with inverted default; unset/empty or exact 0: fused head; every other nonempty value: split head	Runs the DSpark suffix verifier output head and top-1 reduction in a separate GPU command section after the layer loop, for timing/correctness isolation; default keeps them fused into the layer command section.	ds4.c:36725
+runtime/dspark	DS4_DSPARK_VERIFY_TOPS_CHECK	presence flag; unset is off; any defined value, including empty or 0, is on; normal eligibility still applies	Read back verifier logits and compare GPU top IDs with CPU argmax.	ds4.c:36873
+runtime/glm	DS4_GLM_ABLATE_COMBINE	presence ablation; unset: exchange the local TP partial with the peer and add both halves; any presence including empty or 0 skips the exchange	Metal two-rank TP timing probe: doubles the local routed-MoE or split-attention partial instead of combining with the peer, deliberately producing invalid output; both ranks must set it or their exchange gates desynchronize.	ds4.c:44181
+runtime/glm	DS4_GLM_ATTN_NO_LORA_VEC2	presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it	Disable vectorized two-row LoRA accumulation in CUDA GLM indexed attention.	ds4_cuda.cu:34176
+runtime/glm	DS4_GLM_ATTN_NO_SCORE_VEC2	presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it	Disable vectorized two-row score computation in CUDA GLM indexed attention.	ds4_cuda.cu:34165
+runtime/glm	DS4_GLM_ATTN_NO_STAGED_DECODE	presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it	Disable staged CUDA GLM indexed-decode attention for large selected sets.	ds4_cuda.cu:34214
+runtime/glm	DS4_GLM_DECODE_ABLATE	cached substring list; default empty mask; recognized tokens are attn_out, attn_core, qpath, indexer, routed, shared, qklow; unknown text has no effect; matching stages are skipped and output is invalid	Skip selected GLM decode stages for timing attribution; generated output is invalid.	ds4.c:44411
+runtime/glm	DS4_GLM_DECODE_FLUSH_INTERVAL	integer layers via atoi; default 4 for indexed decode and 32 otherwise; <=0/nonnumeric disables periodic flush; capped to layer count and forced to 0 for deferred completion	Set how often non-streaming GLM decode command work is flushed between layers.	ds4.c:49818
+runtime/glm	DS4_GLM_DISABLE_FLASH_PREFILL	presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it	Disable GLM Flash Attention prefill.	ds4.c:45087
+runtime/glm	DS4_GLM_DISABLE_STREAMING_TOKEN_PREFILL	presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it; either backend-specific ROCm/Metal alias also disables	Disable token-major GLM SSD-streaming prefill.	ds4.c:49702
+runtime/glm	DS4_GLM_FENCE_TRACE	presence flag; unset is off; any defined value, including empty or 0, is on; normal eligibility still applies	Log fenced CUDA tier switches used by GLM multi-GPU execution.	ds4_cuda.cu:7984
+runtime/glm	DS4_GLM_GEMM_TRACE	presence flag; unset is off; any defined value, including empty or 0, is on; normal eligibility still applies	Measure and print CUDA GLM dequantization and cuBLAS GEMM timings.	ds4_cuda.cu:19706
+runtime/glm	DS4_GLM_HIDDEN_DUMP	nonempty filesystem path/prefix; unset/empty disables; writes final hidden row or per-layer files selected by DS4_GLM_HIDDEN_DUMP_LAYER	Dump GLM hidden-state rows for correctness isolation.	ds4.c:38523
+runtime/glm	DS4_GLM_HIDDEN_DUMP_LAYER	selector; unset/empty = -1 (no per-layer dump, final hidden still dumped when path set); all = every layer; otherwise atoi result selects a layer, so invalid text selects layer 0	Choose which GLM layer hidden states are dumped.	ds4.c:38543
+runtime/glm	DS4_GLM_KV_DUMP	nonempty filesystem prefix; unset/empty disables; writes layer-0 lora and rope compact-cache files after sync	Dump layer-0 compact GLM KV cache data after prompt synchronization.	ds4.c:63131
+runtime/glm	DS4_GLM_LOGIT_DUMP	nonempty filesystem path; unset/empty disables; dumps the first post-prefill logits vector once per process	Dump the first post-prefill GLM logits vector.	ds4.c:38601
+runtime/glm	DS4_GLM_MEMORY_GUARD	guard is on by default; exact 0 or case-insensitive false/off/no disables it; any other value and unset keep it enabled	Control the pre-allocation GLM host/GPU memory safety guard.	ds4.c:41989
+runtime/glm	DS4_GLM_MEMORY_GUARD_FRACTION	floating-point fraction; default 0.99; parsed numeric prefix is accepted, invalid/nonfinite falls back, values clamp to 0.50..1.00	Set the fraction of detected memory usable by the GLM memory guard.	ds4.c:42028
+runtime/glm	DS4_GLM_MEMORY_GUARD_REPORT	nonempty diagnostic flag; unset/empty is off; any nonempty value including 0 prints successful-admission accounting (refusals always report)	Print successful GLM memory-guard budget accounting.	ds4.c:42068
+runtime/glm	DS4_GLM_MEMORY_GUARD_RESERVE_GB	floating-point GiB; dynamic default (normally 32, 24 on near-full 480..640 GiB hosts, possibly lower for resident ROCm slices); numeric prefixes accepted; invalid falls back; clamp 0..1024	Set fixed headroom subtracted by the GLM memory guard.	ds4.c:42046
+runtime/glm	DS4_GLM_MOE_EXPERT_MAJOR	presence selector; unset: off; any presence including empty or 0 requests the path only for n_tokens >= 16; the automatic tile-8 path takes precedence when enabled (normally n_tokens >= 128)	CUDA only: groups selected token/expert pairs by expert and uses expert-major Q2_K routed-MoE gate/up/down kernels to reuse expert weights; otherwise the normal token-major path is used.	ds4_cuda.cu:35973
+runtime/glm	DS4_GLM_MOE_NO_DOWN_TILE8_EXACT	presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it	Disable the exact tile-8 CUDA GLM routed-MoE down projection.	ds4_cuda.cu:36028
+runtime/glm	DS4_GLM_MOE_NO_EXPERT_TILE8	presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it	Disable automatic expert tile-8 CUDA GLM routed-MoE batching.	ds4_cuda.cu:35971
+runtime/glm	DS4_GLM_MOE_NO_LOCAL_BATCH_IO	presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it	Disable device-local batch scratch I/O for large CUDA GLM MoE batches.	ds4_cuda.cu:35942
+runtime/glm	DS4_GLM_MOE_SCALAR	presence rollback; unset: use optimized warp kernels; any presence including empty or 0 selects scalar baseline kernels where an earlier expert-tile/expert-major path does not return; the special two-token MTP gate/up kernel still takes precedence	CUDA only: forces the baseline scalar Q2_K routed-MoE gate/up and down implementations for A/B or correctness testing (for two-token MTP, only the down half is forced).	ds4_cuda.cu:36154
+runtime/glm	DS4_GLM_MOE_SCRATCH_TIER0	presence placement override; unset: allocate xq/midq quantization scratch on the current logical tier; any presence including empty or 0 allocates it on logical tier 0	CUDA only: pins the routed-MoE xq_scratch and midq_scratch allocations to GPU tier 0 for multi-tier placement experiments; other MoE scratch remains on the current tier.	ds4_cuda.cu:35918
+runtime/glm	DS4_GLM_MTP_NO_ATTN_TOK2	presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it	Disable the exact two-token CUDA GLM MTP attention kernel.	ds4_cuda.cu:34168
+runtime/glm	DS4_GLM_MTP_NO_MOE_TOK2	presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it	Disable the exact two-token CUDA GLM MTP routed-MoE kernel.	ds4_cuda.cu:36140
+runtime/glm	DS4_GLM_MTP_NO_SHARED_TOK2	presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it	Disable the exact two-token CUDA GLM MTP shared-FFN kernel.	ds4_cuda.cu:37940
+runtime/glm	DS4_GLM_MTP_PROBE	presence flag; unset is off; any defined value, including empty or 0, is on; its second call site also rejects the normal batching path	Run the GLM next-N/MTP acceptance quality-and-timing probe without changing output and force probe-compatible scheduling.	ds4.c:64959
+runtime/glm	DS4_GLM_PREFILL_TRUNC	nonempty value parsed by atoi (leading whitespace/sign accepted and trailing junk ignored); effective only when the resulting int is > 0 and < the current prompt length; unset/empty or a result <= 0 or >= prompt length leaves the prompt unchanged	GPU GLM debug hook: truncates the prompt before prefill/checkpoint handling so dumped prefill logits can be aligned with a CPU first-token reference.	ds4.c:63270
+runtime/glm	DS4_GLM_RESUME_PREFILL_MIN	integer suffix tokens, non-ROCm builds only; default 4; parsed numeric prefixes accepted; <=0 maps to UINT32_MAX and effectively disables batched resume; ROCm build ignores it and stays at 4	Set the suffix-length crossover from token decode to batched resumed prefill.	ds4.c:38434
+runtime/glm	DS4_GLM_ROUTER_SCALAR	presence rollback; unset: use the 256-thread parallel router when n_expert <= 256 (the scalar path is already automatic above 256); any presence including empty or 0 forces the scalar path	CUDA only: selects the one-active-thread-per-token sigmoid/top-k router kernel instead of the parallel shared-memory reduction, for A/B or correctness testing.	ds4_cuda.cu:36390
+runtime/glm	DS4_GLM_SHARED_SPLIT	presence rollback; unset: use the fused one-token shared-expert Q8_0 gate+up+SwiGLU kernel when its shape/buffers are eligible; any presence including empty or 0 skips that fused one-token path; the earlier two-token MTP-specialized path is unaffected	CUDA only: forces shared-expert gate and up through two separate Q8_0 matmuls followed by a separate SwiGLU operation for one-token decode.	ds4_cuda.cu:37966
+runtime/glm	DS4_GLM_STREAMING_DECODE_FULL_LAYER_MAP	presence compatibility alias; unset: automatic mapping; any presence including empty or 0 independently forces full-layer mapping, equivalent to the backend-specific DS4_ROCM_GLM_STREAMING_DECODE_FULL_LAYER_MAP or DS4_METAL_GLM_STREAMING_DECODE_FULL_LAYER_MAP control	Backend-neutral alias for supported GLM SSD streaming (Metal/ROCm): maps every tensor in each decode layer instead of using the decode-only map that can omit routed experts served by the expert cache; layers that already require a full map are unchanged.	ds4.c:42617
+runtime/glm	DS4_GLM_STREAMING_DECODE_SYNC_EACH_LAYER	ROCm-only third-priority legacy value: a nonempty DS4_ROCM_GLM_STREAMING_DECODE_SYNC_EACH_LAYER wins, otherwise a nonempty DS4_METAL_GLM_STREAMING_DECODE_SYNC_EACH_LAYER wins, otherwise this name is read; nonempty values are true except exact 0 or case-insensitive false/off/no; unset/empty: false; non-ROCm builds always return true and ignore this name	On ROCm non-static GLM SSD decode, opts into ending/synchronizing commands after token mapping and after every layer; the default keeps ordered work alive across layer mappings. Static-map decode bypasses this control.	ds4.c:49781
+runtime/glm	DS4_GLM_STREAMING_PREFILL_SYNC_EACH_LAYER	ROCm-only third-priority legacy value: a nonempty DS4_ROCM_GLM_STREAMING_PREFILL_SYNC_EACH_LAYER wins, otherwise a nonempty DS4_METAL_GLM_STREAMING_PREFILL_SYNC_EACH_LAYER wins, otherwise this name is read; nonempty values are true except exact 0 or case-insensitive false/off/no; unset/empty: false for compact prefill; full-layer prefill and non-ROCm builds always synchronize and ignore this name	On ROCm compact GLM SSD prefill, opts into ending/synchronizing commands at every layer boundary; the default carries ordered work across mappings, while full-layer expert-cache prefill always retains the boundary.	ds4.c:42397
+runtime/glm	DS4_GLM_STREAMING_TOKEN_PREFILL_MAX	unsigned token limit; backend-specific ROCm/Metal variable takes precedence, generic is fallback; default 0 on ROCm and 64 otherwise; invalid/overflow falls back, numeric prefixes accepted; 0 disables token-major streaming prefill	Set the largest SSD-streaming prefill handled by the token-major decode-like path.	ds4.c:49684
+runtime/glm	DS4_GLM_SYNC_TRACE	presence flag; unset is off; any defined value, including empty or 0, is on; normal eligibility still applies	Log GLM checkpoint/resume and dense-versus-indexed prefill decisions.	ds4.c:63368
+runtime/glm	DS4_GLM_TP_DEBUG	presence flag; unset is off; any defined value, including empty or 0, is on; normal eligibility still applies	Print CUDA/GLM tensor-parallel dispatch, gate, selected-ID, and failure diagnostics.	ds4.c:44083
+runtime/glm	DS4_GLM_TP_EXACT_PREFILL_MAX	integer suffix limit via atoi cast to uint32; default 64; nonnumeric becomes 0; negative values wrap to a very large unsigned limit	Set the maximum two-way TP suffix that uses exact token-by-token prefill.	ds4.c:63310
+runtime/glm	DS4_GLM_TP_HEAD_SPLIT_MIN	cached integer token threshold via atoi; default 64; negative values clamp to 0, nonnumeric becomes 0; 0 admits all otherwise-eligible batches	Set the minimum batch size for GLM tensor-parallel output-head splitting.	ds4.c:38513
+runtime/glm	DS4_GLM_VALUE_NO_TILE16	presence kill switch; unset leaves the eligible path available; any defined value, including empty or 0, disables it	Disable the CUDA GLM 16-token tiled value-projection kernel.	ds4_cuda.cu:36806
+runtime/metal	DS4_METAL_ARGSORT_SOURCE	file path; unset/empty: use the in-tree Metal source file	Overrides the argsort Metal kernel source file loaded at runtime.	ds4_metal.m:4943
+runtime/metal	DS4_METAL_ATTN_OUT_STAGE_PROFILE	presence diagnostic; unset: off; any value including 0 enables	Prints timing/profile diagnostics for attn out stage.	ds4.c:65151
+runtime/metal	DS4_METAL_BIN_SOURCE	file path; unset/empty: use the in-tree Metal source file	Overrides the binary operations Metal kernel source file loaded at runtime.	ds4_metal.m:4952
+runtime/metal	DS4_METAL_COMPRESSOR_PAIR_NR4	presence control; unset: off/default; any value including 0 enables	Selects the NR4 compressor-pair variant.	ds4_metal.m:2651
+runtime/metal	DS4_METAL_CONCAT_SOURCE	file path; unset/empty: use the in-tree Metal source file	Overrides the concatenation Metal kernel source file loaded at runtime.	ds4_metal.m:4945
+runtime/metal	DS4_METAL_CPY_SOURCE	file path; unset/empty: use the in-tree Metal source file	Overrides the copy Metal kernel source file loaded at runtime.	ds4_metal.m:4944
+runtime/metal	DS4_METAL_DECODE_INDEXER_SPARSE_THRESHOLD	integer in {64,128,256,512,1024,2048,4096}; default 1024; invalid restores default	Sets the compressed-row crossover from dense to sparse indexed attention.	ds4.c:20352
+runtime/metal	DS4_METAL_DECODE_STAGE_PROFILE	unset: off; 1/true/yes/on/all enables all layers; a layer index selects one; 0/false/no/off disables	Prints timing/profile diagnostics for decode stage.	ds4.c:17568
+runtime/metal	DS4_METAL_DECODE_STAGE_PROFILE_LAYER	single unsigned layer index; unset/empty: all layers enabled by the parent profile; invalid matches no layer	Restricts the corresponding shared graph stage profiler to one layer.	ds4.c:29120
+runtime/metal	DS4_METAL_DENSE_SOURCE	file path; unset/empty: use the in-tree Metal source file	Overrides the dense matmul Metal kernel source file loaded at runtime.	ds4_metal.m:4936
+runtime/metal	DS4_METAL_DISABLE_AFFINE_ROPE_PAIR	presence rollback; unset: automatic/default path; any value including 0 disables	Disables affine RoPE pair.	ds4_metal.m:25171
+runtime/metal	DS4_METAL_DISABLE_ATTN_OUT_HC_FUSION	nonempty boolean; unset/empty or exact 0: off; every other value: on	Disables attn out HC fusion.	ds4.c:20477
+runtime/metal	DS4_METAL_DISABLE_ATTN_OUT_IDS_CACHE	presence rollback; unset: automatic/default path; any value including 0 disables	Disables attn out ids cache.	ds4_metal.m:27771
+runtime/metal	DS4_METAL_DISABLE_ATTN_OUT_LOW_DIRECT	presence rollback; unset: automatic/default path; any value including 0 disables	Disables attn out low direct.	ds4_metal.m:27760
+runtime/metal	DS4_METAL_DISABLE_BATCH_HC_NORM_FUSION	nonempty value other than exact 0 disables; unset/empty/0 leaves the default enabled path	Dominant rollback for batched HC norm fusion.	ds4.c:20457
+runtime/metal	DS4_METAL_DISABLE_COMPRESSOR_APE_ADD	presence rollback; unset: automatic/default path; any value including 0 disables	Disables compressor APE add.	ds4_metal.m:25393
+runtime/metal	DS4_METAL_DISABLE_COMPRESSOR_EXACT_POOL_RATIO4	presence rollback; unset: automatic/default path; any value including 0 disables	Disables compressor exact pool ratio4.	ds4_metal.m:26379
+runtime/metal	DS4_METAL_DISABLE_COMPRESSOR_PAIR_PROJ	nonempty boolean; unset/empty or exact 0: off; every other value: on	Disables compressor pair proj.	ds4_metal.m:23248
+runtime/metal	DS4_METAL_DISABLE_COMPRESSOR_QUAD_STORE	presence rollback; unset: automatic/default path; any value including 0 disables	Disables compressor quad store.	ds4.c:23534
+runtime/metal	DS4_METAL_DISABLE_COMPRESSOR_RATIO4_DIRECT_POOL	presence rollback; unset: automatic/default path; any value including 0 disables	Disables compressor ratio4 direct pool.	ds4_metal.m:26245
+runtime/metal	DS4_METAL_DISABLE_COMPRESSOR_RATIO4_PACK_FUSION	presence rollback; unset: automatic/default path; any value including 0 disables	Disables compressor ratio4 pack fusion.	ds4_metal.m:26199
+runtime/metal	DS4_METAL_DISABLE_COMPRESSOR_STORE_ONE	presence rollback; unset: automatic/default path; any value including 0 disables	Disables compressor store one.	ds4_metal.m:23249
+runtime/metal	DS4_METAL_DISABLE_CONTIG_F16_F16_COPY	value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables	Disables contig F16 F16 copy.	ds4_metal.m:29562
+runtime/metal	DS4_METAL_DISABLE_CONTIG_F32_F16_COPY	value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables	Disables contig F32 F16 copy.	ds4_metal.m:29338
+runtime/metal	DS4_METAL_DISABLE_DECODE_ARGMAX_TOP1	presence rollback; unset uses the dedicated two-dispatch top-1 reduction for eligible large Metal rows; any defined value including empty or 0 restores the generic full argsort	Restore the generic indexer argsort for decode argmax A/B and emergency rollback.	ds4_metal.m:21517
+runtime/metal	DS4_METAL_DISABLE_DECODE_NORM_EXACT_VIEWS	presence rollback; unset: automatic/default path; any value including 0 disables	Disables decode norm exact views.	ds4_metal.m:36207
+runtime/metal	DS4_METAL_DISABLE_DECODE_RAW_GATHERED_ATTN	presence rollback; unset: raw-only decode uses gathered attention; any value including 0 restores the legacy raw path	Restores the separate raw-only attention path instead of gathered staging and attention.	ds4_metal.m:32968
+runtime/metal	DS4_METAL_DISABLE_DECODE_RAW_PACKED32	presence rollback; unset: raw-only gathered attention may use packed32; any value including 0 disables it for raw-only layers	Disables the packed32 reduce kernel for raw-only gathered attention while leaving compressed layers unchanged.	ds4_metal.m:31464
+runtime/metal	DS4_METAL_DISABLE_DECODE_ROUTER_BIAS_EXACT_VIEWS	presence rollback; unset: automatic/default path; any value including 0 disables	Disables decode router bias exact views.	ds4_metal.m:39397
+runtime/metal	DS4_METAL_DISABLE_DSPARK_CAPTURE_FUSED_LAST	nonempty boolean; unset/empty or exact 0: off; every other value: on	Disables DSpark capture fused last.	ds4.c:28361
+runtime/metal	DS4_METAL_DISABLE_DSPARK_EXACTN_BATCH_HEAD	nonempty boolean; unset/empty or exact 0: off; every other value: on	Disables DSpark exactn batch head.	ds4.c:37728
+runtime/metal	DS4_METAL_DISABLE_EXACT_ROWS_PERSISTENT_CACHE	nonempty boolean; unset/empty or exact 0: off; every other value: on	Disables exact rows persistent cache.	ds4_metal.m:13000
+runtime/metal	DS4_METAL_DISABLE_GATHERED_KV_PAD_FUSION	presence rollback; unset: automatic/default path; any value including 0 disables	Disables gathered KV pad fusion.	ds4_metal.m:29706
+runtime/metal	DS4_METAL_DISABLE_GATHERED_KV_STAGE	presence rollback; unset: automatic/default path; any value including 0 disables	Disables gathered KV stage.	ds4_metal.m:29677
+runtime/metal	DS4_METAL_DISABLE_GLM_DECODE_KV_GROUP4	value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables	Disables GLM decode KV group4.	ds4_metal.m:36742
+runtime/metal	DS4_METAL_DISABLE_GLM_QKLOW_SG	presence rollback; unset: automatic/default path; any value including 0 disables	Disables GLM qklow sg.	ds4_metal.m:37865
+runtime/metal	DS4_METAL_DISABLE_GLM_STREAMING_EXPERT_EARLY_LOAD	presence rollback; unset: automatic/default path; any value including 0 disables	Disables GLM streaming expert early load.	ds4_metal.m:18058
+runtime/metal	DS4_METAL_DISABLE_GLM_STREAMING_EXPERT_SPLIT	presence rollback; unset: automatic/default path; any value including 0 disables	Disables GLM streaming expert split.	ds4_metal.m:39796
+runtime/metal	DS4_METAL_DISABLE_GLM_STREAMING_PREFILL_FULL_LAYER	presence rollback; unset: automatic/default path; any value including 0 disables	Disables GLM streaming prefill full layer.	ds4.c:42710
+runtime/metal	DS4_METAL_DISABLE_GLM_STREAMING_PREFILL_FULL_LAYER_PREPARE	presence rollback; unset: automatic/default path; any value including 0 disables	Disables GLM streaming prefill full layer prepare.	ds4.c:42728
+runtime/metal	DS4_METAL_DISABLE_GLM_STREAMING_PREFILL_SELECTED_ASYNC_LOAD	presence rollback; unset: automatic/default path; any value including 0 disables	Disables GLM streaming prefill selected async load.	ds4.c:46421
+runtime/metal	DS4_METAL_DISABLE_GLM_STREAMING_SELECTED_ASYNC_LOAD	presence rollback; unset: automatic/default path; any value including 0 disables	Disables GLM streaming selected async load.	ds4.c:44329
+runtime/metal	DS4_METAL_DISABLE_HC_FUSION	nonempty boolean; unset/empty or exact 0: off; every other value: on	Disables HC fusion.	ds4.c:20406
+runtime/metal	DS4_METAL_DISABLE_HC_NORM_FUSION	nonempty boolean; unset/empty or exact 0: off; every other value: on	Disables HC norm fusion.	ds4.c:20450
+runtime/metal	DS4_METAL_DISABLE_HC_PRODUCER_PRE_NORM_FUSE	presence rollback; unset: automatic/default path; any value including 0 disables	Disables HC producer pre norm fuse.	ds4_metal.m:46541
+runtime/metal	DS4_METAL_DISABLE_HC_RMS_SCALE_PROJ	presence rollback; unset: automatic/default path; any value including 0 disables	Disables HC RMS scale proj.	ds4_metal.m:24148
+runtime/metal	DS4_METAL_DISABLE_HOT_PIPELINE_STATICS	presence rollback; unset: automatic/default path; any value including 0 disables	Disables hot pipeline statics.	ds4_metal.m:2634
+runtime/metal	DS4_METAL_DISABLE_INPLACE_ROPE_PAIR	presence rollback; unset: automatic/default path; any value including 0 disables	Disables inplace RoPE pair.	ds4_metal.m:25170
+runtime/metal	DS4_METAL_DISABLE_IQ2_SELECTED_EXPERT_VIEWS	presence rollback; unset: automatic/default path; any value including 0 disables	Disables IQ2 selected expert views.	ds4.c:21080
+runtime/metal	DS4_METAL_DISABLE_IQ2_SELECTED_SHARED_OVERLAP	presence rollback; unset: automatic/default path; any value including 0 disables	Disables IQ2 selected shared overlap.	ds4.c:20996
+runtime/metal	DS4_METAL_DISABLE_IQ2_STREAM_ADDR_TABLE	presence rollback; unset: automatic/default path; any value including 0 disables	Disables IQ2 stream address table.	ds4_metal.m:42402
+runtime/metal	DS4_METAL_DISABLE_IQ2_XXS_SSD_PREFILL_MM	value-aware boolean; default off; true disables and dominates ENABLE; false leaves automatic policy	Rolls grouped IQ2_XXS/Q2_K SSD-prefill MM back to sparse matvec.	ds4_metal.m:44784
+runtime/metal	DS4_METAL_DISABLE_KV_FUSION	nonempty boolean; unset/empty or exact 0: off; every other value: on	Disables KV fusion.	ds4.c:20430
+runtime/metal	DS4_METAL_DISABLE_M1_IQ2_MID_ONLY	presence rollback; unset: automatic/default path; any value including 0 disables	Disables M1 IQ2 mid only.	ds4_metal.m:14592
+runtime/metal	DS4_METAL_DISABLE_M3_COMPRESSOR_EXACT_POOL_RATIO4	presence rollback; unset: automatic/default path; any value including 0 disables	Disables M3 compressor exact pool ratio4.	ds4_metal.m:26381
+runtime/metal	DS4_METAL_DISABLE_M3_COMPRESSOR_PAIR_STATE_STORE	presence rollback; unset: automatic/default path; any value including 0 disables	Disables M3 compressor pair state store.	ds4_metal.m:23247
+runtime/metal	DS4_METAL_DISABLE_M3_GATHERED_KV_STAGE	presence rollback; unset: automatic/default path; any value including 0 disables	Disables M3 gathered KV stage.	ds4_metal.m:29678
+runtime/metal	DS4_METAL_DISABLE_M5_COMPRESSOR_EXACT_POOL_RATIO4	presence rollback; unset: automatic/default path; any value including 0 disables	Disables M5 compressor exact pool ratio4.	ds4_metal.m:26384
+runtime/metal	DS4_METAL_DISABLE_M5_COMP_FINALIZE_FUSE	presence rollback; unset: automatic/default path; any value including 0 disables	Disables M5 comp finalize fuse.	ds4.c:23647
+runtime/metal	DS4_METAL_DISABLE_M5_FLASH_ATTN_PACKED32_REDUCE	presence rollback; unset: automatic/default path; any value including 0 disables	Disables M5 flash attn packed32 reduce.	ds4_metal.m:31519
+runtime/metal	DS4_METAL_DISABLE_M5_HC_NORM_MIX_CLUSTER2	presence rollback; unset: automatic/default path; any value including 0 disables	Disables M5 HC norm mix cluster2.	ds4_metal.m:46496
+runtime/metal	DS4_METAL_DISABLE_M5_HC_PRODUCER_PRE_NORM_FUSE	presence rollback; unset: automatic/default path; any value including 0 disables	Disables M5 HC producer pre norm fuse.	ds4_metal.m:46548
+runtime/metal	DS4_METAL_DISABLE_M5_IQ2_PAIR_PACK2	presence rollback; unset: automatic/default path; any value including 0 disables	Disables M5 IQ2 pair pack2.	ds4_metal.m:42020
+runtime/metal	DS4_METAL_DISABLE_M5_PACKED_ZERO_MASK	presence rollback; unset: automatic/default path; any value including 0 disables	Disables M5 packed zero mask.	ds4_metal.m:31475
+runtime/metal	DS4_METAL_DISABLE_M5_PARALLEL_FULL_FFN	presence rollback; unset: automatic/default path; any value including 0 disables	Disables M5 parallel full FFN.	ds4.c:22707
+runtime/metal	DS4_METAL_DISABLE_M5_PERSISTENT_ZERO_ATTN_MASK	presence rollback; unset: automatic/default path; any value including 0 disables	Disables M5 persistent zero attn mask.	ds4_metal.m:31473
+runtime/metal	DS4_METAL_DISABLE_M5_Q8_HC_VEC	presence rollback; unset: automatic/default path; any value including 0 disables	Disables M5 Q8 HC vec.	ds4_metal.m:47580
+runtime/metal	DS4_METAL_DISABLE_M5_QKV_PAIR_COMPRESSOR_FUSE	presence rollback; unset: automatic/default path; any value including 0 disables	Disables M5 QKV pair compressor fuse.	ds4.c:23043
+runtime/metal	DS4_METAL_DISABLE_M5_QKV_PAIR_QUAD_FUSE	presence rollback; unset: automatic/default path; any value including 0 disables	Disables M5 QKV pair quad fuse.	ds4.c:23039
+runtime/metal	DS4_METAL_DISABLE_M5_ROUTER_PROJECT_SELECT_FUSE	presence rollback; unset: automatic/default path; any value including 0 disables	Disables M5 router project select fuse.	ds4.c:24762
+runtime/metal	DS4_METAL_DISABLE_METAL4	value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables	Disables metal4.	ds4_metal.m:2995
+runtime/metal	DS4_METAL_DISABLE_MOE_MM_ID_PAIR_SWIGLU	presence rollback; unset: automatic/default path; any value including 0 disables	Disables MoE MM ID pair SwiGLU.	ds4_metal.m:44968
+runtime/metal	DS4_METAL_DISABLE_MOE_MM_ID_USE_RESOURCES	presence rollback; unset: automatic/default path; any value including 0 disables	Disables MoE MM ID use resources.	ds4_metal.m:35169
+runtime/metal	DS4_METAL_DISABLE_MXFP4_SELECTED_EXPERT_VIEWS	presence rollback; unset: automatic/default path; any value including 0 disables	Disables MXFP4 selected expert views.	ds4.c:21173
+runtime/metal	DS4_METAL_DISABLE_PERSISTENT_ZERO_ATTN_MASK	presence rollback; unset: automatic/default path; any value including 0 disables	Disables persistent zero attn mask.	ds4_metal.m:31480
+runtime/metal	DS4_METAL_DISABLE_PREFILL_FLUSH_PROGRESS	presence rollback; unset commits eligible resident 32..2048-token per-layer progress batches without host drains; any value including 0 disables	Restore synchronous per-layer drains when display progress is active.	ds4.c:35924
+runtime/metal	DS4_METAL_DISABLE_PRE_M5_ATTN_INV_ROPE_FUSE	presence rollback; unset: automatic/default path; any value including 0 disables	Disables pre M5 attn inv RoPE fuse.	ds4.c:22653
+runtime/metal	DS4_METAL_DISABLE_PRE_M5_ATTN_OUT_LOW_Q8_STATIC	presence rollback; unset: exact fixed-shape kernel is automatic on eligible pre-M5 Flash decode; any value including 0 disables	Restores the generic Q8 attention-output low projection kernel.	ds4_metal.m:27987
+runtime/metal	DS4_METAL_DISABLE_PRE_M5_BATCH_ATTN_OUT_HC_FUSION	presence rollback; unset enables the exact resident pre-M5 Q8_0 or Q4_K output-B-to-HC4 tail when all shape and safety gates pass; any value including 0 disables	Restore the separate attention output-B materialization and HC expansion dispatches.	ds4_metal.m:30581; ds4_metal.m:31504
+runtime/metal	DS4_METAL_DISABLE_PRE_M5_BATCH_INDEXER_QUERY_PRUNE	presence rollback; unset: unused zero-prefix indexer queries are pruned before compressed rows exceed top-k; any value including 0 disables	Restores transient indexer query and weight dispatches during eligible pre-M5 prefill.	ds4.c:30101
+runtime/metal	DS4_METAL_DISABLE_PRE_M5_COMPRESSOR_EXACT_POOL_RATIO4	presence rollback; unset: automatic/default path; any value including 0 disables	Disables pre M5 compressor exact pool ratio4.	ds4_metal.m:26383
+runtime/metal	DS4_METAL_DISABLE_PRE_M5_COMPRESSOR_EXACT_REDUCTION_FUSION	presence rollback; unset: automatic/default path; any value including 0 disables	Disables pre M5 compressor exact reduction fusion.	ds4_metal.m:25907
+runtime/metal	DS4_METAL_DISABLE_PRE_M5_COMPRESSOR_QUAD_STORE	presence rollback; unset: automatic/default path; any value including 0 disables	Disables pre M5 compressor quad store.	ds4.c:23535
+runtime/metal	DS4_METAL_DISABLE_PRE_M5_COMPRESSOR_RATIO4_DECODE_PACK_FUSION	presence rollback; unset: automatic/default path; any value including 0 disables	Disables pre M5 compressor ratio4 decode pack fusion.	ds4_metal.m:26214
+runtime/metal	DS4_METAL_DISABLE_PRE_M5_COMP_FINALIZE_FUSE	presence rollback; unset: automatic/default path; any value including 0 disables	Disables pre M5 comp finalize fuse.	ds4.c:23646
+runtime/metal	DS4_METAL_DISABLE_PRE_M5_DECODE_EARLY_PIPELINE_FAST_LOOKUP	presence rollback; unset: automatic/default path; any value including 0 disables	Disables pre M5 decode early pipeline fast lookup.	ds4.c:28003
+runtime/metal	DS4_METAL_DISABLE_PRE_M5_DECODE_EARLY_SECOND_SPLIT12	presence rollback; unset: automatic/default path; any value including 0 disables	Disables pre M5 decode early second split12.	ds4.c:28087
+runtime/metal	DS4_METAL_DISABLE_PRE_M5_DECODE_EARLY_SPLIT3	presence rollback; unset: automatic/default path; any value including 0 disables	Disables pre M5 decode early split3.	ds4.c:27961
+runtime/metal	DS4_METAL_DISABLE_PRE_M5_DECODE_EARLY_SPLIT5	presence rollback; unset: automatic/default path; any value including 0 disables	Disables pre M5 decode early split5.	ds4.c:27973
+runtime/metal	DS4_METAL_DISABLE_PRE_M5_DECODE_PIPELINE_FAST_LOOKUP	presence rollback; unset: automatic/default path; any value including 0 disables	Disables pre M5 decode pipeline fast lookup.	ds4.c:28015
+runtime/metal	DS4_METAL_DISABLE_PRE_M5_DECODE_PORTS	presence rollback; unset: automatic/default path; any value including 0 disables	Disables pre M5 decode ports.	ds4.c:22613
+runtime/metal	DS4_METAL_DISABLE_PRE_M5_DECODE_RAW_ZERO_ATTN_MASK	presence rollback; unset: automatic/default path; any value including 0 disables	Disables pre M5 decode raw zero attn mask.	ds4_metal.m:29920
+runtime/metal	DS4_METAL_DISABLE_PRE_M5_DECODE_SECOND_SPLIT16	presence rollback; unset: automatic/default path; any value including 0 disables	Disables pre M5 decode second split16.	ds4.c:28095
+runtime/metal	DS4_METAL_DISABLE_PRE_M5_FLASH_ATTN_BATCHED_MEMO	presence rollback; unset: automatic/default path; any value including 0 disables	Disables pre M5 flash attn batched memo.	ds4_metal.m:3752
+runtime/metal	DS4_METAL_DISABLE_PRE_M5_FLASH_ATTN_PACKED32_REDUCE	presence rollback; unset: automatic/default path; any value including 0 disables	Disables pre M5 flash attn packed32 reduce.	ds4_metal.m:31460
+runtime/metal	DS4_METAL_DISABLE_PRE_M5_FLASH_ATTN_PAD_BLK_MEMO	presence rollback; unset: automatic/default path; any value including 0 disables	Disables pre M5 flash attn pad blk memo.	ds4_metal.m:3610
+runtime/metal	DS4_METAL_DISABLE_PRE_M5_HC_NORM_MIX_FUSE	presence rollback; unset: automatic/default path; any value including 0 disables	Disables pre M5 HC norm mix fuse.	ds4.c:22867
+runtime/metal	DS4_METAL_DISABLE_PRE_M5_HC_PRODUCER_PRE_NORM_FUSE	presence rollback; unset: automatic/default path; any value including 0 disables	Disables pre M5 HC producer pre norm fuse.	ds4_metal.m:46546
+runtime/metal	DS4_METAL_DISABLE_PRE_M5_HEAD_RMS_ROPE_PIPELINE_STATIC	presence rollback; unset: automatic/default path; any value including 0 disables	Disables pre M5 head RMS RoPE pipeline static.	ds4_metal.m:10026
+runtime/metal	DS4_METAL_DISABLE_PRE_M5_KV_ROPE_FP8_FUSE	presence rollback; unset: automatic/default path; any value including 0 disables	Disables pre M5 KV RoPE fp8 fuse.	ds4.c:23455
+runtime/metal	DS4_METAL_DISABLE_PRE_M5_MXFP4_MM_ID_PAIR_HALF_SCALE	presence rollback; unset: automatic/default path; any value including 0 disables	Disables pre M5 MXFP4 MM ID pair half scale.	ds4_metal.m:45129
+runtime/metal	DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_DECODE_FIXED_ROUTE_PAIR	presence rollback; unset: automatic/default path; any value including 0 disables	Disables pre M5 MXFP4 MoE decode fixed route pair.	ds4_metal.m:41948
+runtime/metal	DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_DECODE_FIXED_ROUTE_SUM6	presence rollback; unset: automatic/default path; any value including 0 disables	Disables pre M5 MXFP4 MoE decode fixed route sum6.	ds4_metal.m:41961
+runtime/metal	DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_DECODE_NSG1	presence rollback; unset: automatic/default path; any value including 0 disables	Disables pre M5 MXFP4 MoE decode nsg1.	ds4_metal.m:41719
+runtime/metal	DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_DECODE_STATIC_TRIP	presence rollback; unset: automatic/default path; any value including 0 disables	Disables pre M5 MXFP4 MoE decode static trip.	ds4_metal.m:41987
+runtime/metal	DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_DECODE_SUM6_FULL_ROWS	presence rollback; unset: automatic/default path; any value including 0 disables	Disables pre M5 MXFP4 MoE decode sum6 full rows.	ds4_metal.m:41974
+runtime/metal	DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_DECODE_TG_MULTIPLE	presence rollback; unset: automatic/default path; any value including 0 disables	Disables pre M5 MXFP4 MoE decode tg multiple.	ds4_metal.m:41936
+runtime/metal	DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_MM_ID_DOWN_HALF_LUT	presence rollback; unset: automatic/default path; any value including 0 disables	Disables pre M5 MXFP4 MoE MM ID down half lut.	ds4_metal.m:45047
+runtime/metal	DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_MM_ID_DOWN_TAIL_SIMDGROUP_CULL	presence rollback; unset: automatic/default path; any value including 0 disables	Disables pre M5 MXFP4 MoE MM ID down tail simdgroup cull.	ds4_metal.m:45030
+runtime/metal	DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_MM_ID_MAP_SCATTER	presence rollback; unset: automatic/default path; any value including 0 disables	Disables pre M5 MXFP4 MoE MM ID map scatter.	ds4_metal.m:44997
+runtime/metal	DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_MM_ID_PAIR_SWIGLU_COMPACT_TILE	presence rollback; unset: automatic/default path; any value including 0 disables	Disables pre M5 MXFP4 MoE MM ID pair SwiGLU compact tile.	ds4_metal.m:44981
+runtime/metal	DS4_METAL_DISABLE_PRE_M5_MXFP4_MOE_MM_ID_PAIR_TAIL_SIMDGROUP_CULL	presence rollback; unset: automatic/default path; any value including 0 disables	Disables pre M5 MXFP4 MoE MM ID pair tail simdgroup cull.	ds4_metal.m:45018
+runtime/metal	DS4_METAL_DISABLE_PRE_M5_PARALLEL_FULL_FFN	presence rollback; unset: automatic/default path; any value including 0 disables	Disables pre M5 parallel full FFN.	ds4.c:22706
+runtime/metal	DS4_METAL_DISABLE_PRE_M5_Q2_DECODE_SPLIT2_32	presence rollback; unset: automatic/default path; any value including 0 disables	Disables pre M5 q2 decode split2 32.	ds4.c:27876
+runtime/metal	DS4_METAL_DISABLE_PRE_M5_QKV_NORM_KV_STORE_FUSE	presence rollback; unset: automatic/default path; any value including 0 disables	Disables pre M5 QKV norm KV store fuse.	ds4.c:23314
+runtime/metal	DS4_METAL_DISABLE_PRE_M5_QKV_PAIR_COMPRESSOR_FUSE	presence rollback; unset: automatic/default path; any value including 0 disables	Disables pre M5 QKV pair compressor fuse.	ds4.c:23042
+runtime/metal	DS4_METAL_DISABLE_PRE_M5_QKV_PAIR_QUAD_FUSE	presence rollback; unset: automatic/default path; any value including 0 disables	Disables pre M5 QKV pair quad fuse.	ds4.c:23038
+runtime/metal	DS4_METAL_DISABLE_PRE_M5_ROUTER_SHARED_FUSE	presence rollback; unset: automatic/default path; any value including 0 disables	Disables pre M5 router shared fuse.	ds4.c:24755
+runtime/metal	DS4_METAL_DISABLE_PRE_M5_ROUTER_SIMD_FINALIZE	presence rollback; unset: automatic/default path; any value including 0 disables	Disables pre M5 router simd finalize.	ds4_metal.m:35827
+runtime/metal	DS4_METAL_DISABLE_PRE_M5_ROUTER_SIMD_WEIGHTS_FUSION	presence rollback; unset: automatic/default path; any value including 0 disables	Disables pre M5 router simd weights fusion.	ds4_metal.m:35836
+runtime/metal	DS4_METAL_DISABLE_PRE_M5_ROUTER_TRANSFORM_FINALIZE_FUSION	presence rollback; unset: automatic/default path; any value including 0 disables	Disables pre M5 router transform finalize fusion.	ds4_metal.m:35840
+runtime/metal	DS4_METAL_DISABLE_PRO_Q4_EXPERT_ADDRESS_AUTO	presence rollback; unset: automatic/default path; any value including 0 disables	Disables pro Q4 expert address auto.	ds4_metal.m:19710
+runtime/metal	DS4_METAL_DISABLE_PRO_Q4_EXPERT_TABLE_AUTO	presence rollback; unset: automatic/default path; any value including 0 disables	Disables pro Q4 expert table auto.	ds4.c:21568
+runtime/metal	DS4_METAL_DISABLE_PRO_Q4_EXPERT_TABLE_PRELOAD	presence rollback; unset: automatic/default path; any value including 0 disables	Disables pro Q4 expert table preload.	ds4.c:63088
+runtime/metal	DS4_METAL_DISABLE_Q4_ATTN_OUT_A_DIRECT	presence rollback; unset enables the automatic fixed-route path for eligible Apple M1-M4 long prefills; any defined value including 0 disables	Restore the generic route-map/work-list Q4 attention output-A path instead of the bit-identical fixed-route direct kernel.	ds4_metal.m:31562
+runtime/metal	DS4_METAL_DISABLE_Q4_ATTN_OUT_B_F16_RHS	value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables	Disables resident pre-M5 Q4 attention output-B F16 RHS materialization and restores per-tile F32 staging.	ds4_metal.m:31558
+runtime/metal	DS4_METAL_DISABLE_Q4_ATTN_OUT_HC_FUSE	presence rollback; unset: automatic/default path; any value including 0 disables	Disables Q4 attn out HC fuse.	ds4_metal.m:47624
+runtime/metal	DS4_METAL_DISABLE_Q4_ATTN_OUT_TINY_BATCH	value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables	Disables Q4 attn out tiny batch.	ds4_metal.m:28396
+runtime/metal	DS4_METAL_DISABLE_Q4_ATTN_Q_B_F16_CACHE	value-aware boolean; unset/0: persistent sidecar remains eligible; empty/1/true/yes/on disables; REQUIRE plus DISABLE fails closed	Disables only the persistent pre-M5 Q4_K attn_q_b F16 weight sidecar; unless REQUIRE is set, the default transient F16 path remains independently eligible, so also set DS4_METAL_DISABLE_Q4_ATTN_Q_B_TRANSIENT_F16=1 to force native per-tile Q4 dequantization.	ds4_metal.m:25819; ds4_metal.m:26131
+runtime/metal	DS4_METAL_DISABLE_Q4_ATTN_Q_B_F16_RHS	value-aware boolean; unset/0/false/no/off keeps the resident path enabled; empty/1/true/yes/on disables	Disables one-time F32-to-F16 RHS materialization for the resident pre-M5 Q4_K attn_q_b F16 sidecar and restores repeated per-tile F32 staging.	ds4_metal.m:25821; ds4_metal.m:26136
+runtime/metal	DS4_METAL_DISABLE_Q4_ATTN_Q_B_TRANSIENT_F16	value-aware rollback; unset/0/false/no/off keeps the automatic path enabled; empty/1/true/yes/on disables	Disables per-layer transient Q4_K attn_q_b-to-F16 scratch for long prefill, including the SSD-safe exact-view path; an enabled/required and eligible persistent sidecar may still run, otherwise dispatch returns to native Q4.	ds4_metal.m:26212; ds4_metal.m:27162
+runtime/metal	DS4_METAL_DISABLE_Q4_BATCH_EXPERT_TABLE	presence rollback; unset: automatic/default path; any value including 0 disables	Disables Q4 batch expert table.	ds4_metal.m:44883
+runtime/metal	DS4_METAL_DISABLE_Q4_DENSE_PAIR	presence rollback; unset: automatic/default path; any value including 0 disables	Disables Q4 dense pair.	ds4_metal.m:21990
+runtime/metal	DS4_METAL_DISABLE_Q4_EXACT_BOUNDARY	presence rollback; unset: automatic/default path; any value including 0 disables	Disables Q4 exact boundary.	ds4_metal.m:42239
+runtime/metal	DS4_METAL_DISABLE_Q4_EXACT_TENSOR_ID	presence rollback; unset: automatic/default path; any value including 0 disables	Disables Q4 exact tensor ID.	ds4_metal.m:42219
+runtime/metal	DS4_METAL_DISABLE_Q4_EXPERT_ADDRESS_TABLE	presence rollback; unset: automatic/default path; any value including 0 disables	Disables Q4 expert address table.	ds4_metal.m:19711
+runtime/metal	DS4_METAL_DISABLE_Q4_EXPERT_TABLE	presence rollback; unset: automatic/default path; any value including 0 disables	Disables Q4 expert table.	ds4.c:21041
+runtime/metal	DS4_METAL_DISABLE_Q4_GATHER_SLOTS	presence rollback; unset: automatic/default path; any value including 0 disables	Disables Q4 gather slots.	ds4_metal.m:42321
+runtime/metal	DS4_METAL_DISABLE_Q4_GROUP24_EXPERT_TABLE	presence rollback; unset: automatic/default path; any value including 0 disables	Disables Q4 group24 expert table.	ds4_metal.m:42201
+runtime/metal	DS4_METAL_DISABLE_Q4_GROUP6_EXPERT_TABLE	presence rollback; unset: automatic/default path; any value including 0 disables	Disables Q4 group6 expert table.	ds4_metal.m:42167
+runtime/metal	DS4_METAL_DISABLE_Q4_GROUP8_EXPERT_TABLE	presence rollback; unset: automatic/default path; any value including 0 disables	Disables Q4 group8 expert table.	ds4_metal.m:42184
+runtime/metal	DS4_METAL_DISABLE_Q4_GROUPED_BOUNDARY	presence rollback; unset: automatic/default path; any value including 0 disables	Disables Q4 grouped boundary.	ds4_metal.m:42149
+runtime/metal	DS4_METAL_DISABLE_Q4_GROUPED_EXPERTS	presence rollback; unset: automatic/default path; any value including 0 disables	Disables Q4 grouped experts.	ds4_metal.m:42130
+runtime/metal	DS4_METAL_DISABLE_Q4_MV_CLASSIC	presence rollback; unset: automatic/default path; any value including 0 disables	Disables Q4 MV classic.	ds4_metal.m:21571
+runtime/metal	DS4_METAL_DISABLE_Q4_PREFILL_PAIR_F16_RHS	value-aware authoritative rollback; unset/0/false/no/off permits the opt-in, while empty/1/true/yes/on or another nonempty value disables it; REQUIRE then fails closed	Restore two independent Q4_K/F32-RHS q_a and KV prefill matmuls instead of materializing their shared F16 RHS once.	ds4_metal.m:22911
+runtime/metal	DS4_METAL_DISABLE_Q4_PREFILL_TAIL_SIMDGROUP_CULL	presence rollback; unset: automatic on Apple M1-M4 for Q4_K single-tile prefill N=9..16 and production attn_q_b tails through N=65; any value including 0 disables	Restores the legacy four-SIMDgroup Q4_K prefill kernel on the measured short-prefill scopes.	ds4_metal.m:22088
+runtime/metal	DS4_METAL_DISABLE_Q4_QKV_COMPRESSOR_FUSE	presence rollback; unset: automatic/default path; any value including 0 disables	Disables Q4 QKV compressor fuse.	ds4_metal.m:22083
+runtime/metal	DS4_METAL_DISABLE_Q4_SELECTED_EXPERT_VIEWS	presence rollback; unset: automatic/default path; any value including 0 disables	Disables Q4 selected expert views.	ds4.c:21136
+runtime/metal	DS4_METAL_DISABLE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN	value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables	Disables Q4 SSD prefill attn out exactn.	ds4_metal.m:28095
+runtime/metal	DS4_METAL_DISABLE_Q4_SSD_PREFILL_ATTN_OUT_SCALE_META	value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables	Disables shared scale/min metadata in the Q4 SSD prefill attention-output exact-N kernel.	ds4_metal.m:28220
+runtime/metal	DS4_METAL_DISABLE_Q4_SSD_SESSION_UNION	nonempty boolean; unset/empty or exact 0: off; every other value: on	Disables Q4 SSD session union.	ds4.c:65160
+runtime/metal	DS4_METAL_DISABLE_Q4_STREAM_OVERLAP	nonempty boolean; unset/empty or exact 0: off; every other value: on	Disables Q4 stream overlap.	ds4.c:65082
+runtime/metal	DS4_METAL_DISABLE_Q4_TABLE_BOUNDARY	presence rollback; unset: automatic/default path; any value including 0 disables	Disables Q4 table boundary.	ds4_metal.m:42318
+runtime/metal	DS4_METAL_DISABLE_Q8_DECODE_EXACT_VIEWS	presence rollback; unset: automatic/default path; any value including 0 disables	Disables Q8 decode exact views.	ds4_metal.m:12851
+runtime/metal	DS4_METAL_DISABLE_QKV_NORM_FUSION	nonempty boolean; unset/empty or exact 0: off; every other value: on	Disables QKV norm fusion.	ds4.c:20435
+runtime/metal	DS4_METAL_DISABLE_QKV_PAIR_PROJ	nonempty boolean; unset/empty or exact 0: off; every other value: on	Disables QKV pair proj.	ds4.c:20440
+runtime/metal	DS4_METAL_DISABLE_QUEUE_RESIDENCY_SET	presence rollback; unset: automatic/default path; any value including 0 disables	Disables queue residency set.	ds4_metal.m:2124
+runtime/metal	DS4_METAL_DISABLE_ROUTED_PAIR_SWIGLU_FUSION	presence rollback; unset: automatic/default path; any value including 0 disables	Disables routed pair SwiGLU fusion.	ds4.c:18529
+runtime/metal	DS4_METAL_DISABLE_ROUTER_SELECT_FUSION	presence rollback; unset: automatic/default path; any value including 0 disables	Disables router select fusion.	ds4_metal.m:35816
+runtime/metal	DS4_METAL_DISABLE_ROUTER_WEIGHTS_BATCH_FUSION	presence rollback; unset: automatic/default path; any value including 0 disables	Disables router weights batch fusion.	ds4_metal.m:36068
+runtime/metal	DS4_METAL_DISABLE_SHARED_DOWN_HC_FUSION	nonempty boolean; unset/empty or exact 0: off; every other value: on	Disables shared down HC fusion.	ds4.c:20472
+runtime/metal	DS4_METAL_DISABLE_SHARED_GATE_UP_SWIGLU_FUSION	presence rollback; unset: automatic/default path; any value including 0 disables	Disables shared gate up SwiGLU fusion.	ds4.c:17567
+runtime/metal	DS4_METAL_DISABLE_SHARED_KV_PAD	presence rollback; unset: automatic/default path; any value including 0 disables	Disables shared KV pad.	ds4_metal.m:31510
+runtime/metal	DS4_METAL_DISABLE_SHARED_ROPE_COEFF	presence rollback; unset: automatic/default path; any value including 0 disables	Disables shared RoPE coeff.	ds4_metal.m:6341
+runtime/metal	DS4_METAL_DISABLE_SMALL_PREFILL_DIRECT	presence rollback; unset: direct one-workgroup FlashAttention output for prefill with at most 32 keys; any defined value including 0 restores split reduction	Disables direct normalized output for small Metal prefill and restores the 32-way temporary plus reduction path.	ds4_metal.m:4356
+runtime/metal	DS4_METAL_DISABLE_STREAMING_COLD_DECODE_PREFILL	presence rollback; unset: automatic/default path; any value including 0 disables	Disables streaming cold decode prefill.	ds4.c:32007
+runtime/metal	DS4_METAL_DISABLE_STREAMING_COMPACT_ADDR	presence rollback; unset: automatic/default path; any value including 0 disables	Disables streaming compact address.	ds4_metal.m:14598
+runtime/metal	DS4_METAL_DISABLE_STREAMING_DECODE_PREFILL	presence rollback; unset: automatic/default path; any value including 0 disables	Disables streaming decode prefill.	ds4.c:31956
+runtime/metal	DS4_METAL_DISABLE_STREAMING_EXPERT_ADDR_TABLE	presence rollback; unset: automatic/default path; any value including 0 disables	Disables streaming expert address table.	ds4.c:18525
+runtime/metal	DS4_METAL_DISABLE_STREAMING_EXPERT_COMBINED_BUFFER	presence rollback; unset: automatic/default path; any value including 0 disables	Disables streaming expert combined buffer.	ds4_metal.m:14023
+runtime/metal	DS4_METAL_DISABLE_STREAMING_EXPERT_EARLY_LOAD	presence rollback; unset: automatic/default path; any value including 0 disables	Disables streaming expert early load.	ds4_metal.m:17232
+runtime/metal	DS4_METAL_DISABLE_STREAMING_EXPERT_EVICT_DONTNEED	presence rollback; unset: automatic/default path; any value including 0 disables	Disables streaming expert evict dontneed.	ds4_metal.m:14395
+runtime/metal	DS4_METAL_DISABLE_STREAMING_EXPERT_HIT_VALIDATOR	presence rollback; unset: automatic/default path; any value including 0 disables	Disables streaming expert hit validator.	ds4_metal.m:14634
+runtime/metal	DS4_METAL_DISABLE_STREAMING_EXPERT_HOTLIST	presence rollback; unset: automatic/default path; any value including 0 disables	Disables streaming expert hotlist.	ds4.c:21290
+runtime/metal	DS4_METAL_DISABLE_STREAMING_EXPERT_LIVE_INDEX	value-aware boolean; default off; true disables and dominates ENABLE	Disables dense live-entry index and uses authoritative cache matrix.	ds4_metal.m:15443
+runtime/metal	DS4_METAL_DISABLE_STREAMING_EXPERT_MASKED_ADDR	presence rollback; unset: automatic/default path; any value including 0 disables	Disables streaming expert masked address.	ds4_metal.m:14628
+runtime/metal	DS4_METAL_DISABLE_STREAMING_EXPERT_READAHEAD	presence rollback; unset: automatic/default path; any value including 0 disables	Disables streaming expert readahead.	ds4_metal.m:13196
+runtime/metal	DS4_METAL_DISABLE_STREAMING_EXPERT_SLABS	presence rollback; unset: automatic/default path; any value including 0 disables	Disables streaming expert slabs.	ds4_metal.m:14028
+runtime/metal	DS4_METAL_DISABLE_STREAMING_EXPERT_TIMING_SUMMARY	presence rollback; unset: automatic/default path; any value including 0 disables	Disables streaming expert timing summary.	ds4_metal.m:13062
+runtime/metal	DS4_METAL_DISABLE_STREAMING_FULL_EXPERT_ADDR_TABLE	presence rollback; unset: automatic/default path; any value including 0 disables	Disables streaming full expert address table.	ds4_metal.m:14716
+runtime/metal	DS4_METAL_DISABLE_STREAMING_IQ2_CPU_ROUTER	presence rollback; unset: automatic/default path; any value including 0 disables	Disables streaming IQ2 CPU router.	ds4.c:20939
+runtime/metal	DS4_METAL_DISABLE_STREAMING_LAYER_BATCH	presence rollback; unset: automatic/default path; any value including 0 disables	Disables streaming layer batch.	ds4.c:18239
+runtime/metal	DS4_METAL_DISABLE_STREAMING_MADVISE_WILLNEED	presence rollback; unset: automatic/default path; any value including 0 disables	Disables streaming madvise willneed.	ds4.c:18212
+runtime/metal	DS4_METAL_DISABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR	presence rollback; unset: automatic/default path; any value including 0 disables	Disables streaming prefill batch selected address.	ds4.c:18523
+runtime/metal	DS4_METAL_DISABLE_STREAMING_PREFILL_LAYER_MADVISE	presence rollback; unset: automatic/default path; any value including 0 disables	Disables streaming prefill layer madvise.	ds4.c:18466
+runtime/metal	DS4_METAL_DISABLE_STREAMING_PREFILL_LAYER_PAGEIN	presence rollback; unset: automatic/default path; any value including 0 disables	Disables streaming prefill layer pagein.	ds4.c:18434
+runtime/metal	DS4_METAL_DISABLE_STREAMING_PREFILL_LAYER_PAGEIN_OVERLAP	presence rollback; unset: automatic/default path; any value including 0 disables	Disables streaming prefill layer pagein overlap.	ds4.c:19321
+runtime/metal	DS4_METAL_DISABLE_STREAMING_PREFILL_LAYER_PREAD	presence rollback; unset: automatic/default path; any value including 0 disables	Disables streaming prefill layer pread.	ds4.c:18454
+runtime/metal	DS4_METAL_DISABLE_STREAMING_PREFILL_LAYER_PREPARE	presence rollback; unset: automatic/default path; any value including 0 disables	Disables streaming prefill layer prepare.	ds4.c:18446
+runtime/metal	DS4_METAL_DISABLE_STREAMING_PREFILL_LAYER_PREPARE_OVERLAP	presence rollback; unset: automatic/default path; any value including 0 disables	Disables streaming prefill layer prepare overlap.	ds4.c:19319
+runtime/metal	DS4_METAL_DISABLE_STREAMING_PREFILL_LAYER_READAHEAD	presence rollback; unset: automatic/default path; any value including 0 disables	Disables streaming prefill layer readahead.	ds4.c:18444
+runtime/metal	DS4_METAL_DISABLE_STREAMING_PREFILL_SELECTED_ASYNC_LOAD	presence rollback; unset: automatic/default path; any value including 0 disables	Disables streaming prefill selected async load.	ds4.c:46424
+runtime/metal	DS4_METAL_DISABLE_STREAMING_PREFILL_SELECTED_MADVISE	presence rollback; unset: automatic/default path; any value including 0 disables	Disables streaming prefill selected madvise.	ds4.c:18424
+runtime/metal	DS4_METAL_DISABLE_STREAMING_PREFILL_SELECTED_PAGEIN	presence rollback; unset: automatic/default path; any value including 0 disables	Disables streaming prefill selected pagein.	ds4.c:18414
+runtime/metal	DS4_METAL_DISABLE_STREAMING_PREFILL_SELECTED_PROFILE	presence rollback; unset: automatic/default path; any value including 0 disables	Disables streaming prefill selected profile.	ds4.c:18908
+runtime/metal	DS4_METAL_DISABLE_STREAMING_PREFILL_SELECTED_READAHEAD	presence rollback; unset: automatic/default path; any value including 0 disables	Disables streaming prefill selected readahead.	ds4.c:19960
+runtime/metal	DS4_METAL_DISABLE_STREAMING_PREFILL_SELECTED_READAHEAD_SHARED	presence rollback; unset: automatic/default path; any value including 0 disables	Disables streaming prefill selected readahead shared.	ds4.c:19970
+runtime/metal	DS4_METAL_DISABLE_STREAMING_READAHEAD	presence rollback; unset: automatic/default path; any value including 0 disables	Disables streaming readahead.	ds4.c:18205
+runtime/metal	DS4_METAL_DISABLE_STREAMING_SELECTED_ASYNC_EARLY_COMMIT	presence rollback; unset: automatic/default path; any value including 0 disables	Disables streaming selected async early commit.	ds4.c:21019
+runtime/metal	DS4_METAL_DISABLE_STREAMING_SELECTED_ASYNC_LOAD	presence rollback; unset: automatic/default path; any value including 0 disables	Disables streaming selected async load.	ds4.c:21003
+runtime/metal	DS4_METAL_DISABLE_STREAMING_SELECTED_READAHEAD_SHARED_DELAY	presence rollback; unset: automatic/default path; any value including 0 disables	Disables streaming selected readahead shared delay.	ds4.c:21720
+runtime/metal	DS4_METAL_DISABLE_STREAMING_SELECTED_SHARED_OVERLAP	presence rollback; unset: automatic/default path; any value including 0 disables	Disables streaming selected shared overlap.	ds4.c:20995
+runtime/metal	DS4_METAL_DISABLE_STREAMING_STATIC_DECODE_MAP	presence rollback; unset: automatic/default path; any value including 0 disables	Disables streaming static decode map.	ds4.c:18217
+runtime/metal	DS4_METAL_DISABLE_STREAMING_STATIC_MAP_STATE_CACHE	presence rollback; unset: automatic/default path; any value including 0 disables	Disables streaming static map state cache.	ds4.c:18230
+runtime/metal	DS4_METAL_DISABLE_SUPPORT_Q8_DECODE_EXACT_VIEWS	presence rollback; unset: automatic/default path; any value including 0 disables	Disables support Q8 decode exact views.	ds4_metal.m:12858
+runtime/metal	DS4_METAL_DISABLE_TINY_PAIR_SWIGLU_FUSION	presence rollback; unset: automatic/default path; any value including 0 disables	Disables tiny pair SwiGLU fusion.	ds4_metal.m:44920
+runtime/metal	DS4_METAL_DISABLE_TOKEN_EMBED_EXACT_VIEW	presence rollback; unset: automatic/default path; any value including 0 disables	Disables token embed exact view.	ds4_metal.m:11907
+runtime/metal	DS4_METAL_DISABLE_ZERO_PREFIX_PREFILL_MASK_CACHE	presence rollback; unset: automatic/default path; any value including 0 disables	Disables zero prefix prefill mask cache.	ds4_metal.m:2404
+runtime/metal	DS4_METAL_DSPARK_ACCEPTANCE_ONLY_VERIFY	nonempty boolean; unset/empty or exact 0: off; every other value: on	Verifies only the draft rows still needed for acceptance after the base target logit.	ds4.c:52499
+runtime/metal	DS4_METAL_DSPARK_DEVICE_PROPOSER	boolean true values enable; false values/unset disable; NO_DEVICE_PROPOSER presence dominates	Keeps DSpark Q8 confidence/Markov proposal work on Metal and reads one compact result.	ds4.c:34884
+runtime/metal	DS4_METAL_DSPARK_EXACT2	nonempty boolean; unset/empty or exact 0: off; every other value: on	Enables the resident single-GPU Metal exact-2 verifier.	ds4.c:52290
+runtime/metal	DS4_METAL_DSPARK_EXACTN	nonempty boolean; unset/empty or exact 0: off; every other value: on	Enables the single-GPU Metal exact-N verifier.	ds4.c:52311
+runtime/metal	DS4_METAL_DSPARK_EXACTN_BATCH_HEAD	nonempty boolean; unset/empty or exact 0: off; every other value: on	Batches the output head across exact-N verifier rows.	ds4.c:37726
+runtime/metal	DS4_METAL_DSPARK_EXACTN_UNION	nonempty boolean; unset/empty or exact 0: off; every other value: on	Loads the union of experts for exact-N verifier rows once.	ds4.c:52332
+runtime/metal	DS4_METAL_DSPARK_EXACT_ROWS_ASYNC_TAILS	presence control; unset: off/default; any value including 0 enables	Runs exact-row routed tails asynchronously after union routing.	ds4.c:37932
+runtime/metal	DS4_METAL_DSPARK_EXACT_ROWS_PROFILE	presence diagnostic; unset: off; any value including 0 enables	Prints timing/profile diagnostics for DSpark exact rows.	ds4.c:37930
+runtime/metal	DS4_METAL_DSPARK_HEADLESS_REPLAY	unset/empty: enabled; exact 0 disables; every other nonempty value enables	Skips output heads for accepted intermediate DSpark replay tokens.	ds4.c:52479
+runtime/metal	DS4_METAL_DSPARK_NO_DEVICE_PROPOSER	presence rollback; unset: automatic/default path; any value including 0 disables	Dominant presence-based rollback for the Metal DSpark device proposer.	ds4.c:34886
+runtime/metal	DS4_METAL_DSPARK_PIN_MAIN_PROJ	nonempty value other than exact 0 enables; unset/empty/0 disables	mlock-pins only DSpark stage-0 main_norm/main_proj in Metal SSD streaming.	ds4.c:39443
+runtime/metal	DS4_METAL_DSPARK_PROPOSER_BLOCK_MAX	uint32; unset: automatic cache/verifier cap; 0 or invalid: native width; positive: clamped to DSpark/native maximum	Caps rows proposed by single-device Metal DSpark.	ds4.c:52430
+runtime/metal	DS4_METAL_DSPARK_SAFE_EXPERT_COUNT	exact 1 enables; unset or any other value disables	Caps an explicit expert-count cache request to the safe Metal working-set budget for DSpark SSD streaming.	ds4.c:4880
+runtime/metal	DS4_METAL_DSV4_HC_SOURCE	file path; unset/empty: use the in-tree Metal source file	Overrides the DeepSeek hidden-context Metal kernel source file loaded at runtime.	ds4_metal.m:4938
+runtime/metal	DS4_METAL_DSV4_KV_SOURCE	file path; unset/empty: use the in-tree Metal source file	Overrides the DeepSeek KV Metal kernel source file loaded at runtime.	ds4_metal.m:4940
+runtime/metal	DS4_METAL_DSV4_MISC_SOURCE	file path; unset/empty: use the in-tree Metal source file	Overrides the DeepSeek miscellaneous Metal kernel source file loaded at runtime.	ds4_metal.m:4942
+runtime/metal	DS4_METAL_DSV4_ROPE_SOURCE	file path; unset/empty: use the in-tree Metal source file	Overrides the DeepSeek RoPE Metal kernel source file loaded at runtime.	ds4_metal.m:4941
+runtime/metal	DS4_METAL_DUMP_PREFILL_LOGITS	file path; unset/empty: no dump	Writes final GPU prefill logits as f32 binary.	ds4.c:51347
+runtime/metal	DS4_METAL_ENABLE_BATCH_HC_NORM_FUSION	legacy value-aware alias; default enabled; exact 0 disables; unset/empty/every other value enables unless DISABLE is active	Legacy control for the now-default batched HC norm fusion.	ds4.c:20462
+runtime/metal	DS4_METAL_ENABLE_COMPRESSOR_EXACT_POOL_RATIO4	presence opt-in; unset: off/automatic; any value including 0 enables	Enables compressor exact pool ratio4.	ds4_metal.m:26385
+runtime/metal	DS4_METAL_ENABLE_COMPRESSOR_PAIR_STATE_STORE	presence opt-in; unset: off/automatic; any value including 0 enables	Enables compressor pair state store.	ds4_metal.m:23243
+runtime/metal	DS4_METAL_ENABLE_COMPRESSOR_QUAD_STORE	presence opt-in; unset: off/automatic; any value including 0 enables	Enables compressor quad store.	ds4.c:23527
+runtime/metal	DS4_METAL_ENABLE_DSPARK_CAPTURE_FUSED_LAST	nonempty boolean; unset/empty or exact 0: off; every other value: on	Enables DSpark capture fused last.	ds4.c:28359
+runtime/metal	DS4_METAL_ENABLE_GATHERED_KV_STAGE	presence opt-in; unset: off/automatic; any value including 0 enables	Enables gathered KV stage.	ds4_metal.m:29675
+runtime/metal	DS4_METAL_ENABLE_GLM_STREAMING_SELECTED_ASYNC_LOAD	presence opt-in; unset: off/automatic; any value including 0 enables	Enables GLM streaming selected async load.	ds4.c:44335
+runtime/metal	DS4_METAL_ENABLE_HC_NORM_MIX_FUSE	presence opt-in; unset: off/automatic; any value including 0 enables	Enables HC norm mix fuse.	ds4.c:22870
+runtime/metal	DS4_METAL_ENABLE_HC_PRODUCER_PRE_NORM_FUSE	presence opt-in; unset: off/automatic; any value including 0 enables	Enables HC producer pre norm fuse.	ds4_metal.m:46552
+runtime/metal	DS4_METAL_ENABLE_IQ2_SELECTED_ASYNC_EARLY_COMMIT	presence opt-in; unset: off/automatic; any value including 0 enables	Enables IQ2 selected async early commit.	ds4.c:21018
+runtime/metal	DS4_METAL_ENABLE_IQ2_XXS_SSD_PREFILL_MM	value-aware boolean; default automatic/on for eligible shape; explicit 0 turns request off unless REQUIRE=1	Overrides automatic IQ2_XXS/Q2_K grouped address-MM selection for SSD prefill.	ds4_metal.m:44780
+runtime/metal	DS4_METAL_ENABLE_PRO_Q4_EXPERT_ADDRESS_AUTO	presence opt-in; unset: off/automatic; any value including 0 enables	Enables pro Q4 expert address auto.	ds4.c:20982
+runtime/metal	DS4_METAL_ENABLE_PRO_Q4_EXPERT_TABLE_AUTO	presence opt-in; unset: off/automatic; any value including 0 enables	Enables pro Q4 expert table auto.	ds4.c:20981
+runtime/metal	DS4_METAL_ENABLE_PRO_Q4_SELECTED_EXPERT_VIEWS	presence opt-in; unset: off/automatic; any value including 0 enables	Enables pro Q4 selected expert views.	ds4.c:20978
+runtime/metal	DS4_METAL_ENABLE_Q4_ATTN_OUT_TINY_BATCH	value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables	Enables Q4 attn out tiny batch.	ds4_metal.m:28405
+runtime/metal	DS4_METAL_ENABLE_Q4_ATTN_Q_B_F16_CACHE_WITH_SSD_STREAMING	value-aware opt-in, default off; empty/1/true/yes/on enables and 0/false/no/off disables; DS4_METAL_DISABLE_Q4_ATTN_Q_B_F16_CACHE wins	Allow resident Q4_K attn_q_b F16 sidecars while Metal continues streaming routed experts from SSD; admission reserves the configured streaming cache and prefill headroom and enforces the 7/8 working-set safety limit.	ds4_metal.m:25488; ds4_metal.m:25808; ds4_metal.m:26711
+runtime/metal	DS4_METAL_ENABLE_Q4_BATCH_EXPERT_TABLE	presence opt-in; unset: off/automatic; any value including 0 enables	Enables Q4 batch expert table.	ds4_metal.m:44869
+runtime/metal	DS4_METAL_ENABLE_Q4_EXACT_TENSOR_ID	presence opt-in; unset: off/automatic; any value including 0 enables	Enables Q4 exact tensor ID.	ds4_metal.m:42218
+runtime/metal	DS4_METAL_ENABLE_Q4_EXPERT_ADDRESS_TABLE	presence opt-in; unset: off/automatic; any value including 0 enables	Enables Q4 expert address table.	ds4.c:20980
+runtime/metal	DS4_METAL_ENABLE_Q4_EXPERT_TABLE	presence opt-in; unset: off/automatic; any value including 0 enables	Enables Q4 expert table.	ds4.c:20979
+runtime/metal	DS4_METAL_ENABLE_Q4_GATHER_SLOTS	presence opt-in; unset: off/automatic; any value including 0 enables	Enables Q4 gather slots.	ds4_metal.m:42320
+runtime/metal	DS4_METAL_ENABLE_Q4_GROUP24_EXPERT_TABLE	presence opt-in; unset: off/automatic; any value including 0 enables	Enables Q4 group24 expert table.	ds4_metal.m:42200
+runtime/metal	DS4_METAL_ENABLE_Q4_GROUP6_EXPERT_TABLE	presence opt-in; unset: off/automatic; any value including 0 enables	Enables Q4 group6 expert table.	ds4_metal.m:42166
+runtime/metal	DS4_METAL_ENABLE_Q4_GROUP8_EXPERT_TABLE	presence opt-in; unset: off/automatic; any value including 0 enables	Enables Q4 group8 expert table.	ds4_metal.m:42183
+runtime/metal	DS4_METAL_ENABLE_Q4_GROUPED_EXPERTS	presence opt-in; unset: off/automatic; any value including 0 enables	Enables Q4 grouped experts.	ds4_metal.m:42129
+runtime/metal	DS4_METAL_ENABLE_Q4_PREFILL_PAIR_F16_RHS	value-aware opt-in, default off; empty/1/true/yes/on or another nonempty value enables, while 0/false/no/off disables; REQUIRE also requests it; DISABLE wins	On Apple M1-M4, convert the shared q_a/KV prefill activation to F16 once and feed two bit-exact Q4_K kernels for N=32,64,96,128.	ds4_metal.m:22909
+runtime/metal	DS4_METAL_ENABLE_Q4_QKV_COMPRESSOR_FUSE	presence opt-in; unset: off/automatic; any value including 0 enables	Enables Q4 QKV compressor fuse.	ds4.c:23140
+runtime/metal	DS4_METAL_ENABLE_Q4_SELECTED_EXPERT_VIEWS	presence opt-in; unset: off/automatic; any value including 0 enables	Enables Q4 selected expert views.	ds4.c:20977
+runtime/metal	DS4_METAL_ENABLE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN	value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables	Enables Q4 SSD prefill attn out exactn.	ds4_metal.m:28097
+runtime/metal	DS4_METAL_ENABLE_Q4_SSD_SESSION_UNION	nonempty boolean; unset/empty or exact 0: off; every other value: on	Enables Q4 SSD session union.	ds4.c:65171
+runtime/metal	DS4_METAL_ENABLE_Q4_STREAM_OVERLAP	nonempty boolean; unset/empty or exact 0: off; every other value: on	Enables Q4 stream overlap.	ds4.c:65080
+runtime/metal	DS4_METAL_ENABLE_Q8_DECODE_EXACT_VIEWS	presence opt-in; unset: off/automatic; any value including 0 enables	Enables Q8 decode exact views.	ds4_metal.m:12865
+runtime/metal	DS4_METAL_ENABLE_Q8_QKV_COMPRESSOR_FUSE	nonempty boolean; unset/empty/exact 0: no streamed/union opt-in; other values enable; eligible resident full-decode remains automatic	Extends the automatic resident Q8 QKV/compressor compound fusion to SSD streaming or exact-N union scope.	ds4.c:23027
+runtime/metal	DS4_METAL_ENABLE_STREAMING_COMPACT_ADDR	presence opt-in; unset: off/automatic; any value including 0 enables	Enables streaming compact address.	ds4_metal.m:14597
+runtime/metal	DS4_METAL_ENABLE_STREAMING_EXPERT_ADDR_TABLE	presence opt-in; unset: off/automatic; any value including 0 enables	Enables streaming expert address table.	ds4_metal.m:14604
+runtime/metal	DS4_METAL_ENABLE_STREAMING_EXPERT_EVICT_DONTNEED	presence opt-in; unset: off/automatic; any value including 0 enables	Enables streaming expert evict dontneed.	ds4_metal.m:14394
+runtime/metal	DS4_METAL_ENABLE_STREAMING_EXPERT_HIT_VALIDATOR	presence opt-in; unset: off/automatic; any value including 0 enables	Enables streaming expert hit validator.	ds4_metal.m:14605
+runtime/metal	DS4_METAL_ENABLE_STREAMING_EXPERT_LIVE_INDEX	value-aware boolean; default automatic/on for validated IQ2 cache shape; explicit 0 disables	Overrides automatic dense live-entry index selection.	ds4_metal.m:15442
+runtime/metal	DS4_METAL_ENABLE_STREAMING_EXPERT_MASKED_ADDR	presence opt-in; unset: off/automatic; any value including 0 enables	Enables streaming expert masked address.	ds4_metal.m:14606
+runtime/metal	DS4_METAL_ENABLE_STREAMING_FULL_EXPERT_ADDR_TABLE	presence opt-in; unset: off/automatic; any value including 0 enables	Enables streaming full expert address table.	ds4_metal.m:14715
+runtime/metal	DS4_METAL_ENABLE_STREAMING_IQ2_CPU_ROUTER	presence opt-in; unset: off/automatic; any value including 0 enables	Enables streaming IQ2 CPU router.	ds4.c:20938
+runtime/metal	DS4_METAL_ENABLE_STREAMING_MADVISE_WILLNEED	presence opt-in; unset: off/automatic; any value including 0 enables	Enables streaming madvise willneed.	ds4.c:18210
+runtime/metal	DS4_METAL_ENABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR	presence opt-in; unset: off/automatic; any value including 0 enables	Enables streaming prefill batch selected address.	ds4_metal.m:14609
+runtime/metal	DS4_METAL_ENABLE_STREAMING_PREFILL_CACHE_SEED	presence opt-in; unset: off/automatic; any value including 0 enables	Enables streaming prefill cache seed.	ds4.c:21259
+runtime/metal	DS4_METAL_ENABLE_STREAMING_PREFILL_EXPERT_READAHEAD	value-aware boolean; for batches <32 readahead is automatic; for batches >=32 unset/false disables and true enables; global READHEAD rollback and F_NOCACHE still dominate	Restores F_RDADVISE immediately before parallel pread for large SSD-prefill batches.	ds4_metal.m:13212
+runtime/metal	DS4_METAL_ENABLE_STREAMING_PREFILL_LAYER_PAGEIN	presence opt-in; unset: off/automatic; any value including 0 enables	Enables streaming prefill layer pagein.	ds4.c:18432
+runtime/metal	DS4_METAL_ENABLE_STREAMING_PREFILL_LAYER_READAHEAD	presence opt-in; unset: off/automatic; any value including 0 enables	Enables streaming prefill layer readahead.	ds4.c:18442
+runtime/metal	DS4_METAL_ENABLE_STREAMING_PREFILL_SELECTED_MADVISE	presence opt-in; unset: off/automatic; any value including 0 enables	Enables streaming prefill selected madvise.	ds4.c:18422
+runtime/metal	DS4_METAL_ENABLE_STREAMING_PREFILL_SELECTED_PAGEIN	presence opt-in; unset: off/automatic; any value including 0 enables	Enables streaming prefill selected pagein.	ds4.c:18412
+runtime/metal	DS4_METAL_ENABLE_STREAMING_PREFILL_SELECTED_READAHEAD	presence opt-in; unset: off/automatic; any value including 0 enables	Enables streaming prefill selected readahead.	ds4.c:19956
+runtime/metal	DS4_METAL_ENABLE_STREAMING_PREFILL_SELECTED_READAHEAD_SHARED	presence opt-in; unset: off/automatic; any value including 0 enables	Enables streaming prefill selected readahead shared.	ds4.c:19958
+runtime/metal	DS4_METAL_ENABLE_STREAMING_READAHEAD	presence opt-in; unset: off/automatic; any value including 0 enables	Enables streaming readahead.	ds4.c:18203
+runtime/metal	DS4_METAL_ENABLE_STREAMING_SELECTED_READAHEAD_SHARED_DELAY	presence opt-in; unset: off/automatic; any value including 0 enables	Enables streaming selected readahead shared delay.	ds4.c:21719
+runtime/metal	DS4_METAL_ENABLE_STREAMING_STATIC_DECODE_MAP	presence opt-in; unset: off/automatic; any value including 0 enables	Enables streaming static decode map.	ds4.c:18222
+runtime/metal	DS4_METAL_ENABLE_TOKEN_EMBED_EXACT_VIEW	presence opt-in; unset: off/automatic; any value including 0 enables	Enables token embed exact view.	ds4_metal.m:12219
+runtime/metal	DS4_METAL_EXACT_VIEW_CACHE_GIB	unsigned GiB; default 64; 0 disables size-triggered eviction; MIB overrides it	Sets the cached exact-model-view eviction threshold.	ds4_metal.m:1333
+runtime/metal	DS4_METAL_EXACT_VIEW_CACHE_MIB	unsigned MiB; unset: inherit GIB/default; 0 disables size-triggered eviction; overrides GIB	Sets the cached exact-model-view eviction threshold with MiB precision.	ds4_metal.m:1342
+runtime/metal	DS4_METAL_EXACT_VIEW_CACHE_PROFILE	presence diagnostic; unset: off; any value including 0 enables	Prints timing/profile diagnostics for exact view cache.	ds4_metal.m:1377
+runtime/metal	DS4_METAL_FLASH_ATTN_SOURCE	file path; unset/empty: use the in-tree Metal source file	Overrides the FlashAttention Metal kernel source file loaded at runtime.	ds4_metal.m:4935
+runtime/metal	DS4_METAL_FLASH_ATTN_STAGE_PROFILE	presence diagnostic; unset: off; any value including 0 enables	Prints timing/profile diagnostics for flash attn stage.	ds4.c:65152
+runtime/metal	DS4_METAL_FLASH_ATTN_STAGE_PROFILE_FILTER	substring; unset/empty: all profiled modes/stages	Filters FlashAttention stage-profile output by mode or stage substring.	ds4_metal.m:11178
+runtime/metal	DS4_METAL_GET_ROWS_SOURCE	file path; unset/empty: use the in-tree Metal source file	Overrides the get-rows Metal kernel source file loaded at runtime.	ds4_metal.m:4946
+runtime/metal	DS4_METAL_GLM_DISABLE_STREAMING_EXPERT_CACHE	presence rollback; unset: automatic/default path; any value including 0 disables	Disables streaming expert cache for GLM.	ds4_metal.m:39667
+runtime/metal	DS4_METAL_GLM_DISABLE_STREAMING_GROUPED_ADDR_PREFILL	presence rollback; unset: automatic/default path; any value including 0 disables	Disables streaming grouped address prefill for GLM.	ds4_metal.m:41081
+runtime/metal	DS4_METAL_GLM_DISABLE_STREAMING_SEED_BEFORE_PREFILL	presence rollback; unset: automatic/default path; any value including 0 disables	Disables streaming seed before prefill for GLM.	ds4.c:51074
+runtime/metal	DS4_METAL_GLM_DISABLE_STREAMING_TOKEN_PREFILL	presence rollback; unset: automatic/default path; any value including 0 disables	Disables streaming token prefill for GLM.	ds4.c:49704
+runtime/metal	DS4_METAL_GLM_MOE_ONE_STAGE_PROFILE	unset: off; 1/true/yes/on/all enables all layers; accepts layer lists/ranges; 0/false/no/off disables	Prints timing/profile diagnostics for GLM MoE one stage.	ds4_metal.m:39964
+runtime/metal	DS4_METAL_GLM_MOE_ONE_STAGE_PROFILE_LAYER	layer index/list/ranges or all; unset: all layers selected by profiler	Restricts GLM one-stage MoE profiling to selected layers.	ds4_metal.m:39965
+runtime/metal	DS4_METAL_GLM_MOE_STAGE_PROFILE_FILTER	substring; unset/empty: all profiled stages	Filters GLM MoE stage-profile output.	ds4_metal.m:39968
+runtime/metal	DS4_METAL_GLM_QKLOW_DEBUG	presence diagnostic; unset: off; any value including 0 enables	Enables debug diagnostics for GLM qklow.	ds4_metal.m:37868
+runtime/metal	DS4_METAL_GLM_STREAMING_ASYNC_PROFILE	presence diagnostic; unset: off; any value including 0 enables	Prints timing/profile diagnostics for GLM streaming async.	ds4.c:44362
+runtime/metal	DS4_METAL_GLM_STREAMING_DECODE_FULL_LAYER_MAP	presence control; unset: off/default; any value including 0 enables	Maps complete GLM layers during SSD-streaming decode instead of decode-only spans.	ds4.c:42616
+runtime/metal	DS4_METAL_GLM_STREAMING_DECODE_SYNC_EACH_LAYER	boolean text; Metal runtime always synchronizes and does not consult it; legacy fallback name read only in ROCm builds	Controls per-layer GLM streaming decode synchronization only as a legacy ROCm fallback alias.	ds4.c:49780
+runtime/metal	DS4_METAL_GLM_STREAMING_PREFILL_FULL_LAYER	presence control; unset: off/default; any value including 0 enables	Forces full-layer GLM SSD prefill regardless of the token crossover.	ds4_metal.m:14710
+runtime/metal	DS4_METAL_GLM_STREAMING_PREFILL_FULL_LAYER_MIN_TOKENS	positive uint32; default 64 on Metal, 1024 when used as ROCm fallback; 0/invalid restores default	Sets the token crossover for GLM full-layer SSD prefill.	ds4.c:42693
+runtime/metal	DS4_METAL_GLM_STREAMING_PREFILL_SYNC_EACH_LAYER	boolean text; Metal runtime always synchronizes and does not consult it; legacy fallback name read only in ROCm builds	Controls per-layer GLM streaming prefill synchronization only as a legacy ROCm fallback alias.	ds4.c:42396
+runtime/metal	DS4_METAL_GLM_STREAMING_TOKEN_PREFILL_MAX	uint32; default 64 on Metal, 0 when used as ROCm fallback; 0 disables; invalid restores default	Sets largest GLM SSD prefill handled token-major by the decode graph.	ds4.c:49683
+runtime/metal	DS4_METAL_GLU_SOURCE	file path; unset/empty: use the in-tree Metal source file	Overrides the GLU Metal kernel source file loaded at runtime.	ds4_metal.m:4950
+runtime/metal	DS4_METAL_GPU_BATCH_EMBED_MIN	uint32 token threshold; default 512; invalid restores default	Sets the batch size at which prompt embedding moves from CPU upload to Metal kernels.	ds4.c:28867
+runtime/metal	DS4_METAL_GPU_BUSY_PROFILE	presence diagnostic; unset: off; any value including 0 enables	Prints timing/profile diagnostics for GPU busy.	ds4_metal.m:1293
+runtime/metal	DS4_METAL_GRAPH_DUMP_LAYER	unsigned layer index or all; unset: every layer	Restricts graph tensor dumps to one layer.	ds4.c:16863
+runtime/metal	DS4_METAL_GRAPH_DUMP_LOGITS	file path; unset/empty: no graph-logit dump	Writes Metal graph-test logits as f32 binary.	ds4.c:39080
+runtime/metal	DS4_METAL_GRAPH_DUMP_NAME	substring; unset/empty: every tensor name	Restricts graph tensor dumps by tensor-name substring.	ds4.c:16859
+runtime/metal	DS4_METAL_GRAPH_DUMP_POS	unsigned token position; unset: every position	Restricts graph tensor dumps to one token position.	ds4.c:16870
+runtime/metal	DS4_METAL_GRAPH_DUMP_PREFIX	path/prefix; unset/empty: tensor dumping disabled	Enables graph tensor dumps and supplies the filename prefix.	ds4.c:65148
+runtime/metal	DS4_METAL_GRAPH_DUMP_TRACE	presence diagnostic; unset: off; any value including 0 enables	Emits trace diagnostics for graph dump.	ds4.c:16900
+runtime/metal	DS4_METAL_GRAPH_OUTPUT_ROW	zero-based row smaller than current batch; default final row; invalid restores final row	Chooses which prefill output row is projected to logits.	ds4.c:35846
+runtime/metal	DS4_METAL_GRAPH_PREFILL_PROFILE	presence diagnostic; unset: off; any value including 0 enables	Prints timing/profile diagnostics for graph prefill.	ds4.c:35523
+runtime/metal	DS4_METAL_GRAPH_PREFILL_SPLIT_PROFILE	presence diagnostic; unset: off; any value including 0 enables	Prints timing/profile diagnostics for graph prefill split.	ds4.c:66204
+runtime/metal	DS4_METAL_GRAPH_PROMPT_TOKENS	integer 1..prompt length; default full prompt	Limits prompt length used by the Metal graph parity test.	ds4.c:39023
+runtime/metal	DS4_METAL_GRAPH_RAW_CAP	positive rows; default from SWA window+prefill; clamped to [raw_window,min(ctx,8192)]	Overrides raw sliding-window KV ring capacity.	ds4.c:38390
+runtime/metal	DS4_METAL_GRAPH_TEACHER_FORCE	presence control; unset: off/default; any value including 0 enables	Feeds CPU reference state back into the first-token graph trace at each layer.	ds4.c:27742
+runtime/metal	DS4_METAL_GRAPH_TOKEN_PROFILE	presence diagnostic; unset: off; any value including 0 enables	Prints timing/profile diagnostics for graph token.	ds4.c:32382
+runtime/metal	DS4_METAL_GRAPH_TOKEN_SECOND_SPLIT_LAYERS	integer 0..layer count; default 0 plus eligible automatic pre-M5 schedules; explicit value wins	Overrides second command-buffer split layer for token decode.	ds4.c:28029
+runtime/metal	DS4_METAL_GRAPH_TOKEN_SPLIT_LAYERS	integer 0..layer count; default 4 on Apple and 0 elsewhere, with eligible pre-M5 adaptive override	Overrides first command-buffer split layer for token decode.	ds4.c:27915
+runtime/metal	DS4_METAL_GRAPH_TRACE_CACHE	presence control; unset: off/default; any value including 0 enables	Prints raw KV cache parity diagnostics in the graph prompt test.	ds4.c:39092
+runtime/metal	DS4_METAL_GRAPH_TRACE_COMP	presence control; unset: off/default; any value including 0 enables	Prints compressed-cache parity diagnostics in the graph prompt test.	ds4.c:39093
+runtime/metal	DS4_METAL_GRAPH_TRACE_LAYERS	presence control; unset: off/default; any value including 0 enables	Enables per-layer first-token CPU/GPU graph tracing.	ds4.c:27739
+runtime/metal	DS4_METAL_GRAPH_TRACE_STAGE_LAYER	signed layer index; unset gives -1/no stage-layer selection	Selects the layer used by first-token stage tracing.	ds4.c:27743
+runtime/metal	DS4_METAL_HC_NORM_FUSION_CHECK	nonempty boolean; unset/empty or exact 0: off; every other value: on	Compares fused HC normalization against the reference result.	ds4.c:20521
+runtime/metal	DS4_METAL_HC_NORM_FUSION_CHECK_TOL	positive finite float; default 2e-4; invalid/nonpositive restores default	Sets the numerical tolerance for the HC norm-fusion oracle.	ds4.c:20530
+runtime/metal	DS4_METAL_HC_STABLE	boolean empty/1/true/yes/on vs 0/false/no/off; default on	Compiles stable hidden-context drift arithmetic into the Metal library.	ds4_metal.m:7047
+runtime/metal	DS4_METAL_INDEXER_STAGE_PROFILE	presence diagnostic; unset: off; any value including 0 enables	Prints timing/profile diagnostics for indexer stage.	ds4.c:17569
+runtime/metal	DS4_METAL_IQ2_XXS_SSD_PREFILL_MM_STATS	value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables	Collects/prints statistics for IQ2 XXS SSD prefill MM.	ds4_metal.m:6980
+runtime/metal	DS4_METAL_KV_RAW_F32	boolean empty/1/true/yes/on vs 0/false/no/off; default off	Compiles raw KV storage as F32 for drift diagnosis.	ds4_metal.m:7049
+runtime/metal	DS4_METAL_LAYER_STAGE_PROFILE	unset: off; 1/true/yes/on/all enables all layers; a layer index selects one; 0/false/no/off disables	Prints timing/profile diagnostics for layer stage.	ds4.c:65150
+runtime/metal	DS4_METAL_LAYER_STAGE_PROFILE_LAYER	single unsigned layer index; unset/empty: all layers enabled by the parent profile; invalid matches no layer	Restricts the corresponding shared graph stage profiler to one layer.	ds4.c:29109
+runtime/metal	DS4_METAL_MATH_SAFE	boolean empty/1/true/yes/on vs 0/false/no/off; default off	Compiles Metal shaders with strict/safe IEEE math instead of fast math.	ds4_metal.m:7051
+runtime/metal	DS4_METAL_MEMORY_REPORT	presence control; unset: off/default; any value including 0 enables	Prints Metal allocation/cache/residency memory reports.	ds4.c:39047
+runtime/metal	DS4_METAL_MODEL_UNTRACKED	presence control; unset: off/default; any value including 0 enables	Creates mapped model buffers with untracked Metal hazard tracking.	ds4_metal.m:1547
+runtime/metal	DS4_METAL_MODEL_VIEW_MAX_GIB	positive integer GiB; default device maximum (128-GiB cap for already-split span maps); cannot exceed device maximum	Caps each no-copy mapped Metal model view.	ds4_metal.m:2216
+runtime/metal	DS4_METAL_MODEL_WARMUP_STRIDE_KB	integer 1..1048576 KiB, at least one page; unset inherits MB/default; overrides STRIDE_MB	Sets the model-view warmup touch stride with KiB precision.	ds4_metal.m:3057
+runtime/metal	DS4_METAL_MODEL_WARMUP_STRIDE_MB	integer 1..1024 MiB; default 1 MiB; STRIDE_KB overrides	Sets the model-view warmup touch stride.	ds4_metal.m:3049
+runtime/metal	DS4_METAL_MOE_MM_ID_USE_RESOURCES	presence control; unset: off/default; any value including 0 enables	Declares MM-ID MoE resource usage explicitly on the command encoder.	ds4_metal.m:35168
+runtime/metal	DS4_METAL_MOE_ONE_STAGE_PROFILE	unset: off; 1/true/yes/on/all enables all layers; accepts layer lists/ranges; 0/false/no/off disables	Prints timing/profile diagnostics for MoE one stage.	ds4.c:22714
+runtime/metal	DS4_METAL_MOE_ONE_STAGE_PROFILE_LAYER	layer index/list/ranges or all; unset: all profiler-selected layers	Restricts one-stage MoE profiling to selected layers.	ds4_metal.m:43436
+runtime/metal	DS4_METAL_MOE_SOURCE	file path; unset/empty: use the in-tree Metal source file	Overrides the MoE Metal kernel source file loaded at runtime.	ds4_metal.m:4937
+runtime/metal	DS4_METAL_MOE_STAGE_PROFILE	unset: off; 1/true/yes/on/all enables all layers; accepts layer lists/ranges; 0/false/no/off disables	Prints timing/profile diagnostics for MoE stage.	ds4.c:65154
+runtime/metal	DS4_METAL_MOE_STAGE_PROFILE_FILTER	substring; unset/empty: all profiled stages	Filters MoE stage-profile output.	ds4_metal.m:43438
+runtime/metal	DS4_METAL_MOE_STAGE_PROFILE_LAYER	layer index/list/ranges or all; unset: all profiler-selected layers	Restricts batched MoE stage profiling to selected layers.	ds4_metal.m:45346
+runtime/metal	DS4_METAL_MOE_WRITE_CLAMPED_ACT	presence control; unset: off/default; any value including 0 enables	Makes routed MoE write the clamped activation diagnostic.	ds4.c:18527
+runtime/metal	DS4_METAL_NORM_RSQRT_DISABLE	boolean empty/1/true/yes/on vs 0/false/no/off; default on	Compiles unified normalization-rsqrt arithmetic into the Metal library.	ds4_metal.m:7048
+runtime/metal	DS4_METAL_NORM_SOURCE	file path; unset/empty: use the in-tree Metal source file	Overrides the normalization Metal kernel source file loaded at runtime.	ds4_metal.m:4951
+runtime/metal	DS4_METAL_NO_MODEL_WARMUP	presence rollback; unset: automatic/default path; any value including 0 disables	Disables model warmup.	ds4_metal.m:2309
+runtime/metal	DS4_METAL_NO_PREFILL_KERNEL_WARMUP	presence rollback; unset: automatic/default path; any value including 0 disables	Disables prefill kernel warmup.	ds4.c:28965
+runtime/metal	DS4_METAL_NO_RESIDENCY	presence rollback; unset: automatic/default path; any value including 0 disables	Disables residency.	ds4_metal.m:2092
+runtime/metal	DS4_METAL_OUTPUT_STAGE_PROFILE	presence diagnostic; unset: off; any value including 0 enables	Prints timing/profile diagnostics for output stage.	ds4.c:17570
+runtime/metal	DS4_METAL_PREFILL_CHUNK	positive token count used only when CLI chunk is absent; default full prompt, or 4096 for long non-PRO and 8192 for long PRO prompts; <=0 keeps automatic/full prompt	Provides the historical environment fallback for prefill chunk size.	ds4.c:12802
+runtime/metal	DS4_METAL_PRO_Q4_CPU_ROUTER	nonempty boolean; unset/empty or exact 0: off; every other value: on	Uses the CPU router for PRO Q4 selected-expert decode.	ds4.c:20934
+runtime/metal	DS4_METAL_PRO_Q4_CPU_ROUTER_PROFILE	presence diagnostic; unset: off; any value including 0 enables	Prints timing/profile diagnostics for pro Q4 CPU router.	ds4.c:21644
+runtime/metal	DS4_METAL_Q4_ADDR_USE_RESOURCES	presence control; unset: off/default; any value including 0 enables	Declares Q4 address-table resources explicitly on encoders.	ds4_metal.m:20259
+runtime/metal	DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MB	integer 1..65536 MiB; default 3072; above max clamps, below min/invalid restores default	Caps copied F16 sidecar storage for resident Q4_K attn_q_b weights.	ds4_metal.m:25861; ds4_metal.m:26210
+runtime/metal	DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MIN_TOKENS	integer 32..UINT32_MAX; default 512; below min/invalid restores default	Sets the minimum resident, or explicitly enabled SSD-hybrid, prefill batch that may build or use the Q4_K attn_q_b F16 sidecar; smaller tails remain non-candidates.	ds4_metal.m:25823; ds4_metal.m:26138
+runtime/metal	DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_PROFILE	presence diagnostic; unset: off; any value including 0 enables	Prints resident Q4_K attn_q_b F16 cache counters and circuit state at Metal cleanup.	ds4_metal.m:11494
+runtime/metal	DS4_METAL_Q4_ATTN_Q_B_TRANSIENT_F16_MIN_TOKENS	integer token count; default 4096; invalid or values below 32 restore 4096	Sets the minimum long-N resident or SSD-streamed prefill batch eligible for per-layer transient Q4_K attn_q_b-to-F16 scratch; smaller chunks and tails stay on native Q4.	ds4_metal.m:26065; ds4_metal.m:26944
+runtime/metal	DS4_METAL_Q4_EXPERT_GROUP_SIZE	positive uint32; default 32; clamped to total expert count	Sets experts processed per grouped Q4 dispatch.	ds4_metal.m:34088
+runtime/metal	DS4_METAL_Q4_EXPERT_TABLE_GROUP_SIZE	integer 2..total experts; default/invalid 1 (ungrouped)	Sets grouped exact-view width while building Q4 expert tables.	ds4_metal.m:19604
+runtime/metal	DS4_METAL_Q4_EXPERT_TABLE_PROFILE	presence diagnostic; unset: off; any value including 0 enables	Prints timing/profile diagnostics for Q4 expert table.	ds4_metal.m:20103
+runtime/metal	DS4_METAL_Q4_GROUP24_BASE_VIEWS	presence control; unset: off/default; any value including 0 enables	Uses broad base model views for Q4 group-24 instead of exact views.	ds4_metal.m:42205
+runtime/metal	DS4_METAL_Q4_GROUP24_EXACT_VIEWS	presence control; unset: off/default; any value including 0 enables	Uses exact mapped views for Q4 group-24 experts.	ds4_metal.m:42204
+runtime/metal	DS4_METAL_Q4_GROUPED_CACHE_VIEWS	presence control; unset: off/default; any value including 0 enables	Caches exact Q4 grouped expert views.	ds4_metal.m:42151
+runtime/metal	DS4_METAL_Q4_PRO_MAP_GROUPS	positive divisor of 384 in 1..384; default/invalid 1	Splits each 384-expert PRO Q4 tensor into this many mapped views.	ds4.c:6276
+runtime/metal	DS4_METAL_Q4_SELECTED_EXACT_VIEWS	presence control; unset: off/default; any value including 0 enables	Forces exact/cached views for selected Q4 experts instead of base views.	ds4_metal.m:42528
+runtime/metal	DS4_METAL_Q4_SELECTED_OVERLAP_SHARED	nonempty boolean; unset/empty or exact 0: off; every other value: on	Overlaps selected Q4 expert preparation with the shared expert.	ds4.c:20962
+runtime/metal	DS4_METAL_Q4_SELECTED_PROFILE	presence diagnostic; unset: off; any value including 0 enables	Prints timing/profile diagnostics for Q4 selected.	ds4.c:37944
+runtime/metal	DS4_METAL_Q4_SELECTED_PROFILE_LAYER	single nonnegative layer index; unset: every layer	Restricts legacy Q4 selected-expert profiling to one layer.	ds4_metal.m:42509
+runtime/metal	DS4_METAL_Q4_SELECTED_SHARED_EVENT	presence control; unset: off/default; any value including 0 enables	Coordinates selected Q4 work with a shared Metal event.	ds4_metal.m:42524
+runtime/metal	DS4_METAL_Q4_SELECTED_TRANSIENT_VIEWS	presence control; unset: off/default; any value including 0 enables	Uses transient exact views for selected Q4 experts.	ds4_metal.m:42532
+runtime/metal	DS4_METAL_Q4_SELECTED_USE_BASE_VIEWS	presence control; unset: off/default; any value including 0 enables	Uses broad base model views for selected Q4 experts.	ds4_metal.m:42527
+runtime/metal	DS4_METAL_Q4_TABLE_BIND_ANCHORS	presence control; unset: off/default; any value including 0 enables	Binds anchor buffers alongside the Q4 expert address table.	ds4_metal.m:19716
+runtime/metal	DS4_METAL_Q4_TABLE_MODEL_RESIDENCY_SET	presence control; unset: off/default; any value including 0 enables	Adds Q4 expert table allocations to the model residency set.	ds4_metal.m:19657
+runtime/metal	DS4_METAL_Q4_TABLE_PER_TENSOR_RESIDENCY_SET	presence control; unset: off/default; any value including 0 enables	Builds separate residency sets per Q4 expert tensor.	ds4_metal.m:19760
+runtime/metal	DS4_METAL_Q4_TABLE_QUEUE_RESIDENCY_SET	presence control; unset: off/default; any value including 0 enables	Attaches Q4 expert table residency sets to command queues.	ds4_metal.m:19615
+runtime/metal	DS4_METAL_Q4_TABLE_RESIDENCY_SET	presence control; unset: off/default; any value including 0 enables	Enables Q4 expert table residency-set handling.	ds4_metal.m:19759
+runtime/metal	DS4_METAL_Q4_TABLE_USE_RESOURCES	presence control; unset: off/default; any value including 0 enables	Declares Q4 table resources explicitly on encoders.	ds4_metal.m:20258
+runtime/metal	DS4_METAL_Q8_DECODE_EXACT_VIEW_MAX_MIB	integer 1..4096 MiB; default 1024; above max clamps, below min/invalid restores default	Caps weight ranges eligible for Q8 exact model views.	ds4_metal.m:12876
+runtime/metal	DS4_METAL_Q8_MV_EXT_MAX_TOKENS	integer 2..128; default 16; above max clamps, below min/invalid restores default	Sets largest batch handled by extended Q8 matvec.	ds4_metal.m:21126
+runtime/metal	DS4_METAL_Q8_MV_NSG	integer 1..8 simdgroups; default 4, or 2 with TP world=2; above max clamps, below min/invalid restores default	Overrides simdgroups per Q8 matvec threadgroup.	ds4.c:22730
+runtime/metal	DS4_METAL_Q8_PREFILL_PROFILE	value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables	Prints timing/profile diagnostics for Q8 prefill.	ds4_metal.m:21280
+runtime/metal	DS4_METAL_Q8_PREFILL_PROFILE_FILTER	substring matched against generated operation label; unset/empty: all eligible calls	Filters Q8 prefill profiling.	ds4_metal.m:21295
+runtime/metal	DS4_METAL_Q_STAGE_PROFILE	presence diagnostic; unset: off; any value including 0 enables	Prints timing/profile diagnostics for q stage.	ds4.c:29270
+runtime/metal	DS4_METAL_REPEAT_SOURCE	file path; unset/empty: use the in-tree Metal source file	Overrides the repeat Metal kernel source file loaded at runtime.	ds4_metal.m:4949
+runtime/metal	DS4_METAL_REQUIRE_COMPRESSOR_EXACT_POOL_RATIO4	presence strict check; unset: fallback allowed; any value including 0 requires the path	Requires compressor exact pool ratio4 and makes eligible fallback fail closed.	ds4_metal.m:26387
+runtime/metal	DS4_METAL_REQUIRE_DECODE_ARGMAX_TOP1	presence fail-closed assertion for rows of at least 4096 logits; unset permits generic fallback; any defined value including empty or 0 rejects disabled, ineligible, or failed dedicated top-1 preflight	Require the dedicated Metal top-1 reduction so correctness and performance oracles cannot silently exercise generic argsort.	ds4_metal.m:21515
+runtime/metal	DS4_METAL_REQUIRE_EXACT_ROWS_PERSISTENT_CACHE	nonempty boolean; unset/empty or exact 0: off; every other value: on	Requires exact rows persistent cache and makes eligible fallback fail closed.	ds4_metal.m:13002
+runtime/metal	DS4_METAL_REQUIRE_GATHERED_KV_STAGE	presence strict check; unset: fallback allowed; any value including 0 requires the path	Requires gathered KV stage and makes eligible fallback fail closed.	ds4_metal.m:29680
+runtime/metal	DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM	value-aware boolean; default implicit fail-closed only with complete selected-address domain; explicit 1 is strict, 0 permits fallback	Makes eligible IQ2_XXS/Q2_K grouped SSD-prefill MM fail closed.	ds4_metal.m:44782
+runtime/metal	DS4_METAL_REQUIRE_M1_IQ2_MID_ONLY	presence strict check; unset: fallback allowed; any value including 0 requires the path	Requires M1 IQ2 mid only and makes eligible fallback fail closed.	ds4_metal.m:42074
+runtime/metal	DS4_METAL_REQUIRE_OUTPUT_HC_WEIGHTS4	presence strict check; unset: fallback allowed; any value including 0 requires the path	Requires output HC weights4 and makes eligible fallback fail closed.	ds4_metal.m:46789
+runtime/metal	DS4_METAL_REQUIRE_PRE_M5_BATCH_ATTN_OUT_HC_FUSION	presence strict check; unset permits fallback; any value including 0 requires the exact resident Q8_0 output-B-to-HC4 tail	Fail closed instead of replaying the separate Q8 output-B and HC path when the fused candidate is ineligible or fails.	ds4_metal.m:30536
+runtime/metal	DS4_METAL_REQUIRE_Q4_ATTN_OUT_A_DIRECT	presence strict check; unset permits the automatic path or fallback; any defined value including 0 requires the direct kernel and DISABLE wins	Require the bit-identical fixed-route Q4 attention output-A kernel and fail closed before dispatch when its production contract is unavailable.	ds4_metal.m:31560
+runtime/metal	DS4_METAL_REQUIRE_Q4_ATTN_OUT_B_F16_RHS	value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables	Requires the resident pre-M5 Q4 attention output-B F16 RHS path and makes an ineligible or disabled candidate fail closed.	ds4_metal.m:28551
+runtime/metal	DS4_METAL_REQUIRE_Q4_ATTN_OUT_TINY_BATCH	value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables	Requires Q4 attn out tiny batch and makes eligible fallback fail closed.	ds4_metal.m:28373
+runtime/metal	DS4_METAL_REQUIRE_Q4_ATTN_Q_B_F16_CACHE	value-aware boolean; unset/0: fallback allowed; empty/1/true/yes/on requires; DISABLE fails closed	Requires the sidecar for resident, or explicitly enabled SSD-hybrid, pre-M5 Q4_K Flash attn_q_b batches at or above the configured minimum; decode and below-min batches remain non-candidates.	ds4_metal.m:25801; ds4_metal.m:26129; ds4_metal.m:26704
+runtime/metal	DS4_METAL_REQUIRE_Q4_BATCH_ATTN_OUT_HC_FUSION	presence strict check; unset permits fallback; any value including 0 requires the exact resident Q4_K/F16-RHS output-B-to-HC4 tail	Fail closed instead of replaying the separate Q4 output-B and HC path when the fused candidate is ineligible or fails.	ds4_metal.m:31455
+runtime/metal	DS4_METAL_REQUIRE_Q4_PREFILL_PAIR_F16_RHS	value-aware strict opt-in; unset/0/false/no/off is off; empty/1/true/yes/on or another nonempty value requires the candidate for prefill pair calls; unsupported device/shape, DISABLE, or an attempted-path error fails closed	Require the Apple M1-M4 shared-F16-RHS q_a/KV Q4_K prefill pair instead of silently running two standalone matmuls.	ds4_metal.m:22913
+runtime/metal	DS4_METAL_REQUIRE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN	value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables	Requires Q4 SSD prefill attn out exactn and makes eligible fallback fail closed.	ds4_metal.m:28089
+runtime/metal	DS4_METAL_REQUIRE_Q4_SSD_PREFILL_ATTN_OUT_SCALE_META	value-aware boolean; unset: off; empty/1/true/yes/on enables; 0/false/no/off disables	Requires shared scale/min metadata in the Q4 SSD prefill attention-output exact-N kernel and makes fallback fail closed.	ds4_metal.m:28209
+runtime/metal	DS4_METAL_REQUIRE_Q4_SSD_SESSION_UNION	nonempty boolean; unset/empty or exact 0: off; every other value: on	Requires Q4 SSD session union and makes eligible fallback fail closed.	ds4.c:65164
+runtime/metal	DS4_METAL_REQUIRE_Q8_QKV_COMPRESSOR_FUSE	nonempty boolean; unset/empty/exact 0: fallback allowed; other values require and imply the streamed/union enable	Requires eligible Q8 QKV/compressor compound fusion and fails closed.	ds4.c:23023
+runtime/metal	DS4_METAL_RESUME_PREFILL_MIN	integer token threshold; default 4; <=0 disables resume-prefill	Sets the minimum shared-prefix suffix that uses batched resume-prefill.	ds4.c:38419
+runtime/metal	DS4_METAL_ROPE_EXP2_LOG2	boolean empty/1/true/yes/on vs 0/false/no/off; default off	Compiles the exp2/log2 RoPE drift variant.	ds4_metal.m:7050
+runtime/metal	DS4_METAL_SELECTED_PROFILE	presence diagnostic; unset: off; any value including 0 enables	Prints timing/profile diagnostics for selected.	ds4.c:37943
+runtime/metal	DS4_METAL_SELECTED_PROFILE_LAYER	single nonnegative layer index; unset: every layer	Restricts selected-expert profiling to one layer.	ds4_metal.m:42507
+runtime/metal	DS4_METAL_SESSION_BATCH_LOG	presence diagnostic; unset: off; any value including 0 enables	Logs session batch decisions.	ds4.c:66168
+runtime/metal	DS4_METAL_SESSION_BATCH_QKV	default enabled; exact 0 disables; every other value/unset leaves enabled	Controls native batched QKV work for multi-session decode.	ds4.c:65482
+runtime/metal	DS4_METAL_SESSION_BATCH_SHARED	default enabled; exact 0 disables; every other value/unset leaves enabled	Controls native batched shared-expert work for multi-session decode.	ds4.c:65432
+runtime/metal	DS4_METAL_SET_ROWS_SOURCE	file path; unset/empty: use the in-tree Metal source file	Overrides the set-rows Metal kernel source file loaded at runtime.	ds4_metal.m:4953
+runtime/metal	DS4_METAL_SOFTMAX_SOURCE	file path; unset/empty: use the in-tree Metal source file	Overrides the softmax Metal kernel source file loaded at runtime.	ds4_metal.m:4948
+runtime/metal	DS4_METAL_STREAMING_DECODE_PREFILL_MAX	integer token maximum; default 64 for wide Flash Q4/MXFP4, 18 for other PRO/Flash, 0 otherwise; <=0 disables	Sets maximum SSD-streaming micro-prefill width that reuses decode.	ds4.c:31962
+runtime/metal	DS4_METAL_STREAMING_EXPERT_AUTO_PRELOAD_CAP	uint32 expert cap; default 4096; 0 means unlimited; invalid restores default	Caps automatic streaming-expert hotlist preload.	ds4.c:21455
+runtime/metal	DS4_METAL_STREAMING_EXPERT_BUFFER_MLOCK_PROFILE	presence diagnostic; unset: off; any value including 0 enables	Prints timing/profile diagnostics for streaming expert buffer mlock.	ds4_metal.m:13997
+runtime/metal	DS4_METAL_STREAMING_EXPERT_EARLY_LOAD_PROFILE	presence diagnostic; unset: off; any value including 0 enables	Prints timing/profile diagnostics for streaming expert early load.	ds4_metal.m:17059
+runtime/metal	DS4_METAL_STREAMING_EXPERT_EVICT_DONTNEED_PROFILE	presence diagnostic; unset: off; any value including 0 enables	Prints timing/profile diagnostics for streaming expert evict dontneed.	ds4_metal.m:14436
+runtime/metal	DS4_METAL_STREAMING_EXPERT_HOTLIST	hotlist file path; unset/empty: built-in model hotlist	Loads the streaming-expert preload order from a file.	ds4.c:21495
+runtime/metal	DS4_METAL_STREAMING_EXPERT_HOTLIST_PROFILE	presence diagnostic; unset: off; any value including 0 enables	Prints timing/profile diagnostics for streaming expert hotlist.	ds4.c:32245
+runtime/metal	DS4_METAL_STREAMING_EXPERT_LAYER_STATS	presence diagnostic; unset: off; any value including 0 enables	Collects/prints statistics for streaming expert layer.	ds4_metal.m:4638
+runtime/metal	DS4_METAL_STREAMING_EXPERT_LAYER_STATS_DELTA	presence control; unset: off/default; any value including 0 enables	Prints delta statistics for streaming expert layer.	ds4_metal.m:4675
+runtime/metal	DS4_METAL_STREAMING_EXPERT_NOCACHE	nonempty value whose first character is not 0 enables; unset/empty/0 disables	Uses a reopened F_NOCACHE descriptor for SSD expert preads.	ds4_metal.m:12602
+runtime/metal	DS4_METAL_STREAMING_EXPERT_PREAD_POOL	default enabled; exact 0 disables; every other value/unset keeps enabled	Controls reuse of persistent expert-pread worker threads.	ds4_metal.m:13433
+runtime/metal	DS4_METAL_STREAMING_EXPERT_PREAD_PROFILE	presence diagnostic; unset: off; any value including 0 enables	Prints timing/profile diagnostics for streaming expert pread.	ds4_metal.m:17005
+runtime/metal	DS4_METAL_STREAMING_EXPERT_PREAD_SPLIT	integer clamped 1..8; unset: automatic 1 below 64 cache experts, 4 at 64+	Sets aligned requests per expert pread.	ds4_metal.m:13701
+runtime/metal	DS4_METAL_STREAMING_EXPERT_PREAD_THREADS	unsigned integer clamped 1..18; default 9; invalid restores 9	Sets expert-pread worker limit.	ds4_metal.m:13337
+runtime/metal	DS4_METAL_STREAMING_EXPERT_PROFILE_SUMMARY	presence diagnostic; unset: off; any value including 0 enables	Prints timing/profile diagnostics for streaming expert.	ds4_metal.m:13061
+runtime/metal	DS4_METAL_STREAMING_EXPERT_SLAB_MB	positive unsigned MiB; default 4096; 0/invalid restores default	Sets target allocation size for streaming-expert slabs.	ds4_metal.m:14039
+runtime/metal	DS4_METAL_STREAMING_EXPERT_SPLIT_PROFILE	presence diagnostic; unset: off; any value including 0 enables	Prints timing/profile diagnostics for streaming expert split.	ds4_metal.m:43810
+runtime/metal	DS4_METAL_STREAMING_EXPERT_TIMING_SUMMARY	presence control; unset: off/default; any value including 0 enables	Prints timing/profile diagnostics for streaming expert.	ds4_metal.m:13060
+runtime/metal	DS4_METAL_STREAMING_IQ2_CPU_ROUTER_PROFILE	presence diagnostic; unset: off; any value including 0 enables	Prints timing/profile diagnostics for streaming IQ2 CPU router.	ds4.c:21645
+runtime/metal	DS4_METAL_STREAMING_MAP_TRACE	nonempty value other than exact 0 enables; unset/empty/0 disables	Emits SSD model-map decisions.	ds4_metal.m:4849
+runtime/metal	DS4_METAL_STREAMING_PREFILL_BATCH_SELECTED_ADDR_MAX	integer token maximum; default 800 for 384 experts, 760 for 256, 0 otherwise; <=0 disables automatic selection	Sets automatic maximum batch width for selected-address SSD prefill.	ds4_metal.m:14639
+runtime/metal	DS4_METAL_STREAMING_PREFILL_BATCH_SELECTED_ADDR_MIN	integer token minimum; default 2 for 256/384 experts, 0 otherwise; <=0 disables automatic selection	Sets automatic minimum batch width for selected-address SSD prefill.	ds4_metal.m:14656
+runtime/metal	DS4_METAL_STREAMING_PREFILL_BATCH_SELECTED_ADDR_PROFILE	presence diagnostic; unset: off; any value including 0 enables	Prints timing/profile diagnostics for streaming prefill batch selected address.	ds4_metal.m:18493
+runtime/metal	DS4_METAL_STREAMING_PREFILL_CACHE_SEED_K	uint32 seed rows; default 1; 0 disables; above 64 clamps to 64	Sets how many prefill routing rows seed the decode expert cache.	ds4.c:21268
+runtime/metal	DS4_METAL_STREAMING_PREFILL_CACHE_SEED_PROFILE	presence diagnostic; unset: off; any value including 0 enables	Prints timing/profile diagnostics for streaming prefill cache seed.	ds4_metal.m:17899
+runtime/metal	DS4_METAL_STREAMING_PREFILL_LAYER_MADVISE_PROFILE	presence diagnostic; unset: off; any value including 0 enables	Prints timing/profile diagnostics for streaming prefill layer madvise.	ds4.c:19610
+runtime/metal	DS4_METAL_STREAMING_PREFILL_LAYER_PAGEIN_NO_OVERLAP	presence rollback; unset: automatic/default path; any value including 0 disables	Prevents full-layer page-in preparation from overlapping compute.	ds4.c:19317
+runtime/metal	DS4_METAL_STREAMING_PREFILL_LAYER_PAGEIN_PROFILE	presence diagnostic; unset: off; any value including 0 enables	Prints timing/profile diagnostics for streaming prefill layer pagein.	ds4.c:19606
+runtime/metal	DS4_METAL_STREAMING_PREFILL_LAYER_PAGEIN_THREADS	integer 1..16; default 8; invalid/0 becomes 1; PREPARE_THREADS takes precedence	Sets worker count for full-layer page-in preparation.	ds4.c:19275
+runtime/metal	DS4_METAL_STREAMING_PREFILL_LAYER_PREAD_PROFILE	presence diagnostic; unset: off; any value including 0 enables	Prints timing/profile diagnostics for streaming prefill layer pread.	ds4.c:19608
+runtime/metal	DS4_METAL_STREAMING_PREFILL_LAYER_PREPARE_AHEAD	integer 1..4 layers; default 1; invalid/0 becomes 1	Sets number of future layers prepared concurrently.	ds4.c:19329
+runtime/metal	DS4_METAL_STREAMING_PREFILL_LAYER_PREPARE_NO_OVERLAP	presence rollback; unset: automatic/default path; any value including 0 disables	Prevents generic full-layer preparation from overlapping compute.	ds4.c:19315
+runtime/metal	DS4_METAL_STREAMING_PREFILL_LAYER_PREPARE_THREADS	integer 1..16; default 8; invalid/0 becomes 1; preferred over PAGEIN_THREADS	Sets worker count for full-layer preparation.	ds4.c:19271
+runtime/metal	DS4_METAL_STREAMING_PREFILL_LAYER_READAHEAD_PROFILE	presence diagnostic; unset: off; any value including 0 enables	Prints timing/profile diagnostics for streaming prefill layer readahead.	ds4.c:19612
+runtime/metal	DS4_METAL_STREAMING_PREFILL_SELECTED_MADVISE_PROFILE	presence diagnostic; unset: off; any value including 0 enables	Prints timing/profile diagnostics for streaming prefill selected madvise.	ds4.c:19356
+runtime/metal	DS4_METAL_STREAMING_PREFILL_SELECTED_MADVISE_THREADS	integer 1..16; default inherits layer prepare threads; invalid/0 becomes 1; PREPARE_THREADS preferred	Sets worker count for selected-expert madvise preparation.	ds4.c:19293
+runtime/metal	DS4_METAL_STREAMING_PREFILL_SELECTED_PAGEIN_PROFILE	presence diagnostic; unset: off; any value including 0 enables	Prints timing/profile diagnostics for streaming prefill selected pagein.	ds4.c:19354
+runtime/metal	DS4_METAL_STREAMING_PREFILL_SELECTED_PREPARE_GAP	integer 0..8 layers; default 0; above 8 clamps; invalid restores 0	Sets lookahead gap for selected-expert preparation.	ds4.c:19305
+runtime/metal	DS4_METAL_STREAMING_PREFILL_SELECTED_PREPARE_THREADS	integer 1..16 for madvise preparation; default inherits layer prepare threads; invalid/0 becomes 1	Sets worker count for selected-expert preparation.	ds4.c:19289
+runtime/metal	DS4_METAL_STREAMING_PREFILL_SELECTED_PROFILE	presence diagnostic; unset: off; any value including 0 enables	Prints timing/profile diagnostics for streaming prefill selected.	ds4.c:18906
+runtime/metal	DS4_METAL_STREAMING_PREFILL_SELECTED_READAHEAD_GAP	integer 0..8 layers; default 0; above 8 clamps; invalid restores 0	Sets lookahead gap for selected-expert readahead.	ds4.c:19978
+runtime/metal	DS4_METAL_STREAMING_PREFILL_SELECTED_READAHEAD_PROFILE	presence diagnostic; unset: off; any value including 0 enables	Prints timing/profile diagnostics for streaming prefill selected readahead.	ds4.c:20063
+runtime/metal	DS4_METAL_STREAMING_SELECTED_READAHEAD_PROFILE	presence diagnostic; unset: off; any value including 0 enables	Prints timing/profile diagnostics for streaming selected readahead.	ds4.c:21737
+runtime/metal	DS4_METAL_SUM_ROWS_SOURCE	file path; unset/empty: use the in-tree Metal source file	Overrides the sum-rows Metal kernel source file loaded at runtime.	ds4_metal.m:4947
+runtime/metal	DS4_METAL_TEST_POISON_COMPRESSOR_EXACT_REDUCTION_SCRATCH	internal test presence flag; unset: off; any value including 0 poisons scratch before the exact reduction	Validates that compressor exact-reduction kernels overwrite all scratch state.	ds4_metal.m:25925
+runtime/metal	DS4_METAL_TP_SESSION_BATCH	default enabled; exact 0 disables; every other value/unset leaves enabled	Controls batched session evaluation with Metal TP.	ds4.c:65310
+runtime/metal	DS4_METAL_TRACE_ALLOCS	presence diagnostic; unset: off; any value including 0 enables	Emits trace diagnostics for allocs.	ds4_metal.m:4057
+runtime/metal	DS4_METAL_TRACE_M5_FLASH_ATTN_PACKED32_REDUCE	presence diagnostic; unset: off; any value including 0 enables	Emits trace diagnostics for M5 flash attn packed32 reduce.	ds4_metal.m:31535
+runtime/metal	DS4_METAL_UNARY_SOURCE	file path; unset/empty: use the in-tree Metal source file	Overrides the unary operations Metal kernel source file loaded at runtime.	ds4_metal.m:4939
+runtime/metal	DS4_METAL_UNRETAINED_COMMAND_BUFFERS	presence control; unset: off/default; any value including 0 enables	Creates Metal command buffers with unretained references.	ds4_metal.m:1315
+runtime/metal	DS4_METAL_USE_QUEUE_RESIDENCY_SET	presence control; unset: off/default; any value including 0 enables	Allows queue-residency state to trigger Q4 expert address/table paths.	ds4_metal.m:42280
+runtime/moe-debug	DS4_MOE_RECORD_SELECTED_HOTLIST	nonempty output path; unset=off	Record per-layer selected-expert hit counts to a Metal hotlist file.	ds4_metal.m:1741
+runtime/moe-debug	DS4_MOE_RECORD_SELECTED_HOTLIST_FRESH	presence flag; only relevant with HOTLIST; overrides MERGE	Start the selected-expert hotlist from empty state.	ds4_metal.m:1642
+runtime/moe-debug	DS4_MOE_RECORD_SELECTED_HOTLIST_MERGE	presence flag; active only when FRESH is absent	Merge an existing selected-expert hotlist before recording.	ds4_metal.m:1641
+runtime/moe-debug	DS4_MOE_RECORD_SELECTED_IDS	nonempty output path; unset=off	Record routed-MoE six-expert selections; also disables incompatible optimized paths.	ds4.c:65290
+runtime/moe-debug	DS4_MOE_REPLAY_SELECTED_IDS	nonempty input path; unset=off	Replay routed-MoE six-expert selections; also disables incompatible optimized paths.	ds4.c:25308
+runtime/mtp	DS4_MTP_BATCH_VERIFY	Pure presence flag: any defined value, including empty or "0", suppresses the exact two-row decode verifier. Unset selects exact decode-2 when draft_n==2 and either strict mode is active or the build is ROCm; other cases already use the generic verifier.	Diagnostic rollback from the exact Q8/one-token-equivalent MTP decode-2 verifier to the generic microbatch verifier.	ds4.c:71021
+runtime/mtp	DS4_MTP_CAPTURE_PREFIX1	Pure presence flag. In the generic verifier with exactly two drafts it enables prefix-1 state capture under strict mode; non-strict mode already captures prefix-1 without the variable. Unset under strict mode instead snapshots and replays a partial acceptance.	Let a one-of-two MTP partial acceptance commit the verifier's captured prefix directly, avoiding an exact one-token replay.	ds4.c:71134
+runtime/mtp	DS4_MTP_CONF_LOG	Pure presence flag; default off. It forces materialization of full draft logits, computes the top-2 margin, and after a successful generic microbatch verification prints drafted/committed counts, top candidates, margin, target-next and draft-next. Exact decode-2 success does not emit that generic log line.	Inspect MTP draft confidence and compare the recursive draft token with the target verifier result.	ds4.c:70900
+runtime/mtp	DS4_MTP_EXACT_REPLAY	Pure presence flag; default off. In the generic microbatch verifier it forces a pre-verifier frontier snapshot; after verification the snapshot is restored and every accepted draft is decoded sequentially to rebuild exact final state/logits.	Validate MTP acceptance while committing through the normal one-token decode path rather than retaining batched-verifier state.	ds4.c:71139
+runtime/mtp	DS4_MTP_FORCE_SNAPSHOT	Pure presence flag; default off. It forces a speculative-frontier snapshot before the generic verifier regardless of draft count or prefix-capture mode; it does not by itself force restoration or replay after a successful full acceptance.	Measure/debug snapshot behavior and guarantee a restorable pre-verifier frontier for generic MTP verification.	ds4.c:71143
+runtime/mtp	DS4_MTP_FULL_LOGITS	Pure presence flag; default off. When set, legacy and recursive MTP draft calls write the full vocabulary logits to s->mtp_logits; unset permits the faster top-token-only output unless confidence/margin logic independently needs logits.	Force full MTP draft-logit materialization for correctness comparison, inspection, or downstream confidence calculations.	ds4.c:64260
+runtime/mtp	DS4_MTP_MIN_MARGIN	non-negative float; default engine --mtp-margin value	Set confidence margin threshold for speculative MTP verification.	ds4.c:70893
+runtime/mtp	DS4_MTP_PROBE	Pure presence flag; default off. For legacy MTP it prepares drafts even when configured depth<=1, compares the previous draft with the next committed token, and prints cumulative hit counts/failures; generated output is unchanged.	Measure legacy MTP next-token draft accuracy without enabling speculative acceptance.	ds4.c:64990
+runtime/mtp	DS4_MTP_SPEC_DISABLE	Pure presence flag: any defined value, including empty or "0", disables MTP speculative argmax in CLI/chat/server loops. Unset permits it for greedy temperature<=0 generation with draft depth>1; unrelated split-KV speculation can still be independently requested.	Fall back from MTP multi-token speculative evaluation to normal one-token session evaluation.	ds4_cli.c:580
+runtime/mtp	DS4_MTP_SPEC_LOG	Pure presence flag; default off. It only emits diagnostics for first-draft misses, exact/generic verifier failures and sequential fallback misses/acceptance outcomes; it does not select a verifier.	Trace why MTP drafts were accepted, partially accepted, rejected, or sent to sequential fallback.	ds4.c:71542
+runtime/mtp	DS4_MTP_STRICT	Pure presence flag; engine quality mode also enables strictness automatically. Strict mode skips the non-strict low-margin shortcut, selects exact decode-2 for two drafts unless DS4_MTP_BATCH_VERIFY is set, and disables default prefix-1 capture unless explicitly restored.	Force the exact/quality-oriented MTP verification policy on otherwise non-quality runs.	ds4.c:71517
+runtime/mtp	DS4_MTP_TIMING	Pure presence flag; default off. When set, timestamps and prints draft, snapshot, verifier, prefix/replay and total durations for the path taken; algorithm selection is otherwise unchanged.	Profile end-to-end MTP speculative decoding and separate draft, verification and state-commit costs.	ds4.c:71525
+runtime/rocm	DS4_ROCM_DECODE_STAGE_PROFILE	presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose)	Collect timing/profile diagnostics for rocm decode stage profile.	ds4.c:18259
+runtime/rocm	DS4_ROCM_DECODE_STAGE_PROFILE_LAYER	layer filter subordinate to DS4_ROCM_DECODE_STAGE_PROFILE; unset or whitespace-only: all layers allowed by the parent flag; otherwise the whitespace-trimmed value must be a complete base-10 strtoul result <= UINT32_MAX equal to the current layer; invalid values match none	Restricts the ROCm decode stage profiler to one layer; it does not enable profiling by itself.	ds4.c:29131
+runtime/rocm	DS4_ROCM_DISABLE_BATCH_INDEXER_QUERY_PRUNE	presence rollback; unset prunes unused zero-prefix indexer query work on eligible resident ROCm prefills; any value including 0 disables	Restore transient indexer Q projection, RoPE, QAT, and weight projection before compressed rows exceed top-k.	ds4.c:30131
+runtime/rocm	DS4_ROCM_DISABLE_GFX1151_PREFILL_WMMA_ATTN	value-aware rollback evaluated on every eligible launch; unset/0/false/no/off is inactive, while empty or 1/true/yes/on disables; invalid values fail closed; true overrides both ENABLE variables	Force both experimental gfx1151 mixed-attention WMMA prefill paths back to their established scalar/online fallbacks for correctness or A/B measurement.	rocm/ds4_rocm_attention_launch.cuh:13
+runtime/rocm	DS4_ROCM_DISABLE_GLM_STREAMING_PREFILL_FULL_LAYER	integer selector/tuning value; unset or invalid uses internal automatic/default value	Disable/roll back rocm disable glm streaming prefill full layer.	ds4.c:42970
+runtime/rocm	DS4_ROCM_DISABLE_GLM_STREAMING_PREFILL_FULL_LAYER_PREPARE	presence rollback flag; unset keeps automatic/default path	Disable/roll back rocm disable glm streaming prefill full layer prepare.	ds4.c:42988
+runtime/rocm	DS4_ROCM_DISABLE_GLM_STREAMING_PREFILL_SELECTED_ASYNC_LOAD	presence rollback flag; unset keeps automatic/default path	Disable/roll back rocm disable glm streaming prefill selected async load.	ds4.c:46681
+runtime/rocm	DS4_ROCM_DISABLE_GLM_STREAMING_SELECTED_ASYNC_LOAD	presence rollback flag; unset keeps automatic/default path	Disable/roll back rocm disable glm streaming selected async load.	ds4.c:44589
+runtime/rocm	DS4_ROCM_DISABLE_IQ2_MOE_WMMA_TAIL_CULL	value-aware boolean evaluated on every eligible launch; unset is inactive; after trimming whitespace, empty or 1/true/yes/on enables and 0/false/no/off disables (words case-insensitive); invalid values disable the candidate with a one-time diagnostic; true overrides ENABLE	Force the opt-in tail-wave cull off for mixed IQ2_XXS gate/up plus Q2_K down hot-list rocWMMA prefill, providing the A/B baseline and rollback arm.	rocm/ds4_rocm_moe_launch.cuh:20
+runtime/rocm	DS4_ROCM_DISABLE_IQ2_SELECTED_EXPERT_VIEWS	presence rollback flag; unset keeps automatic/default path	Disable/roll back rocm disable iq2 selected expert views.	ds4.c:21094
+runtime/rocm	DS4_ROCM_DISABLE_IQ2_STREAM_ADDR_TABLE	presence rollback flag; unset keeps automatic/default path	Disable/roll back rocm disable iq2 stream addr table.	ds4.c:6548
+runtime/rocm	DS4_ROCM_DISABLE_Q4_ATTN_Q_B_F16_CACHE	value-aware rollback; unset/0/false/no/off keeps the experiment available, empty or any other value disables it	Disable the resident ROCm Q4_K attn_q_b-to-F16 prefill cache even when ENABLE or REQUIRE is set.	rocm/ds4_rocm_q4_qb_sidecar.cuh:140
+runtime/rocm	DS4_ROCM_DISABLE_Q4_ATTN_Q_B_TRANSIENT_F16	value-aware rollback; unset/0/false/no/off keeps the automatic path eligible, empty or any other non-false value disables it	Disable per-layer transient Q4_K attn_q_b-to-F16 scratch for device-image or device-range-resident weights; the existing resident-cache controls remain independent and otherwise eligible calls use native Q4.	rocm/ds4_rocm_q4_qb_sidecar.cuh:145
+runtime/rocm	DS4_ROCM_DISABLE_Q4_DENSE_PAIR	presence rollback; unset leaves opt-in policy unchanged	Disable/roll back rocm disable q4 dense pair.	rocm/ds4_rocm_q4.cuh:435
+runtime/rocm	DS4_ROCM_DISABLE_Q4_GROUPED_ATTN_A	presence rollback; unset permits the caller-marked resident decode production-shape default and explicit ENABLE/REQUIRE; any defined value including empty or 0 disables all grouped attention-A paths and wins over ENABLE/REQUIRE	Restore eight standalone Q4 attention-A projections instead of the two-dispatch grouped path.	rocm/ds4_rocm_q4.cuh:868
+runtime/rocm	DS4_ROCM_DISABLE_Q4_PREFILL_K1024_TILE4	value-aware authoritative rollback; unset/0/false/no/off preserves the resident automatic default and any explicit SSD request; empty or any other value disables; overrides ENABLE and causes REQUIRE to fail closed	Restore the generic eight-block Q4_K tiled-prefill kernel for K=1024 in both resident and SSD-streaming execution.	rocm/ds4_rocm_q4.cuh:624
+runtime/rocm	DS4_ROCM_DISABLE_Q4_PREFILL_Q8_K_WAVE32	value-aware authoritative rollback; unset/0/false/no/off permits ENABLE or REQUIRE, while empty or any other value disables; REQUIRE then fails closed	Restore the canonical one-workgroup-per-Q8_K-block activation quantizer for Q4 prefill.	rocm/ds4_rocm_q4.cuh:803
+runtime/rocm	DS4_ROCM_DISABLE_Q4_PREFILL_TILE8	presence rollback; TILE8 is default for 9..4096 tokens	Disable/roll back rocm disable q4 prefill tile8.	rocm/ds4_rocm_q4.cuh:448
+runtime/rocm	DS4_ROCM_DISABLE_Q4_PREFILL_WMMA	value-aware authoritative opt-out for the automatic resident path and explicit SSD/REQUIRE requests; unset/0/false/no/off leaves policy unchanged, while empty or any other value disables; REQUIRE then fails closed	Prevent the gfx1151 direct-Q4 WMMA prefill path from dispatching and retain the Q8_K-plus-TILE8/TILE4 path.	rocm/ds4_rocm_q4.cuh:1626
+runtime/rocm	DS4_ROCM_DISABLE_Q4_PREFILL_WMMA_K128	value-aware rollback for the default K128/P144 stage; unset/0/false/no/off keeps K128 after the normal direct-Q4 WMMA gates, K64 control, 256-row geometry, and 16-byte activation alignment pass; empty or any other value restores K64; incompatible launches also retain K64 and K64=0 retains the K32 rollback	Roll aligned resident q_b-shaped 256-row direct-WMMA launches back from four-qgroup K128/P144 staging and float4 activation loads to K64/P80.	rocm/ds4_rocm_q4.cuh:1874
+runtime/rocm	DS4_ROCM_DISABLE_Q4_SELECTED_EXPERT_VIEWS	presence rollback flag; unset keeps automatic/default path	Disable/roll back rocm disable q4 selected expert views.	ds4.c:21150
+runtime/rocm	DS4_ROCM_DISABLE_RESIDENT_IQ2_SORTED	presence rollback flag; unset keeps automatic/default path	Disable/roll back rocm disable resident iq2 sorted.	rocm/ds4_rocm_moe_launch.cuh:751
+runtime/rocm	DS4_ROCM_DISABLE_ROUTED_PAIR_SWIGLU_FUSION	presence rollback flag; unset keeps automatic/default path	Disable/roll back rocm disable routed pair swiglu fusion.	ds4.c:18543
+runtime/rocm	DS4_ROCM_DISABLE_STREAMING_COLD_DECODE_PREFILL	presence rollback flag; unset keeps automatic/default path	Disable/roll back rocm disable streaming cold decode prefill.	ds4.c:32021
+runtime/rocm	DS4_ROCM_DISABLE_STREAMING_DECODE_PREFILL	presence rollback flag; unset keeps automatic/default path	Disable/roll back rocm disable streaming decode prefill.	ds4.c:31970
+runtime/rocm	DS4_ROCM_DISABLE_STREAMING_EXPERT_ADDR_TABLE	presence rollback flag; unset keeps automatic/default path	Disable/roll back rocm disable streaming expert addr table.	ds4.c:18539
+runtime/rocm	DS4_ROCM_DISABLE_STREAMING_EXPERT_HOTLIST	presence rollback flag; unset keeps automatic/default path	Disable/roll back rocm disable streaming expert hotlist.	ds4.c:21304
+runtime/rocm	DS4_ROCM_DISABLE_STREAMING_FULL_EXPERT_ADDR_TABLE	presence rollback flag; unset keeps automatic/default path	Disable/roll back rocm disable streaming full expert addr table.	ds4.c:18257
+runtime/rocm	DS4_ROCM_DISABLE_STREAMING_LAYER_BATCH	presence rollback flag; unset keeps automatic/default path	Disable/roll back rocm disable streaming layer batch.	ds4.c:18253
+runtime/rocm	DS4_ROCM_DISABLE_STREAMING_MADVISE_WILLNEED	presence rollback flag; unset keeps automatic/default path	Disable/roll back rocm disable streaming madvise willneed.	ds4.c:18226
+runtime/rocm	DS4_ROCM_DISABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR	presence rollback flag; unset keeps automatic/default path	Disable/roll back rocm disable streaming prefill batch selected addr.	ds4.c:18537
+runtime/rocm	DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_MADVISE	presence rollback flag; unset keeps automatic/default path	Disable/roll back rocm disable streaming prefill layer madvise.	ds4.c:18480
+runtime/rocm	DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_PAGEIN	presence rollback flag; unset keeps automatic/default path	Disable/roll back rocm disable streaming prefill layer pagein.	ds4.c:18448
+runtime/rocm	DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_PAGEIN_OVERLAP	presence rollback flag; unset keeps automatic/default path	Disable/roll back rocm disable streaming prefill layer pagein overlap.	ds4.c:19335
+runtime/rocm	DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_PREAD	presence rollback flag; unset keeps automatic/default path	Disable/roll back rocm disable streaming prefill layer pread.	ds4.c:18468
+runtime/rocm	DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_PREPARE	presence rollback flag; unset keeps automatic/default path	Disable/roll back rocm disable streaming prefill layer prepare.	ds4.c:18460
+runtime/rocm	DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_PREPARE_OVERLAP	presence rollback flag; unset keeps automatic/default path	Disable/roll back rocm disable streaming prefill layer prepare overlap.	ds4.c:19333
+runtime/rocm	DS4_ROCM_DISABLE_STREAMING_PREFILL_LAYER_READAHEAD	presence rollback flag; unset keeps automatic/default path	Disable/roll back rocm disable streaming prefill layer readahead.	ds4.c:18458
+runtime/rocm	DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_ASYNC_LOAD	presence rollback flag; unset keeps automatic/default path	Disable/roll back rocm disable streaming prefill selected async load.	ds4.c:46684
+runtime/rocm	DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_MADVISE	presence rollback flag; unset keeps automatic/default path	Disable/roll back rocm disable streaming prefill selected madvise.	ds4.c:18438
+runtime/rocm	DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_PAGEIN	presence rollback flag; unset keeps automatic/default path	Disable/roll back rocm disable streaming prefill selected pagein.	ds4.c:18428
+runtime/rocm	DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_PROFILE	presence rollback flag; unset keeps automatic/default path	Collect timing/profile diagnostics for rocm disable streaming prefill selected profile.	ds4.c:18922
+runtime/rocm	DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_READAHEAD	presence rollback flag; unset keeps automatic/default path	Disable/roll back rocm disable streaming prefill selected readahead.	ds4.c:19974
+runtime/rocm	DS4_ROCM_DISABLE_STREAMING_PREFILL_SELECTED_READAHEAD_SHARED	presence rollback flag; unset keeps automatic/default path	Disable/roll back rocm disable streaming prefill selected readahead shared.	ds4.c:19984
+runtime/rocm	DS4_ROCM_DISABLE_STREAMING_READAHEAD	presence rollback flag; unset keeps automatic/default path	Disable/roll back rocm disable streaming readahead.	ds4.c:18219
+runtime/rocm	DS4_ROCM_DISABLE_STREAMING_SELECTED_ASYNC_LOAD	presence rollback flag; unset keeps automatic/default path	Disable/roll back rocm disable streaming selected async load.	ds4.c:44587
+runtime/rocm	DS4_ROCM_DISABLE_STREAMING_SPLIT_SELECTED	presence rollback flag; unset keeps automatic/default path	Disable/roll back rocm disable streaming split selected.	rocm/ds4_rocm_moe_launch.cuh:665
+runtime/rocm	DS4_ROCM_DISABLE_STREAMING_STATIC_DECODE_MAP	presence rollback flag; unset keeps automatic/default path	Disable/roll back rocm disable streaming static decode map.	ds4.c:18231
+runtime/rocm	DS4_ROCM_DISABLE_STREAMING_STATIC_MAP_STATE_CACHE	presence rollback flag; unset keeps automatic/default path	Disable/roll back rocm disable streaming static map state cache.	ds4.c:18244
+runtime/rocm	DS4_ROCM_DSV4_PREQUANT_DECODE	sampled once; unset: enabled; present empty or exact 0: disabled; every other present value: enabled; quality mode and GLM models force it off regardless	ROCm DeepSeek-V4 decode: quantizes one-token F32 activations to Q8 once and selects the prequantized Q8_0/DP4A projection kernels instead of the full-F32 activation paths.	rocm/ds4_rocm_runtime.cuh:4775
+runtime/rocm	DS4_ROCM_ENABLE_GFX1151_PREFILL_WMMA_INDEXED	value-aware per-launch opt-in, default off; empty or 1/true/yes/on enables and 0/false/no/off disables; invalid values fail closed; DISABLE wins; eligibility additionally requires actual gfx1151, runtime warpSize=32, quality off, N>1, 64 model heads, head_dim=512, and top_k=512	Use 32-head rocWMMA workgroups with 80-key tiles and vectorized float2-to-half2 KV staging for DeepSeek-V4 indexed mixed-attention prefill; all failed gates preserve the existing heads32/heads16 fallback.	rocm/ds4_rocm_attention_launch.cuh:11
+runtime/rocm	DS4_ROCM_ENABLE_GFX1151_PREFILL_WMMA_RING	value-aware per-launch opt-in, default off; empty or 1/true/yes/on enables and 0/false/no/off disables; invalid values fail closed; DISABLE wins; eligibility additionally requires actual gfx1151, runtime warpSize=32, quality off, unmasked N>1, 64 model heads, and head_dim=512	Use 32-head rocWMMA workgroups with 80-key tiles and vectorized float2-to-half2 KV staging for causal raw-ring/mixed prefill; all failed gates preserve the existing online or score-buffer fallback.	rocm/ds4_rocm_attention_launch.cuh:9
+runtime/rocm	DS4_ROCM_ENABLE_IQ2_MOE_WMMA_TAIL_CULL	value-aware boolean evaluated on every eligible launch; unset defaults off; after trimming whitespace, empty or 1/true/yes/on enables and 0/false/no/off disables (words case-insensitive); invalid values fail closed with a one-time diagnostic; true DISABLE overrides	Opt into skipping inactive 16-row tail waves in both wave32 mixed IQ2_XXS gate/up and Q2_K down hot-list rocWMMA prefill kernels; other shapes and paths retain their existing fallback.	rocm/ds4_rocm_moe_launch.cuh:18
+runtime/rocm	DS4_ROCM_ENABLE_MXFP4_LDSB	presence opt-in; unset=off; any defined value including empty or 0 enables the candidate when the MXFP4 path, sorted expert tiles, token count >= 128, LDS-size limit, and dimension-alignment gates all pass	Select the ROCm MXFP4 prefill gate/up kernel that stages eight gate and eight up weight rows in LDS and reuses them across expert tiles of up to 128 tokens.	rocm/ds4_rocm_moe_launch.cuh:782
+runtime/rocm	DS4_ROCM_ENABLE_MXFP4_ROW64	presence opt-in; unset=off; any defined value including empty or 0 enables the candidate when the MXFP4 sorted-tile path has at least 8 tokens and the TILE32, LDSB, and TILE4 candidates are not selected	Select the ROCm MXFP4 gate/up tile8 occupancy variant with 64 row slots and 512 threads per block.	rocm/ds4_rocm_moe_launch.cuh:798
+runtime/rocm	DS4_ROCM_ENABLE_MXFP4_TILE32	presence opt-in; unset=off; any defined value including empty or 0 enables the candidate when the MXFP4 sorted-tile path has at least 32 tokens and the expert intermediate dimension is divisible by 32	Select the ROCm MXFP4 gate/up tile32 kernel, reusing each loaded expert-weight chunk across as many as 32 tokens.	rocm/ds4_rocm_moe_launch.cuh:786
+runtime/rocm	DS4_ROCM_ENABLE_MXFP4_TILE4	presence opt-in; unset=off; any defined value including empty or 0 enables the candidate when the MXFP4 sorted-tile path has at least 5 tokens and neither TILE32 nor LDSB is selected	Select the ROCm MXFP4 gate/up tile4 occupancy variant, reducing staged-activation LDS per block.	rocm/ds4_rocm_moe_launch.cuh:794
+runtime/rocm	DS4_ROCM_ENABLE_Q4_ATTN_Q_B_F16_CACHE	value-aware persistent-cache opt-in, default off; unset/0/false/no/off is off; DISABLE cancels an optional persistent request but leaves the automatic transient path independent; REQUIRE plus DISABLE fails closed	Prewarm and use persistent resident F16 sidecars for eligible ROCm Q4_K attn_q_b prefills.	rocm/ds4_rocm_q4_qb_sidecar.cuh:130
+runtime/rocm	DS4_ROCM_ENABLE_Q4_ATTN_Q_B_F16_OUTPUT	value-aware experimental opt-in; unset/empty/0/false/no/off keeps the release F32 projection boundary; other nonempty values enable	Write eligible resident Q4_K attn_q_b GEMM output in F16 and run the half-input norm/RoPE epilogue; SSD remains excluded.	rocm/ds4_rocm_q4_qb_sidecar.cuh:152
+runtime/rocm	DS4_ROCM_ENABLE_Q4_DENSE_PAIR	presence opt-in; unset=off; DISABLE takes precedence	Enable rocm enable q4 dense pair.	rocm/ds4_rocm_q4.cuh:434
+runtime/rocm	DS4_ROCM_ENABLE_Q4_GROUPED_ATTN_A	presence opt-in outside the default scope; the exact caller-marked resident decode shape groups=8, N=1, K=4096, M=1024 is automatic, while row-at-a-time batch fallbacks are not; DISABLE wins	Enable grouped Q4 attention-A for eligible slices, non-production shapes, or explicit experiments in addition to the resident decode default.	rocm/ds4_rocm_q4.cuh:872
+runtime/rocm	DS4_ROCM_ENABLE_Q4_PREFILL_K1024_TILE4_SSD	value-aware SSD-only opt-in, default off; unset/0/false/no/off retains TILE8, while empty or any other value requests TILE4; eligibility additionally requires N=9..4096, K=1024, M=32768, TILE8 enabled, and the complete weight range in device storage rather than mapped/registered host memory; DISABLE wins	Allow the four-lane K=1024 Q4_K prefill specialization to consume an already device-resident/cache-backed attn_q_b weight range during SSD streaming without changing model I/O.	rocm/ds4_rocm_q4.cuh:624
+runtime/rocm	DS4_ROCM_ENABLE_Q4_PREFILL_Q8_K_WAVE32	value-aware opt-in, default off; unset/0/false/no/off retains the canonical quantizer, while empty or any other value requests the candidate for N=9..4096 on gfx1151 wave32; DISABLE wins; a selected exact Q8 path takes precedence over automatic direct-Q4 WMMA, REQUIRE_WMMA overrides an optional request, and dual REQUIRE fails closed	Quantize eight independent Q8_K activation blocks per 256-thread workgroup using one wave32 per block, without LDS or workgroup barriers, before the exact Q4 prefill matmul (TILE8 or its legacy rollback).	rocm/ds4_rocm_q4.cuh:858
+runtime/rocm	DS4_ROCM_ENABLE_Q4_PREFILL_WMMA	value-aware compatibility control; unset keeps automatic resident direct-Q4 WMMA for standalone dense and attention-output A while attention-output B remains on Q8_K+TILE8; empty or any value other than 0/false/no/off explicitly retains the same eligible A paths but no longer opts B into direct WMMA; explicit 0/false/no/off opts out unless REQUIRE is set, while DISABLE is the authoritative rollback; N=256..4096, K a positive multiple of 256, resident non-quality gfx1151 wave32 only; SSD has a separate gate	Use compressed Q4_K-to-F16 register dequantization plus shape-selected 64-token by 64/128/256-row WMMA tiles, K64/P80 staging on 64/128 rows, and default K128/P144 float4 staging on aligned 256 rows, without Q8_K activation scratch or an F16 weight sidecar.	rocm/ds4_rocm_q4.cuh:1622
+runtime/rocm	DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_K64	value-aware base staging control; unset, empty, or any value other than 0/false/no/off uses K64/P80 on 64/128-row or K128-incompatible launches and permits default K128/P144 on aligned 256-row launches; 0/false/no/off suppresses both wider stages and rolls back to K32; DS4_ROCM_DISABLE_Q4_PREFILL_WMMA wins	Stage two adjacent 32-value Q4_K groups and a 64-value activation slice in one padded P80 LDS tile as the narrower geometry and K128 fallback, halving K32 workgroup barriers while preserving its activation traffic and accumulation order.	rocm/ds4_rocm_q4.cuh:1870
+runtime/rocm	DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_SSD	value-aware SSD-only opt-in, default off; unset/0/false/no/off retains TILE8/TILE4, while empty or any other value requests direct-Q4 WMMA for eligible standalone projections and attention-output A but leaves attention-output B on Q8_K+TILE8; eligibility additionally requires each complete projection weight range in physical device storage rather than mapped/registered host memory; DISABLE wins	Allow the compressed direct-Q4 WMMA kernel to consume an already device-resident/cache-backed Q4_K projection during SSD streaming without changing model I/O.	rocm/ds4_rocm_q4.cuh:1356
+runtime/rocm	DS4_ROCM_ENABLE_STREAMING_FULL_EXPERT_ADDR_TABLE	presence opt-in flag; unset=off unless paired policy is automatic	Enable rocm enable streaming full expert addr table.	ds4.c:18255
+runtime/rocm	DS4_ROCM_ENABLE_STREAMING_MADVISE_WILLNEED	presence opt-in flag; unset=off unless paired policy is automatic	Enable rocm enable streaming madvise willneed.	ds4.c:18224
+runtime/rocm	DS4_ROCM_ENABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR	presence opt-in flag; unset=off unless paired policy is automatic	Enable rocm enable streaming prefill batch selected addr.	ds4.c:18584
+runtime/rocm	DS4_ROCM_ENABLE_STREAMING_PREFILL_CACHE_SEED	presence opt-in flag; unset=off unless paired policy is automatic	Enable rocm enable streaming prefill cache seed.	ds4.c:21273
+runtime/rocm	DS4_ROCM_ENABLE_STREAMING_PREFILL_LAYER_PAGEIN	presence opt-in flag; unset=off unless paired policy is automatic	Enable rocm enable streaming prefill layer pagein.	ds4.c:18446
+runtime/rocm	DS4_ROCM_ENABLE_STREAMING_PREFILL_LAYER_READAHEAD	presence opt-in flag; unset=off unless paired policy is automatic	Enable rocm enable streaming prefill layer readahead.	ds4.c:18456
+runtime/rocm	DS4_ROCM_ENABLE_STREAMING_PREFILL_SELECTED_MADVISE	presence opt-in flag; unset=off unless paired policy is automatic	Enable rocm enable streaming prefill selected madvise.	ds4.c:18436
+runtime/rocm	DS4_ROCM_ENABLE_STREAMING_PREFILL_SELECTED_PAGEIN	presence opt-in flag; unset=off unless paired policy is automatic	Enable rocm enable streaming prefill selected pagein.	ds4.c:18426
+runtime/rocm	DS4_ROCM_ENABLE_STREAMING_PREFILL_SELECTED_READAHEAD	presence opt-in flag; unset=off unless paired policy is automatic	Enable rocm enable streaming prefill selected readahead.	ds4.c:19970
+runtime/rocm	DS4_ROCM_ENABLE_STREAMING_PREFILL_SELECTED_READAHEAD_SHARED	presence opt-in flag; unset=off unless paired policy is automatic	Enable rocm enable streaming prefill selected readahead shared.	ds4.c:19972
+runtime/rocm	DS4_ROCM_ENABLE_STREAMING_READAHEAD	presence opt-in flag; unset=off unless paired policy is automatic	Enable rocm enable streaming readahead.	ds4.c:18217
+runtime/rocm	DS4_ROCM_ENABLE_STREAMING_STATIC_DECODE_MAP	presence opt-in flag; unset=off unless paired policy is automatic	Enable rocm enable streaming static decode map.	ds4.c:18236
+runtime/rocm	DS4_ROCM_GLM_CAUSAL_ATTN_GEMM	Enabled by default when unset. Exact "0" or an empty value disables; every other nonempty value enables (including false/off/no), because cuda_env_present only tests nonempty and != "0". Eligibility still requires causal_range && !has_selected; a failed GEMM helper falls through to the scalar attention kernel.	Use FP16 BLAS GEMMs for dense causal GLM indexed prefill; =0 is the correctness/performance rollback to the scalar attention kernel.	rocm/ds4_rocm_glm.cuh:3283
+runtime/rocm	DS4_ROCM_GLM_DISABLE_STREAMING_EXPERT_CACHE	Pure presence flag: any defined value, including empty or "0", disables. Unset leaves automatic GLM streaming expert-cache eligibility enabled for supported model/quant/quality/SSD configurations. On ROCm builds DS4_METAL_GLM_DISABLE_STREAMING_EXPERT_CACHE is an accepted fallback alias.	Disable selected/resident streamed-expert cache paths and force generic/full-layer expert handling for GLM SSD streaming.	ds4.c:21197
+runtime/rocm	DS4_ROCM_GLM_DISABLE_STREAMING_SEED_BEFORE_PREFILL	Pure presence flag: any defined value, including empty or "0", disables. Unset seeds before prefill whenever SSD streaming is active. On ROCm builds DS4_METAL_GLM_DISABLE_STREAMING_SEED_BEFORE_PREFILL is an accepted fallback alias.	Skip the pre-prefill hotlist seed of the streaming expert cache in both one-shot GLM generation and session setup.	ds4.c:51073
+runtime/rocm	DS4_ROCM_GLM_DISABLE_STREAMING_TOKEN_PREFILL	Pure presence flag: any defined value, including empty or "0", disables. Unset leaves the token-major path eligible only for SSD streaming, non-quality mode, a nonempty batch fitting full attention, and n_tokens <= the configured nonzero maximum. DS4_METAL_GLM_DISABLE_STREAMING_TOKEN_PREFILL and generic DS4_GLM_DISABLE_STREAMING_TOKEN_PREFILL are also accepted presence aliases.	Roll back GLM SSD-streaming token-major prefill to the normal prefill implementation.	ds4.c:49703
+runtime/rocm	DS4_ROCM_GLM_GROUPED_QK_LOW	sampled once; unset: enabled; present empty or exact 0: disabled; every other present value: enabled	Selects the grouped shared-input ROCm kernel for eligible multi-token GLM qk-lowrank projection; disabling uses the per-head/per-token projection kernel.	rocm/ds4_rocm_runtime.cuh:4800
+runtime/rocm	DS4_ROCM_GLM_GROUPED_VALUE_PROJECT	sampled once; unset: enabled; present empty or exact 0: disabled; every other present value: enabled	Selects the grouped shared-input ROCm kernel for eligible multi-token GLM value projection; disabling uses the non-grouped batch projection path.	rocm/ds4_rocm_runtime.cuh:4791
+runtime/rocm	DS4_ROCM_GLM_LAYER_SLICE_TOKEN_DECODE	Opt-in truthy parser; unset, empty, "0", false, off, or no (case-insensitive words) are false, every other nonempty value is true. Default off and compiled only for ROCm.	Allow a one-token, pos>0 GLM layer-slice with inter-node input/output hidden buffers to use the optimized resident token graph; without it only the no-hidden-buffer case takes that shortcut.	ds4.c:43573
+runtime/rocm	DS4_ROCM_GLM_SELECTED_ATTN_GEMM	Enabled by default when unset. Exact "0" or an empty value disables; every other nonempty value enables (including false/off/no). Eligibility still requires !causal_range && has_selected; failure/ineligibility falls through to the scalar attention kernel.	Gather per-token selected cache rows into FP16 matrices and use strided-batched BLAS GEMMs for GLM selected indexed prefill; =0 forces the scalar path.	rocm/ds4_rocm_glm.cuh:3239
+runtime/rocm	DS4_ROCM_GLM_SELECTED_ATTN_HEAD_TILE	Unsigned integer read and cached once; valid values are exactly 1,2,4,8,16,32,64. Unset/empty defaults to 16. A nonnumeric, partially parsed, overflowed, or unsupported value prints a warning and uses 16; the effective tile is min(requested,n_head).	Set how many attention heads each selected-attention GEMM workspace tile processes.	rocm/ds4_rocm_glm.cuh:2509
+runtime/rocm	DS4_ROCM_GLM_SELECTED_ATTN_PROFILE	presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose)	Collect timing/profile diagnostics for rocm glm selected attn profile.	rocm/ds4_rocm_glm.cuh:2534
+runtime/rocm	DS4_ROCM_GLM_STREAMING_ASYNC_PROFILE	presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose)	Collect timing/profile diagnostics for rocm glm streaming async profile.	ds4.c:44361
+runtime/rocm	DS4_ROCM_GLM_STREAMING_DECODE_FULL_LAYER_MAP	Pure presence flag: any defined value, including empty or "0", forces full mapping. Unset uses automatic mapping: resident layers map fully, eligible expert-cache layers use decode-only/expert mapping, otherwise full mapping. DS4_METAL_GLM_STREAMING_DECODE_FULL_LAYER_MAP and generic DS4_GLM_STREAMING_DECODE_FULL_LAYER_MAP are also accepted presence aliases.	Force every GLM SSD-streaming decode layer through full-layer mapping, bypassing the selected-expert/decode mapping optimization.	ds4.c:42615
+runtime/rocm	DS4_ROCM_GLM_STREAMING_DECODE_SYNC_EACH_LAYER	ROCm-only primary value; nonempty takes priority over DS4_METAL_GLM_STREAMING_DECODE_SYNC_EACH_LAYER and the generic DS4_GLM_STREAMING_DECODE_SYNC_EACH_LAYER fallback; empty acts as unset; truthy unless exact 0 or case-insensitive false/off/no; with all aliases unset: false	For non-static GLM SSD decode on ROCm, opts into a full command/device synchronization after token mapping and every layer; default keeps ordered work queued across layer mappings, and static-map decode bypasses it.	ds4.c:49779
+runtime/rocm	DS4_ROCM_GLM_STREAMING_GROW_CACHE_AFTER_PREFILL	Enabled by default when absent. If defined, only a truthy nonempty value enables; empty, "0", false, off, or no disable. Growth also requires SSD streaming plus nonzero base cache and prefill-headroom budgets, and occurs only if the recomputed expert count exceeds the current count.	After successful ROCm GLM prefill, add the released prefill headroom to the dynamic streaming expert-cache byte budget.	ds4.c:51111
+runtime/rocm	DS4_ROCM_GLM_STREAMING_PREFILL_FULL_LAYER	presence force-on; any presence including empty or 0 enables; unset falls back to the Metal alias and then the automatic token threshold (1024 by default on ROCm); DS4_ROCM_DISABLE_GLM_STREAMING_PREFILL_FULL_LAYER dominates	Forces GLM SSD prefill into full-layer mapping/cache mode even below the automatic large-batch threshold.	ds4.c:42692
+runtime/rocm	DS4_ROCM_GLM_STREAMING_PREFILL_FULL_LAYER_MIN_TOKENS	Positive uint32 threshold parsed with strtoul; ROCm default is 1024. Missing/empty, no leading number, errno/overflow, zero, or >UINT32_MAX returns 1024. The parser does not require end-of-string, so trailing junk after a valid leading number is accepted. A nonempty ROCm value takes precedence; otherwise DS4_METAL_GLM_STREAMING_PREFILL_FULL_LAYER_MIN_TOKENS is a fallback alias.	Set the automatic token-count crossover for ROCm GLM SSD prefill to load/use full resident expert layers when the layer supports that mode.	ds4.c:42692
+runtime/rocm	DS4_ROCM_GLM_STREAMING_PREFILL_SYNC_EACH_LAYER	ROCm-only primary value; nonempty takes priority over DS4_METAL_GLM_STREAMING_PREFILL_SYNC_EACH_LAYER and the generic DS4_GLM_STREAMING_PREFILL_SYNC_EACH_LAYER fallback; empty acts as unset; truthy unless exact 0 or case-insensitive false/off/no; with all aliases unset: false for compact prefill; full-layer prefill always returns true	For compact GLM SSD prefill on ROCm, opts into a full command/device synchronization at every layer boundary; default preserves queued work across mappings, while full-layer cache mode always synchronizes.	ds4.c:42395
+runtime/rocm	DS4_ROCM_GLM_STREAMING_TOKEN_PREFILL_MAX	primary nonempty value, then the Metal alias, then generic DS4_GLM_STREAMING_TOKEN_PREFILL_MAX; parsed by strtoul without requiring full-string consumption; 0 is valid and disables; no digits, ERANGE, or > UINT32_MAX uses the ROCm default 0	Sets the largest non-quality GLM SSD-prefill chunk eligible for token-major/decode-style execution; ROCm defaults to canonical indexed batch prefill (0 disables token-major mode).	ds4.c:49682
+runtime/rocm	DS4_ROCM_GLM_VALUE_PROJECT_WAVE_DECODE	Enabled by default when unset. Exact "0" or empty disables; every other nonempty value enables (including false/off/no). It applies only when n_tokens==1; =0 or multi-token input uses the generic per-head batch kernel.	Select the validated wave-per-output-row ROCm Q8 GLM value-projection kernel for one-token decode; =0 is the generic-kernel rollback.	rocm/ds4_rocm_glm.cuh:2019
+runtime/rocm	DS4_ROCM_GRAPH_DUMP_LAYER	unsigned layer or all; unset=all layers	Filter ROCm graph dumps by layer.	ds4.c:16862
+runtime/rocm	DS4_ROCM_GRAPH_DUMP_NAME	nonempty substring filter; unset=all tensor names	Filter ROCm graph dumps by tensor/stage name.	ds4.c:16858
+runtime/rocm	DS4_ROCM_GRAPH_DUMP_NONINVASIVE	truthy value under the shared parser; unset lets dumping select conservative kernels	Keep production ROCm kernel selection while graph dumping.	rocm/ds4_rocm_runtime.cuh:4822
+runtime/rocm	DS4_ROCM_GRAPH_DUMP_POS	unsigned token position; unset=all positions	Filter ROCm graph dumps by position.	ds4.c:16869
+runtime/rocm	DS4_ROCM_GRAPH_DUMP_PREFIX	nonempty output path prefix; unset=off	Enable ROCm intermediate graph/tensor dumps.	ds4_cuda.cu:397
+runtime/rocm	DS4_ROCM_GRAPH_DUMP_TRACE	presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose)	Emit trace diagnostics for rocm graph dump trace.	ds4.c:16914
+runtime/rocm	DS4_ROCM_GRAPH_OUTPUT_ROW	Nonempty string with a leading strtoul-parsable unsigned value < n_tokens selects that zero-based row. Default, empty, unparsable, or out-of-range selects n_tokens-1. Trailing characters are accepted because full consumption/errno are not checked. A nonempty ROCm value takes precedence; otherwise DS4_METAL_GRAPH_OUTPUT_ROW is a fallback alias.	Choose which prefill hidden-state row is sent through the output head to produce logits, primarily for graph/correctness diagnostics.	ds4.c:35860
+runtime/rocm	DS4_ROCM_GRAPH_PREFILL_PROFILE	presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose)	Collect timing/profile diagnostics for rocm graph prefill profile.	ds4.c:32053
+runtime/rocm	DS4_ROCM_GRAPH_PREFILL_SPLIT_PROFILE	presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose)	Collect timing/profile diagnostics for rocm graph prefill split profile.	ds4.c:35783
+runtime/rocm	DS4_ROCM_GRAPH_TOKEN_PROFILE	presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose)	Collect timing/profile diagnostics for rocm graph token profile.	ds4.c:31737
+runtime/rocm	DS4_ROCM_INDEXER_STAGE_PROFILE	presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose)	Collect timing/profile diagnostics for rocm indexer stage profile.	ds4.c:29280
+runtime/rocm	DS4_ROCM_IQ2_MOE_WMMA_PROFILE	value-aware boolean evaluated on every N>1 mixed IQ2/Q2 launch; unset is off; after trimming whitespace, empty or 1/true/yes/on enables and 0/false/no/off disables (words case-insensitive); invalid values disable profiling with a one-time diagnostic	Use HIP cudaEvent timing to report gate/up, down, and summed GPU milliseconds for the affected hot-list rocWMMA stages, excluding SSD/CPU time; profiling is strict and fails the backend call if either stage or event timing is unavailable.	rocm/ds4_rocm_moe_launch.cuh:22
+runtime/rocm	DS4_ROCM_LAYER_STAGE_PROFILE	presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose)	Collect timing/profile diagnostics for rocm layer stage profile.	ds4.c:29119
+runtime/rocm	DS4_ROCM_LAYER_STAGE_PROFILE_LAYER	layer filter subordinate to DS4_ROCM_LAYER_STAGE_PROFILE; unset or whitespace-only: all layers allowed by the parent flag; otherwise the whitespace-trimmed value must be a complete base-10 strtoul result <= UINT32_MAX equal to the current layer; invalid values match none	Restricts the ROCm layer/prefill stage profiler to one layer; it does not enable profiling by itself.	ds4.c:29120
+runtime/rocm	DS4_ROCM_MOE_DECODE_DOWN_RPB	sampled once; nonempty value is parsed by strtoul (a numeric prefix is sufficient), cast to uint32_t, and accepted only if 1/2/4/8/16/32; unset/empty/invalid inherits DS4_ROCM_MOE_DECODE_RPB, with defaults quality=8, non-quality SSD=2, resident=1	Sets output rows (warps) per block for ROCm Q2_K routed-MoE decode down-projection kernels; threads per block are value * 32.	rocm/ds4_rocm_runtime.cuh:4842
+runtime/rocm	DS4_ROCM_MOE_DECODE_GATE_RPB	sampled once; nonempty value is parsed by strtoul (a numeric prefix is sufficient), cast to uint32_t, and accepted only if 1/2/4/8/16/32; unset/empty/invalid defaults to 1 in non-quality SSD mode when DS4_ROCM_MOE_DECODE_RPB is unset/empty, otherwise inherits the resolved base RPB	Sets output rows (warps) per block for ROCm Q2_K routed-MoE decode gate/up kernels; threads per block are value * 32.	rocm/ds4_rocm_runtime.cuh:4836
+runtime/rocm	DS4_ROCM_MOE_DECODE_PROFILE	presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose)	Collect timing/profile diagnostics for rocm moe decode profile.	rocm/ds4_rocm_moe_launch.cuh:82
+runtime/rocm	DS4_ROCM_MOE_DECODE_RPB	sampled once; nonempty value is parsed by strtoul (a numeric prefix is sufficient), cast to uint32_t, and accepted only if 1/2/4/8/16/32; unset/empty/invalid default: quality=8, non-quality SSD=2, resident=1	Sets the base ROCm Q2_K decode-MoE rows-per-block value inherited by gate/up and down controls, except the automatic SSD gate specialization defaults to 1 when this variable is unset/empty.	rocm/ds4_rocm_runtime.cuh:4832
+runtime/rocm	DS4_ROCM_MOE_PATH_DEBUG	presence diagnostic; unset=off; any defined value including empty or 0 enables it	Print ROCm routed-MoE path selection, sorted-tile scratch state, and MXFP4 gate/up launch diagnostics to stderr.	rocm/ds4_rocm_moe_launch.cuh:831
+runtime/rocm	DS4_ROCM_MOE_WRITE_CLAMPED_ACT	Pure presence sentinel: any defined value, including empty or "0", is active; DS4_METAL_MOE_WRITE_CLAMPED_ACT is an accepted fallback alias. On ROCm the variable is only consumed as a path-admission veto: it disables selected-expert cache/address-table, selected-slot and CPU-router/fused optimized paths. No ROCm call site parses a clamp amount or directly enables a write-clamped kernel.	Force shared graph selection away from optimizations incompatible with the clamped-intermediate MoE diagnostic; on ROCm this is a compatibility/rollback gate, not itself a clamped-write implementation.	ds4.c:18526
+runtime/rocm	DS4_ROCM_MXFP4_DOWN_RGROUP	nonempty value is parsed by strtol and a numeric prefix is sufficient; integers 1..8 are accepted; unset, empty, invalid, or out-of-range values use 1	Set how many 32-row output blocks each ROCm MXFP4 tiled down-projection block computes, reducing the first launch-grid dimension as the value increases.	rocm/ds4_rocm_moe_launch.cuh:801
+runtime/rocm	DS4_ROCM_Q4_ATTN_Q_B_F16_CACHE_MB	integer MiB with a full-string parse; default 3072; accepted range 1..65536, values above clamp and invalid or smaller values restore the default	Cap device memory used by resident ROCm Q4_K attn_q_b F16 sidecars.	rocm/ds4_rocm_q4_qb_sidecar.cuh:162
+runtime/rocm	DS4_ROCM_Q4_ATTN_Q_B_F16_CACHE_MIN_TOKENS	integer token count with a full-string parse; default 512; accepted range 32..UINT32_MAX, values above clamp and invalid or smaller values restore the default	Set the minimum prefill batch eligible to prepare or use the ROCm Q4 attn_q_b F16 sidecars.	rocm/ds4_rocm_q4_qb_sidecar.cuh:156
+runtime/rocm	DS4_ROCM_Q4_ATTN_Q_B_TRANSIENT_F16_MIN_TOKENS	full-string unsigned token count; default 4096; accepted range 32..UINT32_MAX; values above clamp and invalid or smaller values restore 4096	Set the minimum device-resident, non-SSD prefill batch eligible for per-layer transient ROCm Q4_K attn_q_b-to-F16 expansion.	rocm/ds4_rocm_q4_qb_sidecar.cuh:150
+runtime/rocm	DS4_ROCM_Q4_GROUPED_ATTN_A_STATS	presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose)	Print counters for rocm q4 grouped attn a stats.	rocm/ds4_rocm_q4.cuh:614
+runtime/rocm	DS4_ROCM_Q4_PREFILL_TILE8_STATS	presence diagnostic; unset=off; any defined value including empty or 0 prints at exit	Print tiled-prefill dense/pair/attention counters, total and SSD-specific K=1024 TILE4 dispatches, and direct-WMMA total/K32/K64/K128 launch counts.	rocm/ds4_rocm_q4.cuh:1454
+runtime/rocm	DS4_ROCM_Q4_PREFILL_WMMA_ROW_TILE	unsigned integer; unset, empty, malformed, negative, or values other than 64/128/256 use shape selection (64 rows when M<1024, 128 when M<8192, otherwise 256); 64 retains the previous geometry	Override the number of output rows sharing each direct-Q4 64x32 activation tile for controlled 64/128/256-row ROCm WMMA A/B measurements.	rocm/ds4_rocm_q4.cuh:1517
+runtime/rocm	DS4_ROCM_Q8_DECODE_SHAREDX_64K	sampled once; unset: enabled; present empty or exact 0: disabled; every other present value: enabled; effective only for one-token non-prequant Q8_0 matmul with 8192 < in_dim <= 16384	Allows the ROCm shared-input Q8 decode kernel to use up to 64 KiB dynamic LDS for wide inputs; an unsupported/failed LDS launch automatically falls back to the regular kernel.	rocm/ds4_rocm_runtime.cuh:4805
+runtime/rocm	DS4_ROCM_Q_STAGE_PROFILE	presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose)	Collect timing/profile diagnostics for rocm q stage profile.	ds4.c:30038
+runtime/rocm	DS4_ROCM_REQUIRE_Q4_ATTN_Q_B_F16_CACHE	value-aware strict opt-in, default off; unset/0/false/no/off is off, empty or any other value requires eligible batches to use the cache; DISABLE wins	Fail an eligible ROCm prefill instead of falling back when the resident Q4_K attn_q_b F16 specialization cannot be prepared or dispatched.	rocm/ds4_rocm_q4_qb_sidecar.cuh:135
+runtime/rocm	DS4_ROCM_REQUIRE_Q4_GROUPED_ATTN_A	presence fail-closed assertion; also requests the candidate outside the caller-marked resident decode default; DISABLE remains authoritative and causes failure	Require grouped Q4 attention-A and fail instead of silently falling back.	rocm/ds4_rocm_q4.cuh:870
+runtime/rocm	DS4_ROCM_REQUIRE_Q4_PREFILL_K1024_TILE4	value-aware fail-closed assertion and SSD opt-in; unset/0/false/no/off is off; empty or any other value requires eligible N=9..4096, K=1024, M=32768 dense calls to select TILE4; SSD also requires an actual device-resident weight range; DISABLE wins	Prevent a K=1024 TILE4 correctness/performance oracle from silently falling back to TILE8, including during SSD-streaming A/B runs.	rocm/ds4_rocm_q4.cuh:628
+runtime/rocm	DS4_ROCM_REQUIRE_Q4_PREFILL_Q8_K_WAVE32	value-aware strict opt-in; unset/0/false/no/off is off; empty or any other value requires gfx1151 wave32 and N=9..4096; DISABLE and conflicts with required WMMA or the required q_b F16 cache fail closed	Require the no-LDS wave32 Q8_K activation quantizer for exact Q4 prefill instead of silently using the canonical quantizer or an F16 side path.	rocm/ds4_rocm_q4.cuh:796
+runtime/rocm	DS4_ROCM_REQUIRE_Q4_PREFILL_TILE8	presence fail-closed assertion for eligible TILE8 calls	Require rocm require q4 prefill tile8 and fail instead of silently falling back.	rocm/ds4_rocm_q4.cuh:452
+runtime/rocm	DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA	value-aware strict diagnostic assertion, not required for the automatic standalone/attention-output-A resident default; unset/0/false/no/off is off; empty or any other value requires every selected Q4 dense or attention-output projection to use direct-Q4 WMMA and is the only control that selects the numerically compounded attention-output B stage; unsupported shape/device, quality mode, DISABLE, or an SSD weight range without physical device residency fails before dispatch	Prevent a strict ROCm Q4 prefill WMMA kernel oracle from silently timing TILE8/TILE4; normal runtime controls keep attention-output B on Q8_K+TILE8.	rocm/ds4_rocm_q4.cuh:1360
+runtime/rocm	DS4_ROCM_STREAMING_DECODE_PREFILL_MAX	primary nonempty value over the Metal alias; parsed by strtol when it has a numeric prefix (trailing text is accepted); <= 0 disables, values > UINT32_MAX clamp, no numeric prefix uses automatic default: 64 for Flash with uniform Q4_K/MXFP4 experts, 18 for other Pro/Flash, otherwise 0; the disable flag dominates	Sets the largest short, non-quality SSD-streaming prefill batch routed through the decode-style path instead of canonical layer-major prefill.	ds4.c:31976
+runtime/rocm	DS4_ROCM_STREAMING_EXPERT_AUTO_PRELOAD_CAP	primary nonempty value over the Metal alias; strict full-string strtoul; valid values > UINT32_MAX clamp, invalid uses 4096, and 0 means no cap (not disabled); when CLI preload is auto/0, unset defaults to cap 4096 except ROCm GLM52, where absent/empty disables automatic preload entirely	Caps the number of hot experts synchronously seeded into the SSD-streaming expert cache in automatic preload mode; an explicit CLI preload count bypasses this cap, and setting this variable opts ROCm GLM52 back into auto preload.	ds4.c:21469
+runtime/rocm	DS4_ROCM_STREAMING_EXPERT_CACHE_VERBOSE	presence flag; unset=off	Print verbose ROCm streaming expert-cache seed/load diagnostics.	rocm/ds4_rocm_runtime.cuh:2904
+runtime/rocm	DS4_ROCM_STREAMING_EXPERT_HOTLIST	Nonempty filesystem path; a nonempty ROCm value takes precedence, otherwise DS4_METAL_STREAMING_EXPERT_HOTLIST is a fallback. The file contains whitespace-separated layer expert hits rows; blank/comment lines are ignored, zero-hit rows skipped, malformed/open/read errors fail seeding. Unset/empty uses the built-in Pro/Flash/GLM52 hotlist. Effective only when non-cold SSD hotlist seeding is enabled and cache/preload budget is nonzero.	Select a custom ranked expert hotlist used to preseed the streaming resident expert cache before decode.	ds4.c:21494
+runtime/rocm	DS4_ROCM_STREAMING_EXPERT_HOTLIST_PROFILE	presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose)	Collect timing/profile diagnostics for rocm streaming expert hotlist profile.	ds4.c:32244
+runtime/rocm	DS4_ROCM_STREAMING_MAP_TRACE	presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose)	Emit trace diagnostics for rocm streaming map trace.	ds4.c:42643
+runtime/rocm	DS4_ROCM_STREAMING_PREFILL_BATCH_SELECTED_ADDR_MAX	primary nonempty value over the Metal alias; strtol accepts a numeric prefix; <= 0 returns 0, > UINT32_MAX clamps, invalid uses ROCm default UINT32_MAX for Pro/Flash/GLM52 and 0 otherwise	Sets the inclusive upper token-count bound for automatically using selected-expert address-table kernels during eligible non-quality SSD batch prefill; 0 disables automatic selection.	ds4.c:18471
+runtime/rocm	DS4_ROCM_STREAMING_PREFILL_BATCH_SELECTED_ADDR_MIN	primary nonempty value over the Metal alias; strtol accepts a numeric prefix; <= 0 returns 0, > UINT32_MAX clamps, invalid uses ROCm default 2 for Pro/Flash/GLM52 and 0 otherwise	Sets the inclusive lower token-count bound for automatically using selected-expert address-table kernels during eligible non-quality SSD batch prefill (the path independently requires more than one token).	ds4.c:18494
+runtime/rocm	DS4_ROCM_STREAMING_PREFILL_CACHE_SEED_K	primary nonempty value over the Metal alias; strict full-string strtoul; unset/empty/invalid: 1; 0 disables; positive values clamp to 64; ignored unless SSD streaming and DS4_ROCM_ENABLE_STREAMING_PREFILL_CACHE_SEED (or Metal alias) is present	Chooses how many trailing token router selections per layer are captured from prefill and used to seed the streaming expert cache afterward.	ds4.c:21267
+runtime/rocm	DS4_ROCM_STREAMING_PREFILL_CACHE_SEED_PROFILE	presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose)	Collect timing/profile diagnostics for rocm streaming prefill cache seed profile.	ds4.c:32153
+runtime/rocm	DS4_ROCM_STREAMING_PREFILL_LAYER_MADVISE_PROFILE	presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose)	Collect timing/profile diagnostics for rocm streaming prefill layer madvise profile.	ds4.c:19609
+runtime/rocm	DS4_ROCM_STREAMING_PREFILL_LAYER_PAGEIN_NO_OVERLAP	Pure presence flag: any defined value, including empty or "0", disables overlap. Default overlap is enabled only if this, PREPARE_NO_OVERLAP, DISABLE_*_PREPARE_OVERLAP, and DISABLE_*_PAGEIN_OVERLAP are all absent. The corresponding DS4_METAL name is an accepted fallback alias. In current code PAGEIN_NO_OVERLAP and PREPARE_NO_OVERLAP are exact synonyms.	Serialize SSD-streaming prefill layer page-in/preparation instead of overlapping preparation of upcoming layers.	ds4.c:19316
+runtime/rocm	DS4_ROCM_STREAMING_PREFILL_LAYER_PAGEIN_PROFILE	presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose)	Collect timing/profile diagnostics for rocm streaming prefill layer pagein profile.	ds4.c:19605
+runtime/rocm	DS4_ROCM_STREAMING_PREFILL_LAYER_PAGEIN_THREADS	legacy fallback read only when DS4_ROCM_STREAMING_PREFILL_LAYER_PREPARE_THREADS and its Metal alias are absent/empty; strict full-string strtoul; unset/empty across both names: 8; invalid or 0: 1; values > 16 clamp to 16	Sets worker count for full-layer SSD-prefill preparation (page touch, pread, readahead, or madvise) when the canonical PREPARE_THREADS control is not set.	ds4.c:19274
+runtime/rocm	DS4_ROCM_STREAMING_PREFILL_LAYER_PREAD_PROFILE	presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose)	Collect timing/profile diagnostics for rocm streaming prefill layer pread profile.	ds4.c:19607
+runtime/rocm	DS4_ROCM_STREAMING_PREFILL_LAYER_PREPARE_AHEAD	primary nonempty value over the Metal alias; strict full-string strtoul; unset/empty: 1; invalid or 0: 1; values > 4 clamp to 4	Sets how many future layer-preparation jobs may be queued concurrently while SSD-prefill preparation overlap is enabled.	ds4.c:19328
+runtime/rocm	DS4_ROCM_STREAMING_PREFILL_LAYER_PREPARE_NO_OVERLAP	Pure presence flag: any defined value, including empty or "0", disables overlap. Default overlap is enabled only if this, PAGEIN_NO_OVERLAP, DISABLE_*_PREPARE_OVERLAP, and DISABLE_*_PAGEIN_OVERLAP are all absent. The corresponding DS4_METAL name is an accepted fallback alias. In current code PREPARE_NO_OVERLAP and PAGEIN_NO_OVERLAP are exact synonyms.	Serialize SSD-streaming prefill layer preparation/page-in instead of overlapping preparation of upcoming layers.	ds4.c:19314
+runtime/rocm	DS4_ROCM_STREAMING_PREFILL_LAYER_PREPARE_THREADS	primary nonempty value over the Metal alias; strict full-string strtoul; unset/empty falls back to LAYER_PAGEIN_THREADS, then default 8; invalid or 0: 1; values > 16 clamp to 16	Sets worker count used to split full-layer SSD-prefill page-touch, pread, readahead, or madvise preparation ranges.	ds4.c:19270
+runtime/rocm	DS4_ROCM_STREAMING_PREFILL_LAYER_READAHEAD_PROFILE	presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose)	Collect timing/profile diagnostics for rocm streaming prefill layer readahead profile.	ds4.c:19611
+runtime/rocm	DS4_ROCM_STREAMING_PREFILL_SELECTED_MADVISE_PROFILE	presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose)	Collect timing/profile diagnostics for rocm streaming prefill selected madvise profile.	ds4.c:19355
+runtime/rocm	DS4_ROCM_STREAMING_PREFILL_SELECTED_MADVISE_THREADS	legacy fallback read only for selected-expert madvise preparation when DS4_ROCM_STREAMING_PREFILL_SELECTED_PREPARE_THREADS and its Metal alias are absent/empty; strict full-string strtoul; if all selected controls are unset it inherits layer preparation threads (default 8); invalid or 0: 1; values > 16 clamp to 16	Sets worker count for selected-expert madvise preparation under its legacy name; non-madvise selected page-in always uses one worker.	ds4.c:19292
+runtime/rocm	DS4_ROCM_STREAMING_PREFILL_SELECTED_PAGEIN_PROFILE	presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose)	Collect timing/profile diagnostics for rocm streaming prefill selected pagein profile.	ds4.c:19353
+runtime/rocm	DS4_ROCM_STREAMING_PREFILL_SELECTED_PREPARE_GAP	primary nonempty value over the Metal alias; strict full-string strtoul; unset/empty/invalid: 0; values > 8 clamp to 8	For selected-expert madvise preparation, merges selected expert runs separated by at most this many unselected expert IDs, trading broader hints for fewer ranges.	ds4.c:19304
+runtime/rocm	DS4_ROCM_STREAMING_PREFILL_SELECTED_PREPARE_THREADS	primary nonempty value over the Metal alias; strict full-string strtoul; for selected-expert madvise, unset/empty falls back to SELECTED_MADVISE_THREADS then layer preparation threads (default 8); invalid or 0: 1; values > 16 clamp to 16; non-madvise selected page-in ignores it and uses 1	Sets worker count for selected-expert madvise preparation using the canonical control name.	ds4.c:19288
+runtime/rocm	DS4_ROCM_STREAMING_PREFILL_SELECTED_PROFILE	presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose)	Collect timing/profile diagnostics for rocm streaming prefill selected profile.	ds4.c:18905
+runtime/rocm	DS4_ROCM_STREAMING_PREFILL_SELECTED_READAHEAD_GAP	primary nonempty value over the Metal alias; strict full-string strtoul; unset/empty/invalid: 0; values > 8 clamp to 8	For selected-expert file readahead, merges selected expert runs separated by at most this many unselected expert IDs, reducing readahead calls at the cost of hinting extra weights.	ds4.c:19977
+runtime/rocm	DS4_ROCM_STREAMING_PREFILL_SELECTED_READAHEAD_PROFILE	presence/nonempty diagnostic; unset=off (path-valued DUMP names are noted by purpose)	Collect timing/profile diagnostics for rocm streaming prefill selected readahead profile.	ds4.c:20062
+runtime/rocm	DS4_ROCM_STREAM_CACHE_LAYER_STATS	presence flag; unset=off	Collect per-layer ROCm streaming cache statistics; also enables aggregate stats.	rocm/ds4_rocm_runtime.cuh:390
+runtime/rocm	DS4_ROCM_STREAM_CACHE_STATS	presence flag; unset=off unless layer stats are enabled	Collect aggregate ROCm streaming cache statistics.	rocm/ds4_rocm_runtime.cuh:398
+runtime/rocm	DS4_ROCM_STREAM_EVICT_PAST_LAYERS_FIRST	nonempty and not 0 enables; unset/empty/0=off	Prefer evicting cached experts from already-processed layers.	rocm/ds4_rocm_runtime.cuh:406
+runtime/rocm	DS4_ROCM_STREAM_FREE_RESERVE_GB	integer 2..64 GiB; default 16	Reserve unified-memory headroom while growing the ROCm expert cache.	rocm/ds4_rocm_runtime.cuh:1526
+runtime/rocm	DS4_ROCM_STREAM_MODEL_CACHE_GB	positive GiB integer; unset/invalid uses automatic streaming model cache limit	Cap cached streaming model spans.	rocm/ds4_rocm_runtime.cuh:5464
+runtime/rocm	DS4_ROCM_STREAM_NO_DIRECT	nonempty and not 0 disables direct reads; unset/empty/0 keeps direct I/O eligible	Force the buffered ROCm SSD-streaming read path.	rocm/ds4_rocm_runtime.cuh:1932
+runtime/rocm	DS4_ROCM_STREAM_Q8_F16_CACHE_GB	non-negative GiB integer; unset/invalid uses automatic Q8-F16 cache limit	Cap converted Q8-to-F16 weights in SSD mode.	rocm/ds4_rocm_runtime.cuh:4859
+runtime/rocm	DS4_ROCM_STREAM_READ_PROFILE	nonempty and not 0 enables; unset/empty/0=off	Print ROCm SSD-streaming read/locality statistics at exit.	rocm/ds4_rocm_runtime.cuh:1918
+runtime/rocm	DS4_ROCM_STREAM_READ_WORKERS	integer; default DS4_ROCM_STREAM_READ_DEFAULT_WORKERS; 0 coerces to 1; capped at compile-time max	Set parallel ROCm SSD read/upload workers.	rocm/ds4_rocm_runtime.cuh:2077
+runtime/server	DS4_SERVER_BATCH_LOG	Pure presence flag read once when the decode worker starts; default off. Any defined value, including empty or "0", logs one record per coalesced decode batch with count, elapsed milliseconds and ok/error status.	Observe server-side decode coalescing size, latency and result without changing batching behavior.	ds4_server.c:11090
+runtime/server	DS4_SERVER_DECODE_COALESCE_US	integer 0..100000 microseconds; default 2000; 0 disables wait	Control server micro-batch coalescing delay.	ds4_server.c:11069
+runtime/ssd	DS4_SSD_AUTO_CACHE_PCT	integer 50..95; default 80	Choose the RAM percentage used by automatic SSD expert-cache planning.	ds4_ssd.c:81
+runtime/test-hook	DS4_TEST_METAL_EXACTN_ORACLE	presence flag compiled only with DS4_TEST_HOOKS; unset is off; any defined value enables	Force allocation of the Metal exact-N verifier/oracle workspace in test builds.	ds4.c:61991
+runtime/tp	DS4_TP_ABLATE	comma/list string matched for hcpre,router,kv,compidx; unset=no ablation; must match on both ranks	Skip named TP encode chains for timing; output is semantically wrong.	ds4.c:22520
+runtime/tp	DS4_TP_EVENT_GATES	presence flag; unset uses lower-latency slab flag gates when available	Fall back to Metal shared-event arrival gates.	ds4_metal.m:10768
+runtime/tp	DS4_TP_GATE_PROFILE	presence diagnostic flag; unset=off	Collect timing/profile diagnostics for tp gate profile.	ds4_metal.m:10662
+runtime/tp	DS4_TP_GATE_TRACE	presence diagnostic flag; unset=off	Emit trace diagnostics for tp gate trace.	ds4_tp.c:911
+runtime/tp	DS4_TP_KEEPALIVE_ITERS	atoi unsigned iteration count; default 1200000	Tune work per Metal TP keep-alive dispatch.	ds4_metal.m:10626
+runtime/tp	DS4_TP_KEEPALIVE_TGS	integer 1..2048; invalid/out of range uses 1	Tune threadgroups per Metal TP keep-alive dispatch.	ds4_metal.m:10612
+runtime/tp	DS4_TP_NO_KEEPALIVE	presence flag; unset starts Metal TP keep-alive	Disable the Metal TP GPU keep-alive worker.	ds4_metal.m:10798
+runtime/tp	DS4_TP_PREFILL_SPLIT_MIN	atoi token threshold; default 32; values below 2 clamp to 2	Set when TP prefill row-splits the replicated shared expert.	ds4.c:29192
+runtime/tp	DS4_TP_SUBGATE_PIPELINE	nonempty integer; nonzero enables; default off; must match on both ranks	Enable TP prefill sub-chunk gate pipelining.	ds4.c:29206
+runtime/tp	DS4_TP_TIMEOUT_SEC	atoi seconds stored unsigned; default DS4_TP_DEFAULT_TIMEOUT_SEC	Override TP control/data socket operation timeout.	ds4_tp.c:1329
+runtime/web	DS4_CHROME	executable path; unset auto-detects Chrome/Chromium via standard paths and PATH	Select the browser executable used by web tooling.	ds4_web.c:1014
+script/downloader	DS4_GGUF_DIR	path; default repository gguf/ directory	Choose the model download directory.	download_model.sh:23
+script/downloader	FLATTEN_DOWNLOADS	exact integer 1 enables; unset or 0 preserves normal shard paths	Move downloaded Hugging Face files from nested cache paths into the requested output directory.	download_model.sh:244
+script/downloader	FORCE_HF_DOWNLOAD	exact integer 1 enables; unset or 0 uses the available downloader automatically	Force download_model.sh to use hf download instead of curl when available.	download_model.sh:216
+script/downloader	HF_TOKEN	secret string; unset tries cached Hugging Face token or unauthenticated download	Authenticate Hugging Face downloads.	download_model.sh:28
+script/downloader	HOME	Filesystem directory string. In ds4-agent, unset or empty falls back to "." for the default cache and history roots; the web helper applies the same fallback for its browser profile. A nonempty value roots .ds4/kvcache, .ds4_agent_history and .ds4/browser.	Choose the user's persistent ds4-agent cache, line-history and Chrome-profile base directory.	ds4_agent.c:4023
+script/quality-collector	<NAME_FROM_--api-key-env>	dynamic environment-variable name; no fixed identifier; overrides endpoint-derived key name	Allow a caller-selected credential environment variable.	gguf-tools/quality-testing/collect_official.py:243
+script/quality-collector	DEEPSEEK_API_KEY	secret string; default credential for non-OpenRouter endpoint; required unless --api-key-env selects another name	Authenticate official DeepSeek continuation collection.	gguf-tools/quality-testing/collect_official.py:242
+script/quality-collector	OPENROUTER_API_KEY	secret string; default credential when endpoint contains openrouter.ai; required unless --api-key-env selects another name	Authenticate OpenRouter continuation collection.	gguf-tools/quality-testing/collect_official.py:242
+script/server-wrapper	DS4_BATCHED_SESSIONS	unset/empty defaults to 16; otherwise passed verbatim to --batched-session; the wrapper does not validate it	Sets the maximum batched-session count for the managed CUDA tensor-parallel server.	run-nvidia-tp-server.sh:14
+script/server-wrapper	DS4_CTX	unset/empty defaults to 100000; otherwise passed verbatim to --ctx; the wrapper does not validate it	Sets the managed server context size.	run-nvidia-tp-server.sh:9
+script/server-wrapper	DS4_KV_DIR	directory path; unset/empty defaults to /data/ds4-kv	Sets --kv-disk-dir for the managed server disk-backed KV cache.	run-nvidia-tp-server.sh:12
+script/server-wrapper	DS4_KV_SPACE_MB	unset/empty defaults to 8192; otherwise passed verbatim to --kv-disk-space-mb; the wrapper does not validate it	Sets the managed server disk-KV capacity in MiB.	run-nvidia-tp-server.sh:13
+script/server-wrapper	DS4_LOCK_FILE	lock-file path; unset/empty defaults to /tmp/ds4.lock	Selects the PID/instance lock inspected by start, stop, restart, and status; an explicit environment value is inherited by ds4-server.	run-nvidia-tp-server.sh:15
+script/server-wrapper	DS4_MODEL	model path; unset/empty defaults to /home/antirez/models/deepseek-v4-gguf/DeepSeek-V4-Flash-MXFP4Experts-F16HC-F16Compressor-F16Indexer-Q8Attn-Q8Shared-Q8Out-chat-v2-mxfp4-0731.gguf; unreadable paths fail	Sets the target GGUF passed to the managed CUDA tensor-parallel server.	run-nvidia-tp-server.sh:8
+script/server-wrapper	DS4_SERVER_HOST	host string; unset/empty defaults to 0.0.0.0	Sets the HTTP listen address passed to the managed server.	run-nvidia-tp-server.sh:10
+script/server-wrapper	DS4_SERVER_LOG	log-file path; unset/empty defaults to /tmp/ds4-server.log	Receives detached-server stdout/stderr and supplies the readiness probe and failure tail.	run-nvidia-tp-server.sh:16
+script/server-wrapper	DS4_SERVER_PORT	unset/empty defaults to 8000; otherwise passed verbatim to --port; the wrapper does not validate it	Sets the HTTP listen port passed to the managed server.	run-nvidia-tp-server.sh:11
+script/server-wrapper	DS4_START_TIMEOUT	positive decimal integer with no leading zero; unset/empty defaults to 180; invalid values fail when starting	Sets how many seconds detached startup waits for the lock owner and listening log marker.	run-nvidia-tp-server.sh:17
+script/server-wrapper	DS4_STOP_TIMEOUT	positive decimal integer with no leading zero; unset/empty defaults to 120; invalid values fail when stopping	Sets how many seconds graceful stop waits after SIGTERM before failing.	run-nvidia-tp-server.sh:18
+test-only	DS4_CUDA_TOPK_REGRESSION_SEC	positive floating-point seconds; default 2.0; invalid or nonpositive input restores the default	Set the CUDA large-top-k elapsed-time regression limit.	tests/cuda_long_context_smoke.c:72
+test-only	DS4_METAL_MOE_TILE_MAX	cleanup-only historical spelling; no production consumer exists, so setting it has no runtime effect	Clears a legacy Metal MoE tile override while preparing the test environment.	tests/ds4_test.c:7516
+test-only	DS4_ROCM_ENABLE_Q4_PREFILL_TILE8	cleanup-only legacy spelling; no runtime or test reader; setting it has no effect	Remove a stale opt-in name while preparing ROCm Q4 test cases; TILE8 is automatic.	tests/test_rocm_q4_dense_pair.cpp:58
+test-only	DS4_TEST_ALLOW_FALLBACK	presence flag; unset: native mixed path required; any defined value including empty or 0 permits exactly the serialized-fallback counter outcome	Lets the CUDA mixed prefill/decode oracle accept serialized fallback while retaining bit-exact logit comparisons.	tests/test_cuda_mixed_batch.c:204
+test-only	DS4_TEST_BACKEND	exact cpu selects CPU; every other value, including unset/empty, selects Metal on Apple and CUDA elsewhere	Chooses the backend used by model-backed tests in tests/ds4_test.c.	tests/ds4_test.c:91
+test-only	DS4_TEST_BATCH_ONLY	presence flag; unset: run batched and isolated-control phases; any defined value including empty or 0 stops after the batched archive/hash phase	Runs only the CUDA session-batch phase and skips replay against isolated control sessions.	tests/test_cuda_session_batch.c:288
+test-only	DS4_TEST_CONTEXT	CUDA session/mixed fixtures default to 1024 and require 1024..65536; mixed-batch uses strict full decimal parsing, while session-batch uses atoi and therefore accepts numeric prefixes	Sets the context and placement hint for the CUDA session-batch and mixed prefill/decode oracles.	tests/test_cuda_session_batch.c:125
+test-only	DS4_TEST_DSPARK	nonempty DSpark support-GGUF path; unset/empty skips the DSpark verify-depth test	Loads the DSpark support model for teacher-forced verification of committed speculative tokens.	tests/ds4_test.c:8563
+test-only	DS4_TEST_GPU_DEVICES	GPU device-list string parsed with the normal auto-VRAM parser; unset/empty defaults to 0,2,4,6,1,3,5,7; parse failure is fatal	Selects and orders the CUDA TP/EP devices used by the mixed prefill/decode oracle.	tests/test_cuda_mixed_batch.c:122
+test-only	DS4_TEST_LOCAL_GOLDEN_FILE	nonempty readable fixture path; unset/empty defaults to tests/test-vectors/flash-0731/local-golden.vec	Selects the local-golden vector file used for model-logit regression checks.	tests/ds4_test.c:7501
+test-only	DS4_TEST_LOGPROB_AUTO_METAL	presence flag; unset forces DS4_METAL_DISABLE_METAL4=1; any presence including empty or 0 removes that rollback and permits automatic Metal selection	Runs official log-probability vectors with automatic Metal-path selection instead of the fixed pre-Metal4 baseline.	tests/ds4_test.c:7235
+test-only	DS4_TEST_LONG_PROMPT	nonempty readable prompt-file path; unset/empty defaults to tests/long_context_story_prompt.txt	Selects the rendered story prompt for the long-context fact-recall test.	tests/ds4_test.c:6970
+test-only	DS4_TEST_LONG_WORDS	atoi integer; unset/empty/nonnumeric defaults to 0; valid range is 0..DS4_TEST_CONTEXT-128 and numeric prefixes are accepted	Adds repeated words to alternating CUDA session-batch prompts to exercise long-prefill rows.	tests/test_cuda_session_batch.c:135
+test-only	DS4_TEST_METAL_ARGMAX_TOP1_TIMING	presence flag; unset runs correctness only; any defined value including empty or 0 also runs the GGUF-free resident production-shape A/B	Measures generic argsort versus dedicated Metal top-1 and full-readback versus 4-byte greedy selection using resident synthetic logits, excluding model and SSD I/O.	tests/test_metal_argmax_top1.c:545
+test-only	DS4_TEST_METAL_EXACTN_BATCH_HEAD	nonempty value other than exact 0 enables; unset/empty/0 disables; false/off also enable	Enables the Metal exact-N batch-head path and requires its attempt/use counters for every eligible oracle case.	tests/test_metal_exactn_oracle.c:401
+test-only	DS4_TEST_METAL_EXACTN_ORACLE	presence flag compiled only with DS4_TEST_HOOKS; absent from normal production builds	Force allocation of the exact-N Metal verifier/oracle workspace in tests.	ds4.c:67329
+test-only	DS4_TEST_METAL_Q4_QB_F16_CACHE_TIMING	presence flag; unset runs correctness only; any defined value including empty or 0 also runs resident cold/warm timing	Enables the production-shape timing phase for the Metal Q4_K attn_q_b F16 sidecar oracle.	tests/test_metal_q4_qb_f16_cache.c:59
+test-only	DS4_TEST_METAL_Q4_QB_F16_CACHE_TIMING_TOKENS	strict decimal integer in 32..4096; default 4096	Sets the resident synthetic token count for the Metal Q4_K attn_q_b F16 sidecar timing phase.	tests/test_metal_q4_qb_f16_cache.c:978
+test-only	DS4_TEST_MIXED_INITIAL	integer 128..context-1; default 128	Set the initial prefill length for the CUDA mixed-batch oracle.	tests/test_cuda_mixed_batch.c:111
+test-only	DS4_TEST_MIXED_QUANTUM	integer 1..context-1; default 128	Set the number of prompt tokens added per CUDA mixed-batch round.	tests/test_cuda_mixed_batch.c:113
+test-only	DS4_TEST_MIXED_ROUNDS	integer 1..64; default 3	Set the number of CUDA mixed-batch oracle rounds.	tests/test_cuda_mixed_batch.c:115
+test-only	DS4_TEST_MODEL	nonempty GGUF path; tests/ds4_test.c defaults to ds4flash.gguf, while standalone model-backed CUDA/Metal fixtures generally require a supplied path and fail or skip when absent	Selects the target model shared by model-backed test binaries.	tests/ds4_test.c:14
+test-only	DS4_TEST_MPP_EQ_CASE	comma-separated substring filter; unset/empty runs all cases; tokens are whitespace-trimmed and the filter is truncated to 255 bytes	Restricts Metal tensor-equivalence vectors to IDs containing at least one requested substring.	tests/ds4_test.c:7805
+test-only	DS4_TEST_MTP	nonempty MTP support-GGUF path; unset/empty loads no MTP head; only the fast test engine uses it, with draft depth 4	Enables the legacy MTP verify-depth regression; the test self-skips without this model.	tests/ds4_test.c:104
+test-only	DS4_TEST_Q4_STREAM_ITERS	integer 1..10000; default 2	Set measured iterations for the Metal Q4 stream oracle.	tests/test_metal_q4_streams.c:734
+test-only	DS4_TEST_Q4_STREAM_SOAK	integer 1..100000; default 8	Set bounded overlap-soak iterations for the Metal Q4 stream oracle.	tests/test_metal_q4_streams.c:736
+test-only	DS4_TEST_Q4_STREAM_TIMING	presence flag; unset: correctness/leak checks only; any defined value including empty or 0 also runs timing pairs	Adds FIFO-versus-overlap and native-versus-overlap timing measurements to the Metal Q4 stream oracle.	tests/test_metal_q4_streams.c:737
+test-only	DS4_TEST_Q4_STREAM_TIMING_BLOCKS	integer 5..MAX_TIMING_BLOCKS; default 5	Set the timing block count for the Metal Q4 stream oracle.	tests/test_metal_q4_streams.c:739
+test-only	DS4_TEST_Q4_STREAM_TIMING_ITERS	integer 1..10000; default 20	Set timing iterations for the Metal Q4 stream oracle.	tests/test_metal_q4_streams.c:741
+test-only	DS4_TEST_Q4_STREAM_WARMUP	integer 1..64; default 1	Set warmup iterations for the Metal Q4 stream oracle.	tests/test_metal_q4_streams.c:732
+test-only	DS4_TEST_REQUIRE_MODEL	nonempty value other than exact 0 requires a readable model; unset/empty/0 permits a skip; false/off count as required	Turns a missing Metal exact-N oracle model from a developer skip into a release-gate failure.	tests/test_metal_exactn_oracle.c:390
+test-only	DS4_TEST_REQUIRE_ROCM_DEVICE	nonempty value other than exact 0 requires a visible ROCm device; unset/empty/0 returns the fixture skip code; false/off count as required	Turns absence of a ROCm device from a skip into failure for the ROCm Q4 oracle.	tests/test_rocm_q4_dense_pair.cpp:1569
+test-only	DS4_TEST_SERVER_PREFILL	presence flag; unset: normal prefill; any defined value including empty or 0 installs a no-op display-progress callback	Exercises the progress-split prefill path used by ds4-server in CUDA session-batch and control sessions.	tests/test_cuda_session_batch.c:110
+test-only	DS4_TEST_SESSION_BATCH_ARM	arbitrary nonempty label; unset/empty defaults to unspecified; it is logged only and does not alter execution	Labels the Metal session-batch experiment arm in setup diagnostics.	tests/test_metal_session_batch.c:153
+test-only	DS4_TEST_SESSION_BATCH_TIMING	nonempty boolean; unset/empty/exact 0 disables; every other value enables	Print timing data from the Metal session-batch oracle.	tests/test_metal_session_batch.c:150
+test-only	DS4_TEST_SESSION_COUNT	fixture-specific integer: CUDA session-batch defaults 8 and accepts atoi 2..16; CUDA mixed-batch defaults 8 with strict 3..16; Metal session-batch defaults 2 with strict 2..16	Sets the number of simultaneous sessions exercised by the model-backed batch oracles.	tests/test_cuda_session_batch.c:113
+test-only	DS4_TEST_SSD_CACHE_EXPERTS	strict unsigned integer 30..UINT32_MAX; unset/empty defaults to 30; read only when Metal session-batch SSD streaming is enabled	Sizes the Metal session-batch routed-expert cache used to exercise SSD union policies for N=2..5.	tests/test_metal_session_batch.c:88
+test-only	DS4_TEST_SSD_STREAMING	nonempty value other than exact 0 enables; unset/empty/0 disables; false/off also enable	Runs model-backed test engines through SSD streaming; the Metal session-batch fixture also uses cold mode and shared prefill workspace.	tests/ds4_test.c:109
+test-only	DS4_TEST_SSD_STREAMING_CACHE_EXPERTS	strtoul decimal prefix; unset/empty/nonnumeric becomes 0, values above UINT32_MAX (including a parsed negative) saturate, and trailing text is accepted	Sets the routed-expert cache count on SSD-streaming engines created by tests/ds4_test.c.	tests/ds4_test.c:112
+test-only	DS4_TEST_SSD_STREAMING_CACHE_GB	strtoull decimal GiB prefix; unset/empty/nonnumeric/zero becomes 0, byte overflow (including a parsed negative) saturates to UINT64_MAX, and trailing text is accepted	Sets the routed-expert cache byte budget on SSD-streaming engines created by tests/ds4_test.c.	tests/ds4_test.c:114
+test-only	DS4_TEST_SSD_STREAMING_COLD	nonempty boolean; unset/empty/exact 0 disables; every other value enables	Run test engines in cold SSD-streaming mode and skip hot-expert preload.	tests/ds4_test.c:110
+test-only	DS4_TEST_SSD_STREAMING_PRELOAD_EXPERTS	unsigned integer; default 0; numeric prefixes are accepted and overflow clamps to UINT32_MAX	Set the number of SSD-streaming experts preloaded by test engines.	tests/ds4_test.c:116
+test-only	DS4_TEST_SSD_UNION_POLICY_SWITCH	nonempty boolean; unset/empty/exact 0 disables; every other value enables	Exercise an SSD session-union policy transition in the Metal session-batch oracle.	tests/test_metal_session_batch.c:152
+test-only	DS4_TEST_TP_DISCONNECT	presence flag effective only in leader mode; unset: normal test; any defined value including empty or 0 enters the disconnect oracle	Waits for the TP worker to disconnect, then requires the next batch to fail and every session checkpoint to be invalidated.	tests/test_metal_session_batch.c:268
+test-only	DS4_TEST_TP_LEADER_HOST	nonempty host string required in worker mode; no default; ignored outside worker mode	Sets the TP leader address contacted by the Metal session-batch worker.	tests/test_metal_session_batch.c:212
+test-only	DS4_TEST_TP_LISTEN_HOST	nonempty host string; unset/empty defaults to 0.0.0.0; used only in leader mode	Sets the TP listen address for the Metal session-batch leader.	tests/test_metal_session_batch.c:204
+test-only	DS4_TEST_TP_MODE	unset/empty: no TP; exact leader or worker selects that role; every other nonempty value fails; incompatible with SSD-streaming mode	Selects standalone, TP-leader, or TP-worker execution for the Metal session-batch oracle.	tests/test_metal_session_batch.c:168
+test-only	DS4_TEST_TP_PORT	strict full decimal integer 1..65535; unset/empty defaults to 19452	Sets the listen/connect port shared by Metal session-batch TP leader and worker.	tests/test_metal_session_batch.c:63
+test-only	DS4_TEST_TP_TRANSPORT	unset/empty/auto selects automatic transport; exact tcp or rdma selects that transport; other values fail	Chooses the TP transport for Metal session-batch leader/worker tests.	tests/test_metal_session_batch.c:52
+test-only	DS4_TEST_VECTOR_FILE	nonempty readable vector path; unset/empty defaults to tests/test-vectors/flash-0731/official.vec	Selects the official fixture used by log-probability and Metal tensor-equivalence tests.	tests/ds4_test.c:7222
+test-only	PROTO_Q8_DEBUG	presence diagnostic; unset: summary only; any defined value including empty or 0 prints detailed error structure after a Q8 parity failure	Dumps bad-element tile and row/column histograms for the CUDA Q8 prototype when parity fails.	cuda/mmq/test/proto_gemm_dense_q8_d2r.cu:647
+test-script	DEEPSEEK_API_KEY	secret string; required	Authenticate official test-vector fetch.	tests/test-vectors/fetch_official_vectors.py:236
+test-script	DS4_BIN	executable path; unset/empty defaults to ./ds4; the Q4 matrix requires it executable, the DSpark fixture skips if missing, and the GLM smoke lets command failure fail the test	Selects the ds4 binary launched by model-backed shell test fixtures.	tests/cuda_q4_gb10_fast_matrix.sh:47
+test-script	DS4_CUDA_DISABLE_DSPARK_EXACTN	arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4	Records and passes through the CUDA exact-N rollback used by the acceptance run.	tests/dspark_acceptance_fixture.sh:253
+test-script	DS4_CUDA_DISABLE_DSPARK_EXACTN_BATCH_HEAD	arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4	Records and passes through the CUDA exact-N batch-head rollback.	tests/dspark_acceptance_fixture.sh:255
+test-script	DS4_CUDA_DISABLE_DSPARK_EXACTN_GRAPHS	arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4	Records and passes through the CUDA exact-N graph rollback.	tests/dspark_acceptance_fixture.sh:257
+test-script	DS4_CUDA_DISABLE_DSPARK_NONCAUSAL_ONLINE	arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4	Records and passes through the CUDA noncausal-online-attention rollback.	tests/dspark_acceptance_fixture.sh:266
+test-script	DS4_CUDA_DSPARK_DEVICE_PROPOSER	arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4	Records and passes through the CUDA device-proposer opt-in.	tests/dspark_acceptance_fixture.sh:258
+test-script	DS4_CUDA_DSPARK_EXACT2	arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4	Records and passes through the CUDA exact-2 verifier override.	tests/dspark_acceptance_fixture.sh:250
+test-script	DS4_CUDA_DSPARK_EXACTN	arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4	Records and passes through the CUDA exact-N verifier opt-in.	tests/dspark_acceptance_fixture.sh:252
+test-script	DS4_CUDA_DSPARK_EXACTN_BATCH_HEAD	arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4	Records and passes through the CUDA exact-N batch-head opt-in.	tests/dspark_acceptance_fixture.sh:254
+test-script	DS4_CUDA_DSPARK_EXACTN_GRAPHS	arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4	Records and passes through the CUDA exact-N graph opt-in.	tests/dspark_acceptance_fixture.sh:256
+test-script	DS4_CUDA_DSPARK_NO_DEVICE_PROPOSER	arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4	Records and passes through the CUDA device-proposer rollback.	tests/dspark_acceptance_fixture.sh:259
+test-script	DS4_CUDA_DSPARK_PROPOSER_BLOCK_MAX	arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4	Records and passes through the CUDA proposer block-size cap.	tests/dspark_acceptance_fixture.sh:269
+test-script	DS4_CUDA_ENABLE_DSPARK_NONCAUSAL_ONLINE	arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4	Records and passes through the CUDA noncausal-online-attention opt-in.	tests/dspark_acceptance_fixture.sh:265
+test-script	DS4_CUDA_Q4_MATRIX_CTX	nonempty decimal digits other than exact 0; unset/empty defaults to 4096; no upper bound; leading-zero zero strings such as 00 pass the script's guard	Sets --ctx for every Q4 GB10 smoke and score_official matrix arm.	tests/cuda_q4_gb10_fast_matrix.sh:49
+test-script	DS4_CUDA_Q4_MATRIX_DECODE_GRAPHS	exact default, 0, or 1; unset/empty defaults to default; other values fail	Leaves decode graphs automatic, forces them off, or forces them on with capture logging for every matrix arm.	tests/cuda_q4_gb10_fast_matrix.sh:56
+test-script	DS4_CUDA_Q4_MATRIX_PROMPT	prompt string; unset/empty defaults to Write a complete Python quicksort function with comments.	Sets the deterministic smoke prompt whose log-probability output is compared across Q4 fast-path arms.	tests/cuda_q4_gb10_fast_matrix.sh:52
+test-script	DS4_CUDA_Q4_MATRIX_SCORER	executable path; unset/empty defaults to gguf-tools/quality-testing/score_official; a missing/nonexecutable path fails	Selects the scorer used to produce quality TSVs for each non-oracle Q4 matrix arm.	tests/cuda_q4_gb10_fast_matrix.sh:48
+test-script	DS4_CUDA_Q4_MATRIX_SKIP_PARITY	exact 0 or 1; unset/empty defaults to 0; other values fail	When 1, skips the synthetic MMQ parity prerequisite and marks the resulting QA run incomplete.	tests/cuda_q4_gb10_fast_matrix.sh:57
+test-script	DS4_CUDA_Q4_MATRIX_SSD_CACHE	unset/empty by default; required and passed verbatim as --ssd-streaming-cache-experts when streaming=1; must remain empty when streaming=0; no further validation	Sets the SSD expert cache as a count or NGB value for every streamed matrix arm.	tests/cuda_q4_gb10_fast_matrix.sh:54
+test-script	DS4_CUDA_Q4_MATRIX_SSD_PRELOAD	unset/empty omits preload; a nonempty value is passed verbatim as --ssd-streaming-preload-experts and is allowed only when streaming=1	Sets optional expert preload for streamed Q4 smoke and scoring arms.	tests/cuda_q4_gb10_fast_matrix.sh:55
+test-script	DS4_CUDA_Q4_MATRIX_SSD_STREAMING	exact 0 or 1; unset/empty defaults to 0; other values fail	Runs all Q4 matrix arms resident or with SSD streaming and enforces matching cache/preload arguments.	tests/cuda_q4_gb10_fast_matrix.sh:53
+test-script	DS4_CUDA_Q4_MATRIX_TOKENS	nonempty decimal digits other than exact 0; unset/empty defaults to 32; no upper bound; leading-zero zero strings such as 00 pass the script's guard	Sets the continuation length for each Q4 GB10 smoke arm.	tests/cuda_q4_gb10_fast_matrix.sh:50
+test-script	DS4_CUDA_Q4_MATRIX_TOP_K	nonempty decimal digits other than exact 0 and numerically <=128; unset/empty defaults to 128; leading-zero zero strings such as 00 pass the guard	Sets --logprobs-top-k for the byte-comparable Q4 GB10 smoke dumps.	tests/cuda_q4_gb10_fast_matrix.sh:51
+test-script	DS4_DSPARK_FIXTURE_BACKEND	unset/empty defaults to auto; exact auto, metal, cuda, or rocm accepted; every other value fails	Chooses the explicit backend flag for baseline and DSpark acceptance runs; auto passes none.	tests/dspark_acceptance_fixture.sh:14
+test-script	DS4_DSPARK_FIXTURE_CONFIDENCE	unset/empty omits the option and uses the runtime default; otherwise passed verbatim to --dspark-confidence without fixture-side validation	Overrides the DSpark confidence threshold for acceptance runs and records it in metadata.	tests/dspark_acceptance_fixture.sh:13
+test-script	DS4_DSPARK_FIXTURE_C_ADD_MIN_ACCEPTED	decimal digits including 0; unset/empty defaults to 8; non-digits fail	Sets the minimum accepted-draft count for the c_add case when the proposal-quality guard is active.	tests/dspark_acceptance_fixture.sh:12
+test-script	DS4_DSPARK_FIXTURE_REQUIRE_ACTIVE	exact 0 or 1; unset/empty defaults to 1; other values fail	When 1, requires aggregate proposed and accepted-draft counts to both be nonzero.	tests/dspark_acceptance_fixture.sh:17
+test-script	DS4_DSPARK_FIXTURE_REQUIRE_CUDA_DEVICE_PROPOSER	exact 0 or 1; unset/empty defaults to 0; other values fail; 1 also forces byte-identical output checking	Requires CUDA device-proposer attempts to equal uses, with nonzero use and zero fallback or policy mismatch.	tests/dspark_acceptance_fixture.sh:22
+test-script	DS4_DSPARK_FIXTURE_REQUIRE_CUDA_EXACTN	exact 0 or 1; unset/empty defaults to 0; other values fail; 1 also forces byte-identical output checking	Requires at least one CUDA exact-N attempt and zero exact-N error fallbacks.	tests/dspark_acceptance_fixture.sh:19
+test-script	DS4_DSPARK_FIXTURE_REQUIRE_CUDA_EXACTN_BATCH_HEAD	exact 0 or 1; unset/empty defaults to 0; other values fail; 1 implies REQUIRE_CUDA_EXACTN and identical output	Requires nonzero CUDA exact-N batch-head attempts/uses and zero batch-head fallbacks.	tests/dspark_acceptance_fixture.sh:20
+test-script	DS4_DSPARK_FIXTURE_REQUIRE_CUDA_EXACTN_GRAPHS	exact 0 or 1; unset/empty defaults to 0; other values fail; 1 implies REQUIRE_CUDA_EXACTN and identical output	Requires CUDA exact-N graph attempts, uses, captures, and replays, with zero no-slot or graph failures.	tests/dspark_acceptance_fixture.sh:21
+test-script	DS4_DSPARK_FIXTURE_REQUIRE_DIRECT_COMMIT	exact 0 or 1; unset/empty defaults to 0; other values fail	Requires at least one direct verifier-state commit; with REQUIRE_PARTIAL it also requires a direct partial commit.	tests/dspark_acceptance_fixture.sh:9
+test-script	DS4_DSPARK_FIXTURE_REQUIRE_EXACT2	exact 0 or 1; unset/empty defaults to 0; other values fail; 1 also forces byte-identical output checking	Requires at least one exact-2 attempt and zero exact-2 fallbacks.	tests/dspark_acceptance_fixture.sh:18
+test-script	DS4_DSPARK_FIXTURE_REQUIRE_IDENTICAL	exact 0 or 1; unset/empty defaults to 0; other values fail; several path-specific requirements force it to 1	When 1, fails any byte difference between baseline and DSpark stdout; otherwise mismatches are reported but allowed.	tests/dspark_acceptance_fixture.sh:10
+test-script	DS4_DSPARK_FIXTURE_REQUIRE_METAL_DEVICE_PROPOSER	exact 0 or 1; unset/empty defaults to 0; other values fail; 1 also forces byte-identical output checking	Requires Metal device-proposer attempts to equal uses, with nonzero use and zero fallback or policy mismatch.	tests/dspark_acceptance_fixture.sh:25
+test-script	DS4_DSPARK_FIXTURE_REQUIRE_METAL_EXACTN_BATCH_HEAD	exact 0 or 1; unset/empty defaults to 0; other values fail; 1 also forces byte-identical output checking	Requires nonzero Metal exact-N batch-head attempts/uses and zero batch-head fallbacks.	tests/dspark_acceptance_fixture.sh:23
+test-script	DS4_DSPARK_FIXTURE_REQUIRE_METAL_EXACTN_PARTIAL	exact 0 or 1; unset/empty defaults to 0; other values fail; 1 also forces byte-identical output checking	Requires a Metal exact-N partial replay, matching verify-skip count, and zero union error fallbacks.	tests/dspark_acceptance_fixture.sh:24
+test-script	DS4_DSPARK_FIXTURE_REQUIRE_PARTIAL	unset/empty defaults to 0; exact 0 disables; any other value enables, with no 0/1 validation	Requires at least one partial-accept case and, together with REQUIRE_DIRECT_COMMIT, one direct partial commit.	tests/dspark_acceptance_fixture.sh:8
+test-script	DS4_DSPARK_FIXTURE_REQUIRE_PROPOSAL_QUALITY	unset/empty/auto selects auto; 0/false/no/off disables; 1/true/yes/on enables; values are lowercase and other strings fail; auto enables only without partial mode/confidence override and with tokens >=32	Controls the c_add minimum-accepted-drafts quality guard.	tests/dspark_acceptance_fixture.sh:11
+test-script	DS4_DSPARK_FIXTURE_SSD_STREAMING	exact 0 or 1; unset/empty defaults to 0; other values fail	Adds --ssd-streaming to both baseline and DSpark runs when enabled.	tests/dspark_acceptance_fixture.sh:15
+test-script	DS4_DSPARK_FIXTURE_SSD_STREAMING_CACHE_EXPERTS	unset/empty omits the cache option; otherwise decimal digits including 0 are required, and a value is legal only with SSD streaming enabled	Passes an explicit --ssd-streaming-cache-experts value to both acceptance-run variants.	tests/dspark_acceptance_fixture.sh:16
+test-script	DS4_DSPARK_FIXTURE_TOKENS	token-count argument; unset/empty defaults to 32; the fixture does not validate it before passing --tokens, and auto quality treats nonnumeric or <32 as ineligible	Sets generated-token count for each baseline and DSpark acceptance case.	tests/dspark_acceptance_fixture.sh:7
+test-script	DS4_DSPARK_MODEL	target-model path; unset/empty falls back to DS4_TEST_MODEL, then ./ds4flash.gguf; a missing file causes a successful skip	Selects the target GGUF compared in baseline and DSpark acceptance runs.	tests/dspark_acceptance_fixture.sh:5
+test-script	DS4_DSPARK_SSD_VERIFY_BLOCK_MAX	unsigned integer rows; default/fallback 0 means automatic policy; numeric prefixes accepted; used both as verifier cap and as an exact-2 proposer-policy discriminator	Cap speculative rows verified from SSD and influence exact-2 proposal sizing.	tests/dspark_acceptance_fixture.sh:271
+test-script	DS4_DSPARK_SUPPORT	support-model path; unset/empty defaults to gguf/DeepSeek-V4-Flash-DSpark-support-0731.gguf; a missing file causes a successful skip	Selects the DSpark support GGUF passed through --mtp.	tests/dspark_acceptance_fixture.sh:6
+test-script	DS4_DSPARK_VERIFY_NONCAUSAL	presence diagnostic sampled once after the first successfully submitted CUDA noncausal-attention kernel; unset: verify 0 calls; any presence including empty or 0: verify that call and the next 2	CUDA only: synchronizes and reads back Q/KV/output, computes the DSpark noncausal attention CPU reference, and logs max absolute/relative error; it reports only and does not fail the operation.	tests/dspark_acceptance_fixture.sh:267
+test-script	DS4_GLM_BACKEND	exact metal, cuda, or cpu; unset/empty defaults to metal; other values fail	Selects the backend flag used by the GLM long-context continuation smoke test.	tests/glm_long_context_smoke.sh:29
+test-script	DS4_GLM_EXTRA_ARGS	unset/empty adds no arguments; otherwise intentionally unquoted and therefore shell field-split and pathname-expanded	Adds backend/device options to the ds4 invocation used for every GLM long-context case.	tests/glm_long_context_smoke.sh:91
+test-script	DS4_GLM_LONG_CONTEXT_CTX	context argument; unset/empty defaults to 100000; forwarded to --ctx without script-side validation	Sets the context size for GLM long-context smoke invocations.	tests/glm_long_context_smoke.sh:26
+test-script	DS4_GLM_LONG_CONTEXT_GEN	generation-count argument; unset/empty defaults to 32; forwarded to -n without script-side validation	Sets the number of continuation tokens checked by each GLM long-context case.	tests/glm_long_context_smoke.sh:28
+test-script	DS4_GLM_LONG_CONTEXT_REPEATS	whitespace-separated list of prompt-padding counts; unset/empty defaults to the single count 130; each item must work as a shell integer	Chooses one or more audit-block counts used to construct long GLM prompts.	tests/glm_long_context_smoke.sh:27
+test-script	DS4_GLM_MODEL	model path used only when no nonempty positional MODEL is supplied; unset/empty defaults to models/GLM-5.2-UD-Q4_K_XL.gguf	Selects the GLM-5.2 GGUF used by the long-context continuation smoke test.	tests/glm_long_context_smoke.sh:25
+test-script	DS4_METAL_DISABLE_DSPARK_EXACTN_BATCH_HEAD	arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4	Records and passes through the Metal exact-N batch-head rollback.	tests/dspark_acceptance_fixture.sh:262
+test-script	DS4_METAL_DSPARK_DEVICE_PROPOSER	arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4	Records and passes through the Metal device-proposer opt-in.	tests/dspark_acceptance_fixture.sh:263
+test-script	DS4_METAL_DSPARK_EXACT2	arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4	Records and passes through the Metal exact-2 verifier override.	tests/dspark_acceptance_fixture.sh:251
+test-script	DS4_METAL_DSPARK_EXACTN_BATCH_HEAD	arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4	Records and passes through the Metal exact-N batch-head opt-in.	tests/dspark_acceptance_fixture.sh:261
+test-script	DS4_METAL_DSPARK_EXACTN_UNION	arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4	Records and passes through the Metal exact-N union-verifier opt-in.	tests/dspark_acceptance_fixture.sh:260
+test-script	DS4_METAL_DSPARK_EXACT_ROWS_ASYNC_TAILS	arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4	Records and passes through the Metal exact-row asynchronous-tail override.	tests/dspark_acceptance_fixture.sh:268
+test-script	DS4_METAL_DSPARK_NO_DEVICE_PROPOSER	arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4	Records and passes through the Metal device-proposer rollback.	tests/dspark_acceptance_fixture.sh:264
+test-script	DS4_METAL_DSPARK_PROPOSER_BLOCK_MAX	arbitrary inherited runtime value; the fixture neither parses nor changes it; unset/empty is printed as unset in metadata, and the environment is passed unchanged to ds4	Records and passes through the Metal proposer block-size cap.	tests/dspark_acceptance_fixture.sh:270
+test-script	DS4_TEST_MODEL	fallback target-model path used only when DS4_DSPARK_MODEL is unset/empty; unset/empty then defaults to ./ds4flash.gguf	Provides the shared test-model fallback for the DSpark acceptance fixture.	tests/dspark_acceptance_fixture.sh:5
+test-script	OPENROUTER_API_KEY	secret string; required	Authenticate OpenRouter GLM test-vector fetch.	tests/test-vectors/fetch_openrouter_glm_vectors.py:355
+test-script	TMPDIR	temporary-directory base path; unset/empty defaults to /tmp	Chooses the parent directory for auto-created Q4 matrix, DSpark fixture, and GLM smoke work directories.	tests/cuda_q4_gb10_fast_matrix.sh:108
diff --git a/scripts/generate_environment_variables.py b/scripts/generate_environment_variables.py
new file mode 100644
index 0000000000..85185f569b
--- /dev/null
+++ b/scripts/generate_environment_variables.py
@@ -0,0 +1,584 @@
+#!/usr/bin/env python3
+"""Generate and verify the complete environment-variable reference."""
+
+from __future__ import annotations
+
+import argparse
+import bisect
+import csv
+import os
+import re
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Dict, Iterable, List, Mapping, MutableMapping, Sequence, Set, Tuple
+
+
+ROOT = Path(__file__).resolve().parents[1]
+DOCUMENT = ROOT / "ENVIRONMENT_VARIABLES.md"
+METADATA = ROOT / "scripts" / "environment_variables.tsv"
+START_MARKER = "<!-- BEGIN GENERATED ENVIRONMENT VARIABLE INVENTORY -->"
+END_MARKER = "<!-- END GENERATED ENVIRONMENT VARIABLE INVENTORY -->"
+
+SOURCE_SUFFIXES = {".c", ".cc", ".cpp", ".cu", ".cuh", ".h", ".inc", ".m", ".mm"}
+PRODUCTION_EXCLUDED_DIRS = {
+    ".git",
+    "dir-steering",
+    "tests",
+}
+
+# These uppercase DS4 strings are protocol markers, compile-time identifiers,
+# or display names rather than environment-variable names.  Treat every other
+# DS4_* string literal in production C-family sources as an environment input.
+NON_ENV_DS4_STRINGS = {
+    "DS4_CUDA",
+    "DS4_IMATRIX_PROMPT",
+    "DS4_MAX_GPUS",
+    "DS4_METAL_HAS_TENSOR",
+    "DS4_SORT_ORDER_ASC",
+    "DS4_SORT_ORDER_DESC",
+}
+TEST_NON_ENV_DS4_STRINGS = {"DS4_N_LAYER"}
+
+GROUPS = (
+    ("Metal", "DS4_METAL_"),
+    ("CUDA", "DS4_CUDA_"),
+    ("ROCm", "DS4_ROCM_"),
+    ("GLM shared", "DS4_GLM_"),
+    ("Distributed", "DS4_DIST_"),
+    ("DSpark shared", "DS4_DSPARK_"),
+)
+
+DS4_TOKEN_RE = re.compile(r"\bDS4_[A-Z][A-Z0-9_]*\b")
+C_STRING_RE = re.compile(r'(?:u8|u|U|L)?"(?:\\.|[^"\\])*"')
+DIRECT_GETENV_RE = re.compile(
+    r'(?:std::)?getenv\s*\(\s*"([A-Z][A-Z0-9_]*)"\s*\)', re.MULTILINE
+)
+PYTHON_ENV_RE = re.compile(
+    r'os\.(?:environ\.get|getenv)\s*\(\s*["\']([A-Z][A-Z0-9_]*)["\']'
+)
+PYTHON_API_KEY_RE = re.compile(r'["\']([A-Z][A-Z0-9_]*_API_KEY)["\']')
+SHELL_DEFAULT_RE = re.compile(r"\$\{([A-Z][A-Z0-9_]*):-([^}\n]*)\}")
+SHELL_ENV_RE = re.compile(r"\$\{([A-Z][A-Z0-9_]*)(?::?[-+?=])")
+SHELL_DEFAULT_OVERRIDES = {
+    "DS4_DSPARK_MODEL": "${DS4_TEST_MODEL:-./ds4flash.gguf}",
+}
+PLACEHOLDER_METADATA_PHRASES = (
+    "ambiguous:",
+    "call-site specific",
+    "configure or diagnose ",
+    "configure or require test fixture",
+    "configure test script",
+    "configure the managed nvidia",
+    "test-script flag/value",
+    "string/path test input",
+    "integer test parameter",
+    "test/script input string or numeric value",
+    "semantics owned by os/linenoise/vendor helper",
+)
+
+
+@dataclass(frozen=True)
+class Occurrence:
+    path: Path
+    line: int
+    line_text: str
+    context: str
+
+
+@dataclass(frozen=True)
+class VariableDoc:
+    scope: str
+    name: str
+    value_default: str
+    purpose: str
+    source: str
+
+
+def relative(path: Path) -> Path:
+    return path.resolve().relative_to(ROOT)
+
+
+def source_files() -> List[Path]:
+    files: List[Path] = []
+    for directory, dirnames, filenames in os.walk(ROOT):
+        directory_path = Path(directory)
+        rel_dir = relative(directory_path)
+        if directory_path == ROOT:
+            dirnames[:] = [name for name in dirnames if name not in PRODUCTION_EXCLUDED_DIRS]
+        elif rel_dir.parts[:2] == ("cuda", "mmq"):
+            dirnames[:] = [name for name in dirnames if name != "test"]
+        for filename in filenames:
+            path = directory_path / filename
+            if path.suffix.lower() in SOURCE_SUFFIXES:
+                files.append(path)
+    return sorted(files, key=lambda item: str(relative(item)))
+
+
+def test_files() -> List[Path]:
+    files = [
+        path
+        for path in (ROOT / "tests").rglob("*")
+        if path.is_file() and path.suffix.lower() in SOURCE_SUFFIXES | {".py", ".sh"}
+    ]
+    mmq_tests = ROOT / "cuda" / "mmq" / "test"
+    if mmq_tests.exists():
+        files.extend(
+            path
+            for path in mmq_tests.rglob("*")
+            if path.is_file() and path.suffix.lower() in SOURCE_SUFFIXES | {".py", ".sh"}
+        )
+    return sorted(set(files), key=lambda item: str(relative(item)))
+
+
+def add_line_occurrence(
+    result: MutableMapping[str, List[Occurrence]],
+    name: str,
+    path: Path,
+    lines: Sequence[str],
+    line_index: int,
+) -> None:
+    first = max(0, line_index - 5)
+    last = min(len(lines), line_index + 6)
+    result.setdefault(name, []).append(
+        Occurrence(
+            relative(path),
+            line_index + 1,
+            lines[line_index].strip(),
+            "\n".join(lines[first:last]),
+        )
+    )
+
+
+def scan_ds4_string_literals(files: Iterable[Path]) -> Dict[str, List[Occurrence]]:
+    result: Dict[str, List[Occurrence]] = {}
+    for path in files:
+        lines = path.read_text(encoding="utf-8", errors="replace").splitlines()
+        for line_index, line in enumerate(lines):
+            for literal in C_STRING_RE.finditer(line):
+                for token in DS4_TOKEN_RE.finditer(literal.group(0)):
+                    add_line_occurrence(result, token.group(0), path, lines, line_index)
+    return result
+
+
+def scan_direct_getenv(files: Iterable[Path]) -> Dict[str, List[Occurrence]]:
+    result: Dict[str, List[Occurrence]] = {}
+    for path in files:
+        text = path.read_text(encoding="utf-8", errors="replace")
+        lines = text.splitlines()
+        line_starts = [0]
+        line_starts.extend(match.end() for match in re.finditer("\n", text))
+
+        matches = list(DIRECT_GETENV_RE.finditer(text))
+        if path.suffix == ".py":
+            matches.extend(PYTHON_ENV_RE.finditer(text))
+            matches.extend(PYTHON_API_KEY_RE.finditer(text))
+        for match in sorted(matches, key=lambda item: item.start()):
+            line_index = bisect.bisect_right(line_starts, match.start()) - 1
+            add_line_occurrence(result, match.group(1), path, lines, line_index)
+    return result
+
+
+def scan_shell_inputs(files: Iterable[Path]) -> Tuple[Dict[str, List[Occurrence]], Dict[str, str]]:
+    result: Dict[str, List[Occurrence]] = {}
+    defaults: Dict[str, str] = {}
+    for path in files:
+        if path.suffix != ".sh":
+            continue
+        lines = path.read_text(encoding="utf-8", errors="replace").splitlines()
+        for line_index, line in enumerate(lines):
+            for match in SHELL_ENV_RE.finditer(line):
+                add_line_occurrence(result, match.group(1), path, lines, line_index)
+            for match in SHELL_DEFAULT_RE.finditer(line):
+                defaults.setdefault(match.group(1), match.group(2).strip())
+    for name, default in SHELL_DEFAULT_OVERRIDES.items():
+        if name in result:
+            defaults[name] = default
+    return result, defaults
+
+
+def load_metadata() -> List[VariableDoc]:
+    try:
+        handle = METADATA.open("r", encoding="utf-8", newline="")
+    except OSError as error:
+        raise ValueError(f"cannot read {METADATA.relative_to(ROOT)}: {error}") from error
+    with handle:
+        reader = csv.reader(handle, delimiter="\t")
+        rows = list(reader)
+    expected_header = ["SCOPE", "NAME", "VALUE_DEFAULT_SEMANTICS", "PURPOSE", "SOURCE"]
+    if not rows or rows[0] != expected_header:
+        raise ValueError(f"{METADATA.relative_to(ROOT)} has an invalid header")
+
+    result: List[VariableDoc] = []
+    seen: Set[Tuple[str, str]] = set()
+    for line, row in enumerate(rows[1:], 2):
+        if len(row) != 5 or any(not field.strip() for field in row):
+            raise ValueError(
+                f"{METADATA.relative_to(ROOT)}:{line}: expected five nonempty TSV fields"
+            )
+        item = VariableDoc(*(field.strip() for field in row))
+        searchable = f"{item.value_default}\n{item.purpose}".lower()
+        placeholder = next(
+            (phrase for phrase in PLACEHOLDER_METADATA_PHRASES if phrase in searchable),
+            None,
+        )
+        if placeholder:
+            raise ValueError(
+                f"{METADATA.relative_to(ROOT)}:{line}: placeholder metadata "
+                f"{placeholder!r} remains for {item.name}"
+            )
+        key = (item.scope, item.name)
+        if key in seen:
+            raise ValueError(
+                f"{METADATA.relative_to(ROOT)}:{line}: duplicate scope/name {key!r}"
+            )
+        seen.add(key)
+        result.append(item)
+
+    expected_order = sorted(result, key=lambda item: (item.scope, item.name))
+    if result != expected_order:
+        raise ValueError(
+            f"{METADATA.relative_to(ROOT)} must be sorted by SCOPE and NAME"
+        )
+
+    source_cache: Dict[Path, List[str]] = {}
+    for item in result:
+        found_name = item.name.startswith("<")
+        for reference in (part.strip() for part in item.source.split(";")):
+            match = re.fullmatch(r"(.+):(\d+)", reference)
+            if not match:
+                raise ValueError(
+                    f"{METADATA.relative_to(ROOT)}: invalid source reference {reference!r}"
+                )
+            rel_path, line_text = match.groups()
+            path = ROOT / rel_path
+            if path not in source_cache:
+                try:
+                    source_cache[path] = path.read_text(
+                        encoding="utf-8", errors="replace"
+                    ).splitlines()
+                except OSError as error:
+                    raise ValueError(f"cannot read metadata source {rel_path}: {error}") from error
+            lines = source_cache[path]
+            line = int(line_text)
+            if line < 1 or line > len(lines):
+                raise ValueError(f"metadata source is outside {rel_path}: {line}")
+            first = max(0, line - 6)
+            last = min(len(lines), line + 5)
+            if item.name in "\n".join(lines[first:last]):
+                found_name = True
+        if not found_name:
+            raise ValueError(
+                f"metadata source for {item.scope}/{item.name} does not mention the name"
+            )
+    return result
+
+
+def metadata_source_links(source: str) -> str:
+    links: List[str] = []
+    for value in (part.strip() for part in source.split(";")):
+        match = re.fullmatch(r"(.+):(\d+)", value)
+        if not match:
+            links.append(f"`{value}`")
+            continue
+        path, line = match.groups()
+        links.append(f"[{path}:{line}]({path}#L{line})")
+    return "; ".join(links)
+
+
+def metadata_table(entries: Sequence[VariableDoc]) -> List[str]:
+    lines = [
+        "| Variable | Accepted value and default | Effect | Source |",
+        "| --- | --- | --- | --- |",
+    ]
+    for item in sorted(entries, key=lambda entry: entry.name):
+        lines.append(
+            "| `{}` | {} | {} | {} |".format(
+                item.name,
+                escape_cell(item.value_default),
+                escape_cell(item.purpose),
+                metadata_source_links(item.source),
+            )
+        )
+    return lines
+
+
+def group_for(name: str) -> str:
+    for title, prefix in GROUPS:
+        if name.startswith(prefix):
+            return title
+    return "General and shared"
+
+
+def escape_cell(value: str) -> str:
+    return value.replace("|", "\\|").replace("\n", " ")
+
+
+def merge_occurrences(
+    *mappings: Mapping[str, Sequence[Occurrence]],
+) -> Dict[str, List[Occurrence]]:
+    result: Dict[str, List[Occurrence]] = {}
+    for mapping in mappings:
+        for name, occurrences in mapping.items():
+            result.setdefault(name, []).extend(occurrences)
+    return result
+
+
+def auxiliary_inputs(
+    runtime_names: Set[str], external_runtime_names: Set[str]
+) -> Tuple[Dict[str, List[Occurrence]], Dict[str, str], Dict[str, List[Occurrence]], Dict[str, str]]:
+    tests = test_files()
+    test_direct = scan_direct_getenv(tests)
+    test_literals = scan_ds4_string_literals(
+        path for path in tests if path.suffix.lower() in SOURCE_SUFFIXES
+    )
+    test_shell, test_defaults = scan_shell_inputs(tests)
+    test_all = merge_occurrences(test_direct, test_literals, test_shell)
+    test_only = {
+        name: occurrences
+        for name, occurrences in test_all.items()
+        if name not in runtime_names and name not in external_runtime_names
+        and name not in NON_ENV_DS4_STRINGS
+        and name not in TEST_NON_ENV_DS4_STRINGS
+        and (name.startswith("DS4_") or name in {"DEEPSEEK_API_KEY", "OPENROUTER_API_KEY", "PROTO_Q8_DEBUG", "TMPDIR"})
+    }
+
+    tool_files: List[Path] = []
+    for directory, dirnames, filenames in os.walk(ROOT):
+        directory_path = Path(directory)
+        rel_dir = relative(directory_path)
+        dirnames[:] = [
+            name
+            for name in dirnames
+            if name not in {".git", "__pycache__", "out", "dataset", "tests"}
+            and not name.endswith(".dSYM")
+        ]
+        if rel_dir.parts[:2] == ("cuda", "mmq"):
+            dirnames[:] = [name for name in dirnames if name != "test"]
+        for filename in filenames:
+            path = directory_path / filename
+            if path.suffix not in {".py", ".sh"}:
+                continue
+            if path.resolve() == Path(__file__).resolve():
+                continue
+            tool_files.append(path)
+    tool_direct = scan_direct_getenv(tool_files)
+    tool_shell, tool_defaults = scan_shell_inputs(tool_files)
+    tool_all = {
+        name: occurrences
+        for name, occurrences in merge_occurrences(tool_direct, tool_shell).items()
+        if name.startswith("DS4_")
+        or name in {"DEEPSEEK_API_KEY", "FLATTEN_DOWNLOADS", "FORCE_HF_DOWNLOAD", "HF_TOKEN", "OPENROUTER_API_KEY"}
+    }
+    return test_only, test_defaults, tool_all, tool_defaults
+
+
+def generated_block() -> Tuple[str, int, int, int, int]:
+    production = source_files()
+    literal_occurrences = scan_ds4_string_literals(production)
+    unexpected_allowlist = NON_ENV_DS4_STRINGS - set(literal_occurrences)
+    if unexpected_allowlist:
+        names = ", ".join(sorted(unexpected_allowlist))
+        raise ValueError(f"stale NON_ENV_DS4_STRINGS entries: {names}")
+
+    runtime_names = set(literal_occurrences) - NON_ENV_DS4_STRINGS
+    direct = scan_direct_getenv(production)
+    external_names = {name for name in direct if not name.startswith("DS4_")}
+
+    metadata = load_metadata()
+    runtime_docs: Dict[str, VariableDoc] = {}
+    external_docs: Dict[str, VariableDoc] = {}
+    for item in metadata:
+        target: Dict[str, VariableDoc]
+        if item.scope.startswith("runtime/"):
+            target = runtime_docs
+        elif item.scope == "external/system":
+            target = external_docs
+        else:
+            continue
+        if item.name in target:
+            raise ValueError(f"duplicate documented runtime name: {item.name}")
+        target[item.name] = item
+
+    missing_runtime = runtime_names - set(runtime_docs)
+    stale_runtime = set(runtime_docs) - runtime_names
+    missing_external = external_names - set(external_docs)
+    stale_external = set(external_docs) - external_names
+    if missing_runtime or stale_runtime or missing_external or stale_external:
+        details: List[str] = []
+        if missing_runtime:
+            details.append("undocumented runtime: " + ", ".join(sorted(missing_runtime)))
+        if stale_runtime:
+            details.append("stale runtime metadata: " + ", ".join(sorted(stale_runtime)))
+        if missing_external:
+            details.append("undocumented external runtime: " + ", ".join(sorted(missing_external)))
+        if stale_external:
+            details.append("stale external metadata: " + ", ".join(sorted(stale_external)))
+        raise ValueError("; ".join(details))
+
+    test_binary = [item for item in metadata if item.scope == "test-only"]
+    test_scripts = [item for item in metadata if item.scope == "test-script"]
+    tools = [item for item in metadata if item.scope.startswith("script/")]
+
+    scanned_tests, _test_defaults, scanned_tools, _tool_defaults = auxiliary_inputs(
+        runtime_names, external_names
+    )
+    documented_test_names = {
+        item.name for item in test_binary + test_scripts
+    } - runtime_names - external_names
+    if set(scanned_tests) != documented_test_names:
+        missing = set(scanned_tests) - documented_test_names
+        stale = documented_test_names - set(scanned_tests)
+        details = []
+        if missing:
+            details.append("undocumented test input: " + ", ".join(sorted(missing)))
+        if stale:
+            details.append("stale test metadata: " + ", ".join(sorted(stale)))
+        raise ValueError("; ".join(details))
+
+    documented_tool_names = {item.name for item in tools}
+    allowed_dynamic_tool_names = {"<NAME_FROM_--api-key-env>", "HOME"}
+    if set(scanned_tools) != documented_tool_names - allowed_dynamic_tool_names:
+        missing = set(scanned_tools) - documented_tool_names
+        stale = documented_tool_names - allowed_dynamic_tool_names - set(scanned_tools)
+        details = []
+        if missing:
+            details.append("undocumented tool input: " + ", ".join(sorted(missing)))
+        if stale:
+            details.append("stale tool metadata: " + ", ".join(sorted(stale)))
+        raise ValueError("; ".join(details))
+
+    lines = [
+        START_MARKER,
+        "## Complete implementation inventory",
+        "",
+        "This section is generated by `scripts/generate_environment_variables.py`; do not edit it by hand.",
+        "Human-reviewed value/default and purpose metadata lives in",
+        "`scripts/environment_variables.tsv`; the generator verifies it against the source tree.",
+        "It lists every `DS4_*` string consumed by production C/C++/Objective-C/CUDA/ROCm",
+        "sources, including names passed indirectly through helper functions, macros, and",
+        "source-specification arrays. Unless a variable appears in the user-facing reference",
+        "above, it is an unstable internal diagnostic or tuning interface. The linked source",
+        "remains normative for exact eligibility",
+        "gates, bounds, and architecture-specific defaults.",
+        "",
+        f"Inventory totals: **{len(runtime_names)} `DS4_*` runtime variables** and",
+        f"**{len(external_names)} external runtime variables**.",
+        f"The auxiliary inventories contain **{len(test_binary) + len(test_scripts)} test/test-fixture entries**",
+        f"and **{len(tools)} tool/wrapper entries**.",
+        "",
+    ]
+
+    ordered_groups = [title for title, _prefix in GROUPS] + ["General and shared"]
+    for title in ordered_groups:
+        entries = [item for name, item in runtime_docs.items() if group_for(name) == title]
+        if not entries:
+            continue
+        lines.extend(
+            [
+                "<details>",
+                f"<summary><strong>{title} ({len(entries)})</strong></summary>",
+                "",
+                *metadata_table(entries),
+                "",
+                "</details>",
+                "",
+            ]
+        )
+
+    if external_names:
+        lines.extend(
+            [
+                "### External runtime environment",
+                "",
+                "These names are not owned by the `DS4_*` namespace but are read directly by",
+                "the binaries or vendored runtime code.",
+                "",
+                *metadata_table(list(external_docs.values())),
+                "",
+            ]
+        )
+
+    lines.extend(
+        [
+            "## Test and fixture environment inputs",
+            "",
+            "These entries are consumed by repository test binaries or fixture scripts. Some",
+            "production runtime controls are repeated here because a maintained fixture exposes",
+            "them as part of its own test contract.",
+            "",
+            "### Test binaries and cleanup hooks",
+            "",
+            *metadata_table(test_binary),
+            "",
+            "### Test fixture scripts",
+            "",
+            *metadata_table(test_scripts),
+            "",
+            "## Tool and wrapper environment inputs",
+            "",
+            "These variables configure maintained download, service-wrapper, and offline tooling.",
+            "A tool that accepts a variable name dynamically (for example `--api-key-env`) may read",
+            "the caller-selected name in addition to the literal defaults listed here.",
+            "",
+            *metadata_table(tools),
+            "",
+            END_MARKER,
+        ]
+    )
+    return (
+        "\n".join(lines),
+        len(runtime_names),
+        len(external_names),
+        len(test_binary) + len(test_scripts),
+        len(tools),
+    )
+
+
+def replace_generated_block(document: str, block: str) -> str:
+    start = document.find(START_MARKER)
+    end = document.find(END_MARKER)
+    if start < 0 or end < 0 or end < start:
+        raise ValueError("environment inventory markers are missing from ENVIRONMENT_VARIABLES.md")
+    end += len(END_MARKER)
+    return document[:start] + block + document[end:]
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    mode = parser.add_mutually_exclusive_group()
+    mode.add_argument("--check", action="store_true", help="verify the checked-in reference")
+    mode.add_argument("--emit", action="store_true", help="print the generated inventory block")
+    args = parser.parse_args()
+
+    try:
+        block, runtime_count, external_count, test_count, tool_count = generated_block()
+        if args.emit:
+            print(block)
+            return 0
+        document = DOCUMENT.read_text(encoding="utf-8")
+        expected = replace_generated_block(document, block)
+    except (OSError, ValueError) as error:
+        print(f"environment-variable documentation: {error}", file=sys.stderr)
+        return 1
+
+    summary = (
+        f"{DOCUMENT.relative_to(ROOT)}: {runtime_count} DS4 runtime, "
+        f"{external_count} external runtime, {test_count} test/test-fixture entries, "
+        f"{tool_count} tool/wrapper entries"
+    )
+    if args.check:
+        if document != expected:
+            print(f"{summary}; generated inventory is stale", file=sys.stderr)
+            print("run: python3 scripts/generate_environment_variables.py", file=sys.stderr)
+            return 1
+        print(f"{summary}; generated inventory is current")
+        return 0
+
+    DOCUMENT.write_text(expected, encoding="utf-8")
+    print(f"updated {summary}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
diff --git a/speed-bench/.gitignore b/speed-bench/.gitignore
index 3eac33ffc0..0c92e7210f 100644
--- a/speed-bench/.gitignore
+++ b/speed-bench/.gitignore
@@ -3,3 +3,11 @@ __pycache__/
 local-runs/
 metal_decode_schedule_bench
 metal_prefill_variant_bench
+metal_q4_dense_pair_bench
+metal_q4_prefill_pair_bench
+metal_q4_mm_tail_cull_bench
+metal_iq2_moe_tail_cull_bench
+metal_iq2_moe_top8_pair_bench
+gpu_iq2_moe_prefill_bench
+cuda_q4_prefill_bench
+rocm_q4_prefill_bench
diff --git a/speed-bench/README.md b/speed-bench/README.md
index 6d4188e72c..86ac1b2114 100644
--- a/speed-bench/README.md
+++ b/speed-bench/README.md
@@ -42,7 +42,18 @@ The harness prefills two sessions and alternates both variant order and
 variant-to-session assignment. It aborts unless every full-vocabulary logit
 row is bit-identical and, with `--include-selection`, both variants select the
 same non-EOS token. Use `--candidate-env NAME` to measure a rollback control,
-or `--help` to compare explicit split schedules.
+or `--help` to compare explicit split schedules. Pass `--ssd-streaming` for a
+model larger than RAM; the harness then skips full-weight warmup while keeping
+both variants in the same engine and expert cache. SSD runs can use
+`--ssd-streaming-cold`, `--ssd-streaming-cache-experts N`, and
+`--ssd-streaming-preload-experts N` to hold the cache policy constant.
+
+This paired harness is the exact-logit gate. Since its two sessions share the
+expert cache, confirm SSD throughput separately with one process and engine per
+variant before promoting a scheduling change. Environment variables consumed
+while the engine opens, including the Metal streaming `F_NOCACHE` controls,
+also require separate processes: `--candidate-env` changes them too late to
+create a different model descriptor inside this harness.
 
 To compare the default pre-M5 ratio-4 compressor pack/transpose fusion with the
 legacy decode path, including token selection, use:
@@ -79,4 +90,393 @@ both variants with at least 32 tokens, alternates control/candidate order in
 ABBA and BAAB blocks, poisons host logit buffers before copying, and aborts
 unless every final full-vocabulary logit row is bit-identical. Defaults are an
 8192-token prefix, an automatically sized 8193-token context, and two repeats;
-use `--help` to override them.
+use `--help` to override them. SSD-prefill variants can add `--ssd-streaming`,
+`--ssd-streaming-cold`, `--ssd-streaming-cache-experts N`, and
+`--ssd-streaming-preload-experts N`. Since those paired runs intentionally
+share one engine and expert cache, confirm any SSD throughput win separately
+with one process and engine per variant before promotion.
+Environment variables consumed while the engine opens, including the Metal
+streaming `F_NOCACHE` controls, cannot be compared with `--candidate-env` in
+this harness and likewise require separate processes.
+
+### Metal Q4_K generic-MM tail cull
+
+Build and run the GGUF-free kernel-only comparison with:
+
+```
+make metal-q4-mm-tail-cull-bench
+./speed-bench/metal_q4_mm_tail_cull_bench
+```
+
+The default `4096 -> 1024` shape models a Flash Q-A projection. Use
+`--in-dim 1024 --out-dim 32768` to measure `attn_q_b`. Both arms dispatch the
+checked-in production Metal kernels with resident rotating Q4_K weights and
+GPU timestamps. The harness covers `N=9,16,17,31,33,47,63,65`, alternates
+ABBA/BAAB, and requires bit-exact outputs plus intact input/output canaries.
+No GGUF access, SSD I/O, upload, readback, or CPU wall time is included in a
+measured command buffer.
+
+### Metal Q4_K attention output-A direct routing
+
+Build and run the production-shape, resident GPU comparison with:
+
+```
+make metal-q4-attn-out-a-direct-bench
+./speed-bench/metal_q4_attn_out_a_direct_bench \
+  --n 512,1024,2048,4096 --samples 8 --warmup 2
+```
+
+The fixed `4096 -> 1024 x 8` geometry compares the current map-plus-routed
+path, the routed kernel with a prebuilt map, and the fixed-route direct kernel
+on the production Apple M1–M4 and N=512–4096 scope. All three pairwise
+comparisons are scheduled in balanced ABBA/BAAB blocks and timed only with Metal
+GPU start/end timestamps. The harness requires all three full outputs to be
+bit-identical, hashes immutable weights, heads, ids, and the prebuilt map, and
+checks prefix/suffix canaries around every allocation. Fixture construction,
+map prebuilding, and validation are outside the samples. There is no GGUF,
+SSD I/O, model upload, GPU readback, or CPU wall timing in measured command
+buffers, so these numbers isolate the kernel and routing overhead rather than
+SSD-streaming noise.
+
+### Metal resident IQ2/Q2 routed MoE
+
+Build the production top-6 tail-cull fixture or the GLM-shape top-8 pair-fusion
+fixture with:
+
+```
+make metal-iq2-moe-tail-cull-bench
+./speed-bench/metal_iq2_moe_tail_cull_bench --samples 12 --warmup-cycles 2
+
+make metal-iq2-moe-top8-pair-bench
+./speed-bench/metal_iq2_moe_top8_pair_bench --samples 12 --warmup-cycles 2
+```
+
+Both fixtures keep synthetic IQ2_XXS gate/up and Q2_K down weights resident,
+alternate variants in ABBA/BAAB order, and report per-stage GPU timestamps.
+The top-8 fixture matches the `4096 -> 2048 -> 4096`, 288-expert GLM routed
+MoE geometry. Its baseline launches gate and up separately; its candidate
+uses the grouped pair-SwiGLU kernel. It requires bit-exact F16 mid rows, F32
+expert rows and final output, plus intact allocation canaries. The measured
+stages contain no GGUF loading, file-backed model mapping, application SSD
+reads, uploads, readback, or CPU wall timing. The anonymous fixture is fully
+touched before oracle and warm-up, so a change in these GPU timestamps is
+attributable to the kernel path rather than SSD streaming.
+
+The automatic top-8 dispatch is intentionally limited to the measured
+M1 Max, resident, 4096-token shape. SSD streaming and other batch/device
+shapes keep the separate gate/up path until they have their own A/B data.
+
+### Resident ROCm Q4_K prefill
+
+Build the production-dispatch A/B harness on a ROCm host with:
+
+```
+make rocm-q4-prefill-bench ROCM_ARCH=gfx1151
+./speed-bench/rocm_q4_prefill_bench
+```
+
+The fixture copies four rotating sets of synthetic GGUF-layout Q4_K weights
+to device memory before warmup and forces SSD streaming off. HIP events then
+measure only activation conversion/quantization and projection kernels. The
+row-geometry and activation-loader comparisons use resident pointers plus an
+explicit enqueue-only hook, excluding environment parsing, model lookup, and
+policy selection. The
+comparisons are:
+
+- `dense`: legacy versus TILE8 at the Flash Q-A `K=4096,M=1024` shape;
+- `pair`: two TILE8 calls versus the fused Q-A/KV
+  `K=4096,M=(1024+512)` path;
+- `qb`: TILE8 versus TILE4 at the production `attn_q_b`
+  `K=1024,M=32768` shape.
+- `outb`: TILE8 versus the compressed direct-Q4 WMMA kernel at the
+  production `output_b` `K=8192,M=4096` shape;
+- `output`: the complete grouped `output_a` plus `output_b` production API,
+  comparing an all-TILE8 rollback with the production A-WMMA/B-TILE8
+  pipeline.
+
+On gfx1151 wave32, `dense` and `qb` also emit a direct-Q4 WMMA comparison for
+`N>=256`. The candidate keeps Q4_K weights compressed, rounds each transient
+32-value weight group and the activation tile to F16 in the kernel, accumulates
+through WMMA in F32, and avoids both Q8_K activation scratch and persistent F16
+weight sidecars. Its shape-selected row tile uses 64 rows below output dimension
+1024, 128 below 8192, and 256 otherwise; the wider variants stage two adjacent
+F32 activations into one F16 pair, matching the established Q8 WMMA loader.
+`dense`, `outb`, and `output` also emit a bit-exact 64-row versus shape-selected
+scalar-loader A/B. The large `q_b` shape instead reports adjacent scalar-loader
+64→128 and 128→256 comparisons, so
+register pressure in the 512-thread candidate cannot hide a better midpoint.
+These direct comparisons measure the net effect of the broader row geometry,
+including its changed workgroup and occupancy contract, separately from the
+candidate's arithmetic change versus TILE8. `q_b` additionally compares scalar
+versus two-wide activation staging at fixed 128- and 256-row geometry.
+The direct hook receives both tile and loader explicitly, so every arm attests
+its own configuration. Eligible standalone resident calls and attention-output
+A use direct-Q4 WMMA by default; set `DS4_ROCM_DISABLE_Q4_PREFILL_WMMA=1` to
+opt out. Attention-output B remains on Q8_K+TILE8. The production comparison
+sets `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA=1` without REQUIRE, which explicitly
+retains that same A-WMMA/B-TILE8 policy. The hard B oracle replays TILE8 over
+the same WMMA-low intermediate and requires bitwise-identical output. The composed
+A-WMMA/B-TILE8 versus all-TILE8 delta remains visible as a non-gating
+diagnostic because it includes A's deliberate F16 boundary rather than
+isolating B correctness. Raw two-stage direct-WMMA row-geometry and K32/K64
+comparisons remain diagnostic-only kernel measurements.
+
+Eligible aligned 256-row direct-Q4 WMMA launches use the q_b-focused K128/P144
+stage by default. It groups four adjacent Q4_K qgroups behind one barrier pair
+and uses float4 activation loads, targeting a 2x reduction in synchronization
+and activation-load instructions over K64 while retaining the same F16
+conversions and accumulation order. Set
+`DS4_ROCM_DISABLE_Q4_PREFILL_WMMA_K128=1` to roll the same launch back to
+K64/P80; unset or explicitly false keeps K128. Incompatible alignment or
+64/128-row geometry automatically uses K64.
+
+K64/P80 stages two adjacent 32-value Q4_K groups and a 64-value activation
+slice in one padded P80 LDS tile. Leave
+`DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_K64` unset or true to permit both K64 and the
+eligible K128 default; set it to `0`, `false`, `no`, or `off` to restore K32
+staging. `DS4_ROCM_DISABLE_Q4_PREFILL_WMMA=1` rolls the whole direct-WMMA path
+back to Q8_K plus TILE8/TILE4. A persistent or automatic transient F16 q_b
+projection bypasses the direct-Q4 candidates.
+
+The q_b microbenchmark contains a strict, same-process
+`q_b_wmma_k64_k128` A/B with bitwise output and canary checks:
+
+```
+./speed-bench/rocm_q4_prefill_bench \
+  --case qb --tokens 256,512,1024,2048,2049,4096 \
+  --sets 4 --warmup 4 --samples 12
+```
+
+Use the production 2048-token sweep to exercise default K128/P144 without
+SSD-streaming or policy-selection noise. The same process also reports the
+strict K64/P80 versus K128/P144 A/B above:
+
+```
+./speed-bench/rocm_q4_prefill_bench \
+  --case all --tokens 2048 --sets 4 --warmup 4 --samples 12
+```
+
+The benchmark always keeps SSD streaming disabled. Runtime SSD experiments
+need the separate `DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_SSD=1` gate and only accept
+projection ranges already backed by physical device memory. That gate opts
+attention-output A into WMMA but retains TILE8 for B; only the strict,
+diagnostic `DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA=1` selects direct WMMA for B.
+Model I/O is therefore never folded into the kernel result.
+
+The default token set is `9,17,33,128,256,257,512`, covering the first row
+after the small TILE8 boundaries plus both the exact 256-token occupancy case
+and the first token after a 64-token WMMA boundary. Use `--full` for
+`9,16,17,31,32,33,128,256,257,512,4096`, or select a focused run such as:
+
+```
+./speed-bench/rocm_q4_prefill_bench \
+  --case qb --tokens 256,257,512,1024,2048,2049,4096 \
+  --sets 4 --warmup 4 --samples 12
+```
+
+Every case rotates identical resident weight sets between arms, alternates
+ABBA/BAAB, and verifies allocation guards both before and after timing.
+Comparisons among the integer Q4 paths remain bit-exact. Direct-Q4 WMMA has a
+deliberate F16 arithmetic boundary, so single-stage comparisons require finite
+results within an explicit absolute/relative smoke tolerance. For the chained
+production output, the B-stage same-input replay is bit-exact while the
+end-to-end delta is informational; release validation must also include the
+model-logit or prompt oracle that guards the automatic standalone and
+attention-output-A policy.
+Fixture creation, the host-to-device residency copy, warmup, oracle readback,
+and environment-gate changes are outside the reported HIP-event intervals.
+`candidate_delta_pct` is negative when the candidate is faster; the companion
+`speedup_pct` reports the positive speedup convention.
+
+### Resident CUDA Q4_K prefill
+
+Build the production-API kernel harness on a CUDA host with an explicit
+architecture:
+
+```
+make cuda-q4-prefill-bench CUDA_ARCH=sm_121
+./speed-bench/cuda_q4_prefill_bench --path mmq
+```
+
+The default `dense`, `pair`, `qb`, and GB10-only `outa` cases use production
+shapes and include both sides of the 128-column MMQ tail. The synthetic
+GGUF-layout weights are copied by
+the backend into a `cudaMalloc` allocation before warmup. A CUDA test hook
+checks the backend-owned pointer provenance and device attributes of every
+dense, KV, q_b, output-A, and minimal output-B range in every rotating weight
+set; a global free-memory
+delta is printed only as a diagnostic and is not accepted as residency proof.
+CUDA events measure the production backend GPU interval, including
+stream-ordered scratch allocation/free, tail clears, activation quantization,
+Q4 projection, and output sanitization. Model uploads, output poisoning, full
+finite-output scans, sampled CPU Q4_K oracles, canary checks, and warmups are
+outside the event interval.
+
+`cuda_use_mmq()` caches its first decision for the life of the process, so the
+dense and `attn_q_b` MMQ-versus-Q8_K comparison deliberately uses separate
+processes. The MMQ process also enables a test-only strict control, so an MMQ
+rejection fails the run instead of silently measuring the Q8_K fallback. Run
+both orders to balance thermal/order drift:
+
+```
+# ABBA
+./speed-bench/cuda_q4_prefill_bench --path legacy --case dense
+./speed-bench/cuda_q4_prefill_bench --path mmq    --case dense
+./speed-bench/cuda_q4_prefill_bench --path mmq    --case dense
+./speed-bench/cuda_q4_prefill_bench --path legacy --case dense
+
+# BAAB
+./speed-bench/cuda_q4_prefill_bench --path mmq    --case dense
+./speed-bench/cuda_q4_prefill_bench --path legacy --case dense
+./speed-bench/cuda_q4_prefill_bench --path legacy --case dense
+./speed-bench/cuda_q4_prefill_bench --path mmq    --case dense
+```
+
+Repeat with `--case qb` for `K=1024,M=32768`. Each result records the immutable
+path as `path=legacy` or `path=mmq`; do not toggle `DS4_CUDA_MMQ` around calls
+inside another harness. The default MMQ `pair` case is a true in-process
+ABBA/BAAB comparison between two public dense calls and the public fused pair
+API, with bit-exact pair outputs. Prefill pair is skipped under `--path legacy`
+because the CUDA pair API intentionally returns control to two independent
+dense projections for `N>8` when MMQ is disabled.
+
+To isolate the experimental Q4_K m128n128 16-warp GEMM from activation
+quantization and every storage effect, run the focused bitwise/canary oracle
+and then the resident prequantized A/B benchmark:
+
+```
+make test-mmq-q4-16warp-cuda CUDA_ARCH=sm_121
+make cuda-q4-prefill-bench CUDA_ARCH=sm_121
+./speed-bench/cuda_q4_prefill_bench \
+  --path mmq --kernel-16warp --case dense \
+  --tokens 512,1024,2048,2049,4096,6144,8192 \
+  --sets 4 --samples 16 --warmup 4
+./speed-bench/cuda_q4_prefill_bench \
+  --path mmq --kernel-16warp --case pair \
+  --tokens 512,1024,2048,2049,4096,6144,8192 \
+  --sets 4 --samples 16 --warmup 4
+DS4_CUDA_MMQ_X_MAX=128 \
+./speed-bench/cuda_q4_prefill_bench \
+  --path mmq --kernel-16warp --case outb \
+  --tokens 2048 \
+  --sets 4 --samples 16 --warmup 4
+```
+
+The benchmark quantizes X to canonical Q8_1 DS4 once, before timing, and uses
+CUDA events around only the production-policy Stream-K reference or the
+16-warp kernel with the same partition/fixup policy. Both arms therefore
+include fixup whenever the canonical dispatcher would use it, including the
+GB10 `M=1024,N=4096` case. One guarded fixup allocation is created before the
+samples and reused by both A/B arms and by the two pair legs, so CUDA pool
+allocation/free nodes cannot bias the kernel delta.
+The dense case is the production Q-A shape `K=4096,M=1024`; the pair case
+reuses that same prequantized activation across the asymmetric Q-A/KV shapes
+`M0=1024,M1=512` and times two GEMMs in each arm. It alternates the
+arms ABBA/BAAB over resident Q4_K weight sets, compares every complete output
+bit-for-bit, checks finite values and independent output canaries, and samples
+a CPU Q4_K oracle before and after timing. Results attest
+`timing=kernel_only_prequant`; SSD streaming, model upload, Q8_1 quantization,
+allocation, host copies, and oracle work are outside the samples. Use
+`--case qb` for the additional `K=1024,M=32768` large-projection datapoint and
+`--case outb` for the real attention output-B `K=8192,M=4096` geometry. The
+focused oracle covers that wider K with complete N128 tiles; the resident
+benchmark retains the real `N=2048` prefill geometry for performance claims.
+The focused test also invokes the real standalone dense and pair dispatchers in
+required mode at `N=4096`, so fallback fails instead of producing a misleading
+canonical-path pass. A negative pair case verifies that an ineligible 384-row
+leg returns `DS4_MMQ_NOT_APPLICABLE` without touching either guarded output.
+The CLI accepts contexts through 8192 tokens; the default kernel-only sweep
+adds 6144 and 8192 to expose long-context scaling. A custom token tail must
+also make the canonical picker select `m128n128`; the harness checks the real
+device selector up front and reports `SKIP` instead of running a mismatched
+Stream-K partition.
+
+The production path remains opt-in until the NVIDIA oracle passes and the
+paired median is a repeatable win. `DS4_CUDA_Q4_MMQ_16WARP=1` requests it and
+falls back on ineligible shapes. For an attested production benchmark,
+`DS4_CUDA_REQUIRE_Q4_MMQ_16WARP=1` also prevents a Q8_K/MMQ fallback;
+`DS4_CUDA_NO_Q4_MMQ_16WARP=1` is the value-aware rollback. The strict path
+requires Ampere or newer, complete 128x128 output tiles, `N>=512`,
+`1024<=K<=8192`, the default 128-column MMQ selector, and at least 80% final-wave
+grid efficiency. Single dense admission starts at `M=1024`, including the
+`K=8192` output-B projection; the Q-A/KV pair admission also accepts its
+`M1=512` leg when both projections select the 16-warp path, and remains bounded
+to `K<=4096`.
+
+On GB10, isolate the production Flash attention-output A geometry
+(`groups=8`, `K=4096`, `rank=1024`) and its 127/128/129 token tails with:
+
+```
+./speed-bench/cuda_q4_prefill_bench \
+  --path mmq --case outa --tokens 127,128,129,257,512,2048 \
+  --samples 16 --warmup 4
+./speed-bench/cuda_q4_prefill_bench \
+  --path mmq --case outa --grouped-q81-kernel \
+  --tokens 512,1024,2048,4096,6144,8192 \
+  --sets 4 --samples 16 --warmup 4
+```
+
+This is an in-process ABBA/BAAB comparison between the current eight-group
+pack/MMQ/unpack rollback and the default, strictly required direct-strided
+grouped-prefill dispatch with one canonical MMQ grid per group. Add
+`--grouped-single-grid` to instead compare the grouped eight-grid path with the
+experimental grid.z submission; do not mix that experiment into the default
+promotion measurement.
+`--grouped-q81-kernel` holds those same eight MMQ grids constant and compares
+the canonical strided Q8_1 producer with the default `K=4096`, `groups=8`
+eight-warp kernel. The candidate is required and the reference is forced with
+its narrow rollback, so neither arm can silently time the other quantizer. The
+complete output remains bitwise checked; run
+`make test-mmq-q4-grouped-q81-cuda CUDA_ARCH=sm_121` first for direct
+byte-level Q8_1 parity and canary coverage.
+The public API must also execute output-B, so the fixture uses a valid Q4_K
+`K=8192,M=256` output-B common to both arms. It is 6.25% of output-A's MACs;
+the result prints `focus_macs_per_token` and `common_macs_per_token` separately
+and checks the two arms bit-for-bit. SSD streaming is forced off and every
+measured weight range must resolve to backend-owned device storage.
+
+### Resident IQ2/Q2 MoE prefill on ROCm and CUDA
+
+The backend-neutral fixture uses the production `N=4096`, 256-expert, top-6
+IQ2_XXS/Q2_K geometry and a deterministic routing distribution containing
+every 32-row tail from 1 through 31. Weights and tensors are resident and SSD
+streaming is disabled, so the reported GPU-event intervals isolate kernel
+work rather than storage throughput. The fixture needs roughly 4 GiB of
+explicit host/device storage in addition to backend runtime overhead.
+
+On a wave32 ROCm host, build and run the real balanced A/B with:
+
+```
+make rocm-iq2-moe-prefill-bench ROCM_ARCH=gfx1151
+./speed-bench/gpu_iq2_moe_prefill_bench_rocm
+```
+
+The baseline sets the dominant rollback
+`DS4_ROCM_DISABLE_IQ2_MOE_WMMA_TAIL_CULL=1`; the candidate sets
+`DS4_ROCM_ENABLE_IQ2_MOE_WMMA_TAIL_CULL=1`. The harness alternates ABBA/BAAB,
+requires bit-exact intermediate scratch and final tensors, verifies allocation
+canaries, and prints `cudaEvent` time for only the IQ2 gate/up and Q2 down
+rocWMMA kernels. The candidate remains opt-in until real-hardware results show
+a repeatable win. A wave64 device takes the scalar fallback and therefore
+cannot produce a valid candidate timing.
+
+On CUDA, build and run the measurement-only current path with an explicit
+architecture:
+
+```
+make cuda-iq2-moe-prefill-bench CUDA_ARCH=sm_121
+./speed-bench/gpu_iq2_moe_prefill_bench_cuda
+```
+
+CUDA prints `DS4_CUDA_MOE_PROFILE` stage times for the resident IQ2 MMQ path
+and performs a structural/canary oracle. Every marked call must produce exactly
+one completed fast-path profile record or the harness fails. It intentionally
+does not claim an A/B result: CUDA's cooperative D2R CTA has different
+synchronization and tail semantics, so the ROCm/Metal wave-cull selector cannot
+be copied safely.
+On a discrete CUDA GPU with enough VRAM, prefix the run with
+`DS4_CUDA_COPY_MODEL=1` to keep the raw expert weights in device memory and
+remove mapped-host/PCIe stalls from the measured kernel interval; this adds
+about 1.7 GiB of device storage. Leave it unset on GB10/UMA when measuring the
+normal aligned-artifact production selector, because forcing a raw model copy
+changes that residency path.
diff --git a/speed-bench/cuda_q4_prefill_bench.cu b/speed-bench/cuda_q4_prefill_bench.cu
new file mode 100644
index 0000000000..a8c6e3dde6
--- /dev/null
+++ b/speed-bench/cuda_q4_prefill_bench.cu
@@ -0,0 +1,2332 @@
+// SPDX-License-Identifier: MIT
+// Resident, CUDA-event-only Q4_K prefill microbenchmark.
+
+#include "ds4_gpu.h"
+#include "cuda/mmq/ds4_mmq.h"
+#include "cuda/mmq/ds4_mmq_q4_16warp.cuh"
+
+#include <cuda_runtime.h>
+
+#include <algorithm>
+#include <cerrno>
+#include <cmath>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <functional>
+#include <limits>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace {
+
+constexpr uint32_t kQ4Type = 12u;
+constexpr uint32_t kQkK = 256u;
+constexpr uint32_t kDenseK = 4096u;
+constexpr uint32_t kDenseM = 1024u;
+constexpr uint32_t kKvM = 512u;
+constexpr uint32_t kQbK = 1024u;
+constexpr uint32_t kQbM = 32768u;
+constexpr uint32_t kOutputGroups = 8u;
+constexpr uint32_t kOutputRank = 1024u;
+constexpr uint32_t kOutputLowDim = kOutputGroups * kOutputRank;
+constexpr uint32_t kOutputMinB = 256u;
+constexpr uint32_t kOutputM = 4096u;
+constexpr uint32_t kDefaultSets = 4u;
+constexpr uint32_t kDefaultSamples = 8u;
+constexpr uint32_t kDefaultWarmup = 2u;
+constexpr uint32_t kGuardWords = 64u;
+constexpr uint64_t kCompareChunk = 4u * 1024u * 1024u;
+
+constexpr const char *kGroupedPrefillEnable =
+    "DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_PREFILL";
+constexpr const char *kGroupedPrefillDisable =
+    "DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL";
+constexpr const char *kGroupedPrefillRequire =
+    "DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_PREFILL";
+constexpr const char *kGroupedSingleGridEnable =
+    "DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_SINGLE_GRID";
+constexpr const char *kGroupedSingleGridDisable =
+    "DS4_CUDA_DISABLE_Q4_GROUPED_ATTN_A_SINGLE_GRID";
+constexpr const char *kGroupedSingleGridRequire =
+    "DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_SINGLE_GRID";
+constexpr const char *kGroupedQ81Disable =
+    "DS4_CUDA_NO_Q4_GROUPED_ATTN_A_Q81";
+constexpr const char *kGroupedQ81Require =
+    "DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_Q81";
+constexpr const char *kGroupedGlobalDisable =
+    "DS4_CUDA_NO_Q4_GROUPED_ATTN_A";
+constexpr const char *kGb10GlobalDisable = "DS4_CUDA_NO_Q4_GB10_FAST";
+
+struct block_q4_K_host {
+    uint16_t d;
+    uint16_t dmin;
+    uint8_t scales[12];
+    uint8_t qs[kQkK / 2u];
+};
+
+static_assert(sizeof(block_q4_K_host) == 144u,
+              "Q4_K fixture must match the raw GGUF layout");
+
+enum class bench_case {
+    all,
+    dense,
+    pair,
+    qb,
+    outa,
+    outb,
+};
+
+enum class cuda_path {
+    mmq,
+    legacy,
+};
+
+struct config {
+    bench_case selected = bench_case::all;
+    cuda_path path = cuda_path::mmq;
+    std::vector<uint32_t> tokens = {9u, 17u, 33u, 127u, 128u, 129u, 257u,
+                                    512u};
+    uint32_t sets = kDefaultSets;
+    uint32_t samples = kDefaultSamples;
+    uint32_t warmup = kDefaultWarmup;
+    bool kernel_16warp = false;
+    bool grouped_single_grid = false;
+    bool grouped_q81_kernel = false;
+};
+
+struct weight_set {
+    uint64_t dense_offset = 0;
+    uint64_t kv_offset = 0;
+    uint64_t qb_offset = 0;
+    uint64_t output_a_offset = 0;
+    uint64_t output_b_offset = 0;
+};
+
+struct model_fixture {
+    uint8_t *data = nullptr;
+    uint64_t size = 0;
+    uint64_t payload_bytes = 0;
+    uint32_t output_b_rows = kOutputMinB;
+    std::vector<weight_set> weights;
+
+    ~model_fixture() { std::free(data); }
+    model_fixture() = default;
+    model_fixture(const model_fixture &) = delete;
+    model_fixture &operator=(const model_fixture &) = delete;
+};
+
+struct tensor_owner {
+    ds4_gpu_tensor *ptr = nullptr;
+
+    explicit tensor_owner(uint64_t bytes) : ptr(ds4_gpu_tensor_alloc(bytes)) {}
+    ~tensor_owner() { ds4_gpu_tensor_free(ptr); }
+    tensor_owner(const tensor_owner &) = delete;
+    tensor_owner &operator=(const tensor_owner &) = delete;
+};
+
+struct cuda_buffer {
+    void *ptr = nullptr;
+    cudaError_t status = cudaSuccess;
+
+    explicit cuda_buffer(size_t bytes) {
+        if (bytes == 0u) return;
+        status = cudaMalloc(&ptr, bytes);
+    }
+    ~cuda_buffer() {
+        if (ptr) (void)cudaFree(ptr);
+    }
+    cuda_buffer(const cuda_buffer &) = delete;
+    cuda_buffer &operator=(const cuda_buffer &) = delete;
+};
+
+struct guarded_cuda_buffer {
+    void *storage = nullptr;
+    void *ptr = nullptr;
+    size_t logical_bytes = 0;
+    uint32_t salt = 0;
+    cudaError_t status = cudaSuccess;
+
+    guarded_cuda_buffer(size_t bytes, uint32_t guard_salt)
+        : logical_bytes(bytes), salt(guard_salt) {
+        if (bytes == 0u) return;
+        constexpr size_t guard_bytes = kGuardWords * sizeof(uint32_t);
+        if (bytes > std::numeric_limits<size_t>::max() - 2u * guard_bytes) {
+            status = static_cast<cudaError_t>(1);
+            return;
+        }
+        status = cudaMalloc(&storage, bytes + 2u * guard_bytes);
+        if (status != cudaSuccess) return;
+        ptr = static_cast<uint8_t *>(storage) + guard_bytes;
+        std::vector<uint32_t> prefix(kGuardWords);
+        std::vector<uint32_t> suffix(kGuardWords);
+        for (uint32_t i = 0; i < kGuardWords; ++i) {
+            prefix[i] = 0x6b8b4567u ^ salt ^ (i * 0x00010101u);
+            suffix[i] = 0x327b23c6u ^ salt ^ (i * 0x01000101u);
+        }
+        status = cudaMemcpy(storage, prefix.data(), guard_bytes,
+                            cudaMemcpyHostToDevice);
+        if (status == cudaSuccess) {
+            status = cudaMemcpy(
+                static_cast<uint8_t *>(ptr) + bytes, suffix.data(),
+                guard_bytes, cudaMemcpyHostToDevice);
+        }
+    }
+
+    ~guarded_cuda_buffer() {
+        if (storage) (void)cudaFree(storage);
+    }
+
+    bool intact(const char *label) const {
+        if (logical_bytes == 0u) return true;
+        constexpr size_t guard_bytes = kGuardWords * sizeof(uint32_t);
+        std::vector<uint32_t> prefix(kGuardWords);
+        std::vector<uint32_t> suffix(kGuardWords);
+        if (cudaMemcpy(prefix.data(), storage, guard_bytes,
+                       cudaMemcpyDeviceToHost) != cudaSuccess ||
+            cudaMemcpy(suffix.data(),
+                       static_cast<const uint8_t *>(ptr) + logical_bytes,
+                       guard_bytes, cudaMemcpyDeviceToHost) != cudaSuccess) {
+            std::fprintf(stderr, "%s: scratch guard read failed\n", label);
+            return false;
+        }
+        for (uint32_t i = 0; i < kGuardWords; ++i) {
+            const uint32_t expected_prefix =
+                0x6b8b4567u ^ salt ^ (i * 0x00010101u);
+            const uint32_t expected_suffix =
+                0x327b23c6u ^ salt ^ (i * 0x01000101u);
+            if (prefix[i] != expected_prefix || suffix[i] != expected_suffix) {
+                std::fprintf(stderr,
+                             "%s: scratch guard overwritten at word %u\n",
+                             label, i);
+                return false;
+            }
+        }
+        return true;
+    }
+
+    guarded_cuda_buffer(const guarded_cuda_buffer &) = delete;
+    guarded_cuda_buffer &operator=(const guarded_cuda_buffer &) = delete;
+};
+
+struct env_snapshot {
+    const char *name;
+    bool existed;
+    std::string value;
+
+    explicit env_snapshot(const char *key)
+        : name(key), existed(std::getenv(key) != nullptr),
+          value(existed ? std::getenv(key) : "") {}
+    ~env_snapshot() {
+        if (existed) {
+            (void)setenv(name, value.c_str(), 1);
+        } else {
+            (void)unsetenv(name);
+        }
+    }
+    env_snapshot(const env_snapshot &) = delete;
+    env_snapshot &operator=(const env_snapshot &) = delete;
+};
+
+struct event_timer {
+    cudaEvent_t begin = nullptr;
+    cudaEvent_t end = nullptr;
+
+    event_timer() {
+        if (cudaEventCreate(&begin) != cudaSuccess ||
+            cudaEventCreate(&end) != cudaSuccess) {
+            std::fprintf(stderr,
+                         "cuda-q4-prefill-bench: CUDA event allocation failed\n");
+            std::exit(1);
+        }
+    }
+    ~event_timer() {
+        if (begin) (void)cudaEventDestroy(begin);
+        if (end) (void)cudaEventDestroy(end);
+    }
+
+    bool measure(const std::function<bool()> &dispatch, float *milliseconds) {
+        // Every harness arm uses stream 0.  Production paths enqueue their
+        // complete backend work there; kernel-only arms enqueue exactly one
+        // already-prepared GEMM there.
+        if (cudaEventRecord(begin, nullptr) != cudaSuccess) return false;
+        if (!dispatch()) return false;
+        if (cudaEventRecord(end, nullptr) != cudaSuccess ||
+            cudaEventSynchronize(end) != cudaSuccess ||
+            cudaEventElapsedTime(milliseconds, begin, end) != cudaSuccess) {
+            return false;
+        }
+        return true;
+    }
+};
+
+struct arm {
+    const char *name;
+    std::function<bool(uint32_t)> dispatch;
+    // Host-only path selection must happen before the start event.  Otherwise
+    // an idle device can execute that event while setenv()/unsetenv() is still
+    // running, folding host-side gate switching into the reported GPU time.
+    std::function<bool()> select;
+    // Optional checked boundary for oracle passes. Timed samples continue to
+    // use dispatch so an enqueue-only experimental arm can keep its safety
+    // preflight outside the CUDA-event interval.
+    std::function<bool(uint32_t)> oracle_dispatch = {};
+};
+
+struct stats {
+    double minimum = 0.0;
+    double median = 0.0;
+    double p95 = 0.0;
+    double mean = 0.0;
+};
+
+uint64_t align_up(uint64_t value, uint64_t alignment) {
+    return (value + alignment - 1u) / alignment * alignment;
+}
+
+bool checked_mul(uint64_t a, uint64_t b, uint64_t *out) {
+    if (a != 0u && b > std::numeric_limits<uint64_t>::max() / a) return false;
+    *out = a * b;
+    return true;
+}
+
+bool current_device_nsm(int *nsm) {
+    if (!nsm) return false;
+    int device = -1;
+    cudaDeviceProp prop = {};
+    const cudaError_t device_err = cudaGetDevice(&device);
+    const cudaError_t prop_err = device_err == cudaSuccess
+        ? cudaGetDeviceProperties(&prop, device) : device_err;
+    if (prop_err != cudaSuccess || prop.multiProcessorCount <= 0) {
+        std::fprintf(stderr,
+                     "cuda-q4-prefill-bench: CUDA geometry query failed: %s\n",
+                     cudaGetErrorString(prop_err));
+        return false;
+    }
+    *nsm = prop.multiProcessorCount;
+    return true;
+}
+
+uint32_t lcg_next(uint32_t *state) {
+    *state = *state * 1664525u + 1013904223u;
+    return *state;
+}
+
+void fill_q4(void *storage, uint64_t bytes, uint32_t seed) {
+    auto *blocks = static_cast<block_q4_K_host *>(storage);
+    const uint64_t count = bytes / sizeof(*blocks);
+    uint32_t state = seed;
+    for (uint64_t i = 0; i < count; i++) {
+        // Positive, finite FP16 scales.  Payload values are deterministic.
+        blocks[i].d = static_cast<uint16_t>(0x2400u +
+                                            (lcg_next(&state) & 0xffu));
+        blocks[i].dmin = static_cast<uint16_t>(0x2000u +
+                                               (lcg_next(&state) & 0xffu));
+        for (uint8_t &v : blocks[i].scales) {
+            v = static_cast<uint8_t>(lcg_next(&state) >> 24u);
+        }
+        for (uint8_t &v : blocks[i].qs) {
+            v = static_cast<uint8_t>(lcg_next(&state) >> 24u);
+        }
+    }
+}
+
+uint64_t q4_weight_bytes(uint32_t in_dim, uint32_t out_dim) {
+    return static_cast<uint64_t>(out_dim) * (in_dim / kQkK) *
+           sizeof(block_q4_K_host);
+}
+
+bool make_model(model_fixture *model, uint32_t sets,
+                uint32_t output_b_rows) {
+    if (output_b_rows < kOutputMinB) return false;
+    constexpr uint64_t page = 4096u;
+    const uint64_t dense_bytes = q4_weight_bytes(kDenseK, kDenseM);
+    const uint64_t kv_bytes = q4_weight_bytes(kDenseK, kKvM);
+    const uint64_t qb_bytes = q4_weight_bytes(kQbK, kQbM);
+    const uint64_t output_a_bytes =
+        q4_weight_bytes(kDenseK, kOutputLowDim);
+    const uint64_t output_b_bytes =
+        q4_weight_bytes(kOutputLowDim, output_b_rows);
+
+    model->output_b_rows = output_b_rows;
+    model->weights.resize(sets);
+    uint64_t cursor = 0;
+    auto append = [&](uint64_t bytes) {
+        const uint64_t offset = align_up(cursor, page);
+        cursor = offset + bytes;
+        model->payload_bytes += bytes;
+        return offset;
+    };
+    for (uint32_t i = 0; i < sets; i++) {
+        model->weights[i].dense_offset = append(dense_bytes);
+        model->weights[i].kv_offset = append(kv_bytes);
+        model->weights[i].qb_offset = append(qb_bytes);
+        model->weights[i].output_a_offset = append(output_a_bytes);
+        model->weights[i].output_b_offset = append(output_b_bytes);
+    }
+    model->size = align_up(cursor, page);
+    void *storage = nullptr;
+    if (posix_memalign(&storage, static_cast<size_t>(page),
+                       static_cast<size_t>(model->size)) != 0) {
+        return false;
+    }
+    model->data = static_cast<uint8_t *>(storage);
+    std::memset(model->data, 0, static_cast<size_t>(model->size));
+    for (uint32_t i = 0; i < sets; i++) {
+        fill_q4(model->data + model->weights[i].dense_offset, dense_bytes,
+                0x243f6a88u ^ (i * 0x9e3779b9u));
+        fill_q4(model->data + model->weights[i].kv_offset, kv_bytes,
+                0x85a308d3u ^ (i * 0x7f4a7c15u));
+        fill_q4(model->data + model->weights[i].qb_offset, qb_bytes,
+                0x13198a2eu ^ (i * 0x94d049bbu));
+        fill_q4(model->data + model->weights[i].output_a_offset,
+                output_a_bytes, 0xa4093822u ^ (i * 0x2545f491u));
+        fill_q4(model->data + model->weights[i].output_b_offset,
+                output_b_bytes, 0x299f31d0u ^ (i * 0x369dea0fu));
+    }
+    return true;
+}
+
+void fill_activation(std::vector<float> *values, uint32_t n_tokens,
+                     uint32_t in_dim) {
+    values->resize(static_cast<uint64_t>(n_tokens) * in_dim);
+    for (uint32_t token = 0; token < n_tokens; token++) {
+        for (uint32_t block = 0; block < in_dim / kQkK; block++) {
+            float *dst = values->data() +
+                         static_cast<uint64_t>(token) * in_dim + block * kQkK;
+            for (uint32_t i = 0; i < kQkK; i++) {
+                const int q = static_cast<int>((i * 73u + token * 37u +
+                                                block * 19u) % 241u) - 120;
+                dst[i] = static_cast<float>(q) / 32.0f;
+            }
+            dst[0] = ((token + block) & 1u) ? 127.0f / 32.0f
+                                             : -127.0f / 32.0f;
+        }
+    }
+}
+
+std::vector<uint32_t> guard_pattern(uint32_t salt) {
+    std::vector<uint32_t> guard(kGuardWords);
+    for (uint32_t i = 0; i < kGuardWords; i++) {
+        guard[i] = 0x7fc12000u ^ salt ^ (i * 0x00010101u);
+    }
+    return guard;
+}
+
+bool prepare_guard(ds4_gpu_tensor *tensor, uint64_t logical_bytes,
+                   uint32_t salt) {
+    const std::vector<uint32_t> guard = guard_pattern(salt);
+    return ds4_gpu_tensor_write(tensor, logical_bytes, guard.data(),
+                                guard.size() * sizeof(guard[0])) != 0;
+}
+
+bool poison_output(ds4_gpu_tensor *tensor, uint64_t logical_bytes,
+                   uint32_t pattern, uint32_t guard_salt) {
+    if (!tensor || logical_bytes == 0u ||
+        logical_bytes % sizeof(uint32_t) != 0u) {
+        return false;
+    }
+    const uint64_t chunk_bytes = std::min(kCompareChunk, logical_bytes);
+    std::vector<uint32_t> poison(
+        static_cast<size_t>(chunk_bytes / sizeof(uint32_t)), pattern);
+    for (uint64_t offset = 0; offset < logical_bytes; offset += chunk_bytes) {
+        const uint64_t count = std::min(chunk_bytes, logical_bytes - offset);
+        if (!ds4_gpu_tensor_write(tensor, offset, poison.data(), count)) {
+            return false;
+        }
+    }
+    return prepare_guard(tensor, logical_bytes, guard_salt);
+}
+
+bool check_guard(const ds4_gpu_tensor *tensor, uint64_t logical_bytes,
+                 uint32_t salt, const char *label) {
+    const std::vector<uint32_t> expected = guard_pattern(salt);
+    std::vector<uint32_t> got(expected.size());
+    if (!ds4_gpu_tensor_read(tensor, logical_bytes, got.data(),
+                             got.size() * sizeof(got[0]))) {
+        std::fprintf(stderr, "%s: guard read failed\n", label);
+        return false;
+    }
+    if (got != expected) {
+        const auto mismatch = std::mismatch(got.begin(), got.end(),
+                                            expected.begin());
+        std::fprintf(stderr, "%s: guard overwritten at word %zu\n", label,
+                     static_cast<size_t>(mismatch.first - got.begin()));
+        return false;
+    }
+    return true;
+}
+
+bool output_is_finite(const ds4_gpu_tensor *tensor, uint64_t logical_bytes,
+                      const char *label) {
+    const uint64_t chunk_bytes = std::min(kCompareChunk, logical_bytes);
+    std::vector<float> values(
+        static_cast<size_t>(chunk_bytes / sizeof(float)));
+    for (uint64_t offset = 0; offset < logical_bytes; offset += chunk_bytes) {
+        const uint64_t count = std::min(chunk_bytes, logical_bytes - offset);
+        if (!ds4_gpu_tensor_read(tensor, offset, values.data(), count)) {
+            std::fprintf(stderr, "%s: output read failed\n", label);
+            return false;
+        }
+        const size_t n = static_cast<size_t>(count / sizeof(float));
+        for (size_t i = 0; i < n; i++) {
+            if (!std::isfinite(values[i])) {
+                std::fprintf(stderr,
+                             "%s: non-finite/unwritten output at element %llu\n",
+                             label,
+                             static_cast<unsigned long long>(
+                                 offset / sizeof(float) + i));
+                return false;
+            }
+        }
+    }
+    return true;
+}
+
+bool bitwise_equal(const ds4_gpu_tensor *a, const ds4_gpu_tensor *b,
+                   uint64_t bytes, const char *label) {
+    const uint64_t chunk = std::min(kCompareChunk, bytes);
+    std::vector<uint8_t> lhs(static_cast<size_t>(chunk));
+    std::vector<uint8_t> rhs(static_cast<size_t>(chunk));
+    for (uint64_t offset = 0; offset < bytes; offset += chunk) {
+        const uint64_t count = std::min(chunk, bytes - offset);
+        if (!ds4_gpu_tensor_read(a, offset, lhs.data(), count) ||
+            !ds4_gpu_tensor_read(b, offset, rhs.data(), count)) {
+            std::fprintf(stderr, "%s: oracle read failed\n", label);
+            return false;
+        }
+        if (std::memcmp(lhs.data(), rhs.data(), static_cast<size_t>(count)) !=
+            0) {
+            uint64_t first = 0;
+            while (first < count && lhs[static_cast<size_t>(first)] ==
+                                        rhs[static_cast<size_t>(first)]) {
+                first++;
+            }
+            std::fprintf(stderr, "%s: bitwise mismatch at output byte %llu\n",
+                         label,
+                         static_cast<unsigned long long>(offset + first));
+            return false;
+        }
+    }
+    return true;
+}
+
+float fp16_to_float(uint16_t h) {
+    const float sign = (h & 0x8000u) ? -1.0f : 1.0f;
+    const uint32_t exponent = (h >> 10u) & 0x1fu;
+    const uint32_t mantissa = h & 0x3ffu;
+    if (exponent == 0u) {
+        return mantissa == 0u
+            ? std::copysign(0.0f, sign)
+            : sign * std::ldexp(static_cast<float>(mantissa), -24);
+    }
+    if (exponent == 31u) {
+        return mantissa == 0u
+            ? std::copysign(std::numeric_limits<float>::infinity(), sign)
+            : std::numeric_limits<float>::quiet_NaN();
+    }
+    return sign * std::ldexp(1.0f + static_cast<float>(mantissa) / 1024.0f,
+                             static_cast<int>(exponent) - 15);
+}
+
+void get_scale_min_k4(uint32_t index, const uint8_t *packed,
+                      uint8_t *scale, uint8_t *minimum) {
+    if (index < 4u) {
+        *scale = packed[index] & 63u;
+        *minimum = packed[index + 4u] & 63u;
+    } else {
+        *scale = static_cast<uint8_t>((packed[index + 4u] & 0x0fu) |
+                                      ((packed[index - 4u] >> 6u) << 4u));
+        *minimum = static_cast<uint8_t>((packed[index + 4u] >> 4u) |
+                                        ((packed[index] >> 6u) << 4u));
+    }
+}
+
+void dequantize_q4_row(const block_q4_K_host *blocks, uint32_t in_dim,
+                       std::vector<float> *row) {
+    row->resize(in_dim);
+    float *dst = row->data();
+    const uint32_t n_blocks = in_dim / kQkK;
+    for (uint32_t block = 0; block < n_blocks; block++) {
+        const float d = fp16_to_float(blocks[block].d);
+        const float dmin = fp16_to_float(blocks[block].dmin);
+        const uint8_t *q = blocks[block].qs;
+        uint32_t scale_index = 0;
+        for (uint32_t group = 0; group < kQkK; group += 64u) {
+            uint8_t sc0 = 0, min0 = 0, sc1 = 0, min1 = 0;
+            get_scale_min_k4(scale_index, blocks[block].scales,
+                             &sc0, &min0);
+            get_scale_min_k4(scale_index + 1u, blocks[block].scales,
+                             &sc1, &min1);
+            const float ds0 = d * sc0;
+            const float dm0 = dmin * min0;
+            const float ds1 = d * sc1;
+            const float dm1 = dmin * min1;
+            for (uint32_t i = 0; i < 32u; i++) {
+                *dst++ = ds0 * static_cast<float>(q[i] & 0x0fu) - dm0;
+            }
+            for (uint32_t i = 0; i < 32u; i++) {
+                *dst++ = ds1 * static_cast<float>(q[i] >> 4u) - dm1;
+            }
+            q += 32u;
+            scale_index += 2u;
+            (void)group;
+        }
+    }
+}
+
+std::vector<uint32_t> sample_indices(uint32_t extent) {
+    std::vector<uint32_t> values = {
+        0u, extent / 3u, extent / 2u, extent - 1u,
+    };
+    std::sort(values.begin(), values.end());
+    values.erase(std::unique(values.begin(), values.end()), values.end());
+    return values;
+}
+
+bool sampled_cpu_oracle(const ds4_gpu_tensor *output,
+                        const model_fixture &model, uint64_t weight_offset,
+                        const std::vector<float> &activation,
+                        uint32_t n_tokens, uint32_t in_dim,
+                        uint32_t out_dim, const char *label) {
+    const std::vector<uint32_t> tokens = sample_indices(n_tokens);
+    const std::vector<uint32_t> rows = sample_indices(out_dim);
+    const uint64_t blocks_per_row = in_dim / kQkK;
+    const uint64_t row_bytes = blocks_per_row * sizeof(block_q4_K_host);
+    std::vector<float> weight_row;
+    // cuda/mmq/test/test_mmq_parity.cu validates Q4_K MMQ against this same
+    // dequantized-weight x original-F32 reference at 0.20*sqrt(K) absolute
+    // and 5% relative error.  Use a small shared envelope for both that Q8_1
+    // path and the legacy 256-value Q8_K activation quantizer.  Every fixture
+    // block pins |x|max to 127/32, keeping the Q8_K step bounded and stable.
+    const double abs_tol = 0.25 * std::sqrt(static_cast<double>(in_dim));
+    constexpr double rel_tol = 0.06;
+
+    for (uint32_t row : rows) {
+        const auto *blocks = reinterpret_cast<const block_q4_K_host *>(
+            model.data + weight_offset + static_cast<uint64_t>(row) * row_bytes);
+        dequantize_q4_row(blocks, in_dim, &weight_row);
+        for (uint32_t token : tokens) {
+            const float *x = activation.data() +
+                             static_cast<uint64_t>(token) * in_dim;
+            float reference = 0.0f;
+            for (uint32_t k = 0; k < in_dim; k++) {
+                reference += weight_row[k] * x[k];
+            }
+            float got = 0.0f;
+            const uint64_t element = static_cast<uint64_t>(token) * out_dim + row;
+            if (!ds4_gpu_tensor_read(output, element * sizeof(float), &got,
+                                     sizeof(got))) {
+                std::fprintf(stderr, "%s: sampled output read failed\n", label);
+                return false;
+            }
+            const double abs_error = std::fabs(static_cast<double>(got) -
+                                               reference);
+            const double rel_error = reference != 0.0f
+                ? abs_error / std::fabs(static_cast<double>(reference))
+                : (abs_error == 0.0 ? 0.0
+                                    : std::numeric_limits<double>::infinity());
+            if (!std::isfinite(got) ||
+                (abs_error > abs_tol && rel_error > rel_tol)) {
+                std::fprintf(
+                    stderr,
+                    "%s: CPU oracle mismatch token=%u row=%u got=%.7g "
+                    "reference=%.7g abs=%.5g rel=%.5g limits=%.5g/%.3g\n",
+                    label, token, row, got, reference, abs_error, rel_error,
+                    abs_tol, rel_tol);
+                return false;
+            }
+        }
+    }
+    return true;
+}
+
+bool sampled_grouped_cpu_oracle(
+        const ds4_gpu_tensor *output, const model_fixture &model,
+        uint64_t weight_offset, const std::vector<float> &activation,
+        uint32_t n_tokens, const char *label) {
+    const std::vector<uint32_t> tokens = sample_indices(n_tokens);
+    const std::vector<uint32_t> rows = sample_indices(kOutputRank);
+    const uint64_t blocks_per_row = kDenseK / kQkK;
+    const uint64_t row_bytes = blocks_per_row * sizeof(block_q4_K_host);
+    std::vector<float> weight_row;
+    const double abs_tol = 0.25 * std::sqrt(static_cast<double>(kDenseK));
+    constexpr double rel_tol = 0.06;
+
+    for (uint32_t group = 0; group < kOutputGroups; group++) {
+        for (uint32_t row : rows) {
+            const uint64_t weight_row_index =
+                static_cast<uint64_t>(group) * kOutputRank + row;
+            const auto *blocks =
+                reinterpret_cast<const block_q4_K_host *>(
+                    model.data + weight_offset + weight_row_index * row_bytes);
+            dequantize_q4_row(blocks, kDenseK, &weight_row);
+            for (uint32_t token : tokens) {
+                const float *x = activation.data() +
+                    (static_cast<uint64_t>(token) * kOutputGroups + group) *
+                        kDenseK;
+                float reference = 0.0f;
+                for (uint32_t k = 0; k < kDenseK; k++) {
+                    reference += weight_row[k] * x[k];
+                }
+                const uint64_t element =
+                    static_cast<uint64_t>(token) * kOutputLowDim +
+                    static_cast<uint64_t>(group) * kOutputRank + row;
+                float got = 0.0f;
+                if (!ds4_gpu_tensor_read(output, element * sizeof(float),
+                                         &got, sizeof(got))) {
+                    std::fprintf(stderr,
+                                 "%s: grouped output read failed\n", label);
+                    return false;
+                }
+                const double abs_error = std::fabs(
+                    static_cast<double>(got) - reference);
+                const double rel_error = reference != 0.0f
+                    ? abs_error / std::fabs(static_cast<double>(reference))
+                    : (abs_error == 0.0
+                           ? 0.0
+                           : std::numeric_limits<double>::infinity());
+                if (!std::isfinite(got) ||
+                    (abs_error > abs_tol && rel_error > rel_tol)) {
+                    std::fprintf(
+                        stderr,
+                        "%s: grouped CPU oracle mismatch token=%u group=%u "
+                        "row=%u got=%.7g reference=%.7g abs=%.5g rel=%.5g "
+                        "limits=%.5g/%.3g\n",
+                        label, token, group, row, got, reference, abs_error,
+                        rel_error, abs_tol, rel_tol);
+                    return false;
+                }
+            }
+        }
+    }
+    return true;
+}
+
+bool select_grouped_prefill_legacy() {
+    return unsetenv(kGroupedPrefillEnable) == 0 &&
+           setenv(kGroupedPrefillDisable, "1", 1) == 0 &&
+           unsetenv(kGroupedPrefillRequire) == 0 &&
+           unsetenv(kGroupedSingleGridEnable) == 0 &&
+           setenv(kGroupedSingleGridDisable, "1", 1) == 0 &&
+           unsetenv(kGroupedSingleGridRequire) == 0 &&
+           unsetenv(kGroupedQ81Disable) == 0 &&
+           unsetenv(kGroupedQ81Require) == 0;
+}
+
+bool select_grouped_prefill_grid8() {
+    return setenv(kGroupedPrefillEnable, "1", 1) == 0 &&
+           unsetenv(kGroupedPrefillDisable) == 0 &&
+           setenv(kGroupedPrefillRequire, "1", 1) == 0 &&
+           unsetenv(kGroupedSingleGridEnable) == 0 &&
+           setenv(kGroupedSingleGridDisable, "1", 1) == 0 &&
+           unsetenv(kGroupedSingleGridRequire) == 0 &&
+           unsetenv(kGroupedQ81Disable) == 0 &&
+           unsetenv(kGroupedQ81Require) == 0;
+}
+
+bool select_grouped_prefill_single_grid() {
+    return setenv(kGroupedPrefillEnable, "1", 1) == 0 &&
+           unsetenv(kGroupedPrefillDisable) == 0 &&
+           setenv(kGroupedPrefillRequire, "1", 1) == 0 &&
+           setenv(kGroupedSingleGridEnable, "1", 1) == 0 &&
+           unsetenv(kGroupedSingleGridDisable) == 0 &&
+           setenv(kGroupedSingleGridRequire, "1", 1) == 0 &&
+           unsetenv(kGroupedQ81Disable) == 0 &&
+           unsetenv(kGroupedQ81Require) == 0;
+}
+
+bool select_grouped_q81_reference() {
+    return select_grouped_prefill_grid8() &&
+           setenv(kGroupedQ81Disable, "1", 1) == 0 &&
+           unsetenv(kGroupedQ81Require) == 0;
+}
+
+bool select_grouped_q81_candidate() {
+    return select_grouped_prefill_grid8() &&
+           unsetenv(kGroupedQ81Disable) == 0 &&
+           setenv(kGroupedQ81Require, "1", 1) == 0;
+}
+
+double percentile(std::vector<double> sorted, double fraction) {
+    std::sort(sorted.begin(), sorted.end());
+    if (sorted.empty()) return 0.0;
+    const double position = fraction * static_cast<double>(sorted.size() - 1u);
+    const size_t lo = static_cast<size_t>(std::floor(position));
+    const size_t hi = static_cast<size_t>(std::ceil(position));
+    const double alpha = position - static_cast<double>(lo);
+    return sorted[lo] + (sorted[hi] - sorted[lo]) * alpha;
+}
+
+stats summarize(const std::vector<double> &samples) {
+    stats out;
+    out.minimum = *std::min_element(samples.begin(), samples.end());
+    out.median = percentile(samples, 0.5);
+    out.p95 = percentile(samples, 0.95);
+    for (double sample : samples) out.mean += sample;
+    out.mean /= static_cast<double>(samples.size());
+    return out;
+}
+
+const char *path_name(cuda_path path) {
+    return path == cuda_path::mmq ? "mmq" : "legacy";
+}
+
+const char *case_scope(const config &cfg) {
+    switch (cfg.selected) {
+        case bench_case::all:
+            return cfg.kernel_16warp ? "dense,pair,q_b,output_b"
+                                     : "dense,pair,q_b,outa";
+        case bench_case::dense: return "dense";
+        case bench_case::pair: return "pair";
+        case bench_case::qb: return "q_b";
+        case bench_case::outa: return "outa";
+        case bench_case::outb: return "output_b";
+    }
+    return "unknown";
+}
+
+bool select_arm(const arm &which) {
+    return !which.select || which.select();
+}
+
+bool dispatch_oracle_arm(const arm &which, uint32_t set) {
+    return which.oracle_dispatch ? which.oracle_dispatch(set)
+                                 : which.dispatch(set);
+}
+
+bool benchmark_single_path(
+        const char *case_name, uint32_t n_tokens, uint32_t in_dim,
+        uint32_t out_dim, const config &cfg, const arm &which,
+        const std::function<bool()> &oracle_prepare,
+        const std::function<bool(uint32_t)> &oracle) {
+    auto validate = [&](const char *phase) {
+        for (uint32_t set = 0; set < cfg.sets; set++) {
+            if (!oracle_prepare() || !select_arm(which) ||
+                !dispatch_oracle_arm(which, set) || !ds4_gpu_synchronize() ||
+                !oracle(set)) {
+                std::fprintf(stderr,
+                             "cuda-q4-prefill-bench: %s %s oracle failed "
+                             "for weight set %u\n",
+                             case_name, phase, set);
+                return false;
+            }
+        }
+        return true;
+    };
+
+    // Validate every rotating weight set and prime lazy Q8 scratch before
+    // recording a CUDA event.  Upload, poison, readback, and CPU work remain
+    // outside the measured interval.
+    if (!validate("pre-timing")) return false;
+
+    for (uint32_t i = 0; i < cfg.warmup; i++) {
+        if (!select_arm(which) || !which.dispatch(i % cfg.sets) ||
+            !ds4_gpu_synchronize()) {
+            return false;
+        }
+    }
+
+    event_timer timer;
+    std::vector<double> samples;
+    samples.reserve(cfg.samples);
+    for (uint32_t i = 0; i < cfg.samples; i++) {
+        float elapsed = 0.0f;
+        if (!select_arm(which) ||
+            !timer.measure([&]() { return which.dispatch(i % cfg.sets); },
+                           &elapsed)) {
+            std::fprintf(stderr,
+                         "cuda-q4-prefill-bench: %s/%s timed dispatch failed\n",
+                         case_name, which.name);
+            return false;
+        }
+        samples.push_back(static_cast<double>(elapsed));
+    }
+
+    const uint32_t timed_set = (cfg.samples - 1u) % cfg.sets;
+    if (!oracle(timed_set)) {
+        std::fprintf(stderr,
+                     "cuda-q4-prefill-bench: %s timed-output oracle failed "
+                     "for weight set %u\n",
+                     case_name, timed_set);
+        return false;
+    }
+
+    // Timed launches overwrite the validated buffers.  Re-run the complete
+    // finite/CPU/guard oracle for every resident weight set after timing so a
+    // late or state-dependent corruption cannot survive as a benchmark result.
+    if (!validate("post-timing")) return false;
+
+    const stats result = summarize(samples);
+    const double macs = static_cast<double>(n_tokens) * in_dim * out_dim;
+    const double gmac_s = macs / (result.median * 1.0e6);
+    std::printf(
+        "DS4_CUDA_Q4_PREFILL_BENCH case=%s path=%s N=%u K=%u M=%u "
+        "variant=%s samples=%u sets=%u ms_p50=%.6f ms_min=%.6f "
+        "ms_p95=%.6f ms_mean=%.6f gmac_s=%.3f\n",
+        case_name, path_name(cfg.path), n_tokens, in_dim, out_dim, which.name,
+        cfg.samples, cfg.sets, result.median, result.minimum, result.p95,
+        result.mean, gmac_s);
+    std::fflush(stdout);
+    return true;
+}
+
+bool benchmark_pair_arms(
+        const char *case_name, uint32_t n_tokens, uint32_t in_dim,
+        uint32_t out_dim, uint64_t focus_macs_per_token,
+        uint64_t common_macs_per_token, const config &cfg,
+        const arm &baseline, const arm &candidate,
+        const std::function<bool()> &oracle_prepare,
+        const std::function<bool(uint32_t)> &oracle,
+        const char *timing_scope = "production_api_cuda_events") {
+    auto validate = [&](const char *phase) {
+        for (uint32_t set = 0; set < cfg.sets; set++) {
+            if (!oracle_prepare() || !select_arm(baseline) ||
+                !dispatch_oracle_arm(baseline, set) ||
+                !ds4_gpu_synchronize() || !select_arm(candidate) ||
+                !dispatch_oracle_arm(candidate, set) ||
+                !ds4_gpu_synchronize() || !oracle(set)) {
+                std::fprintf(stderr,
+                             "cuda-q4-prefill-bench: %s %s oracle failed "
+                             "for weight set %u\n",
+                             case_name, phase, set);
+                return false;
+            }
+        }
+        return true;
+    };
+
+    if (!validate("pre-timing")) return false;
+
+    for (uint32_t i = 0; i < cfg.warmup; i++) {
+        const uint32_t set = i % cfg.sets;
+        if (!select_arm(baseline) || !baseline.dispatch(set) ||
+            !ds4_gpu_synchronize() || !select_arm(candidate) ||
+            !candidate.dispatch(set) || !ds4_gpu_synchronize()) {
+            return false;
+        }
+    }
+
+    event_timer timer;
+    std::vector<double> a_samples;
+    std::vector<double> b_samples;
+    a_samples.reserve(cfg.samples);
+    b_samples.reserve(cfg.samples);
+    auto take = [&](const arm &which, uint32_t set,
+                    std::vector<double> *samples) {
+        float elapsed = 0.0f;
+        if (!select_arm(which) ||
+            !timer.measure([&]() { return which.dispatch(set); }, &elapsed)) {
+            std::fprintf(stderr,
+                         "cuda-q4-prefill-bench: %s/%s timed dispatch "
+                         "failed\n",
+                         case_name, which.name);
+            return false;
+        }
+        samples->push_back(static_cast<double>(elapsed));
+        return true;
+    };
+
+    // Two samples per arm per cycle. Alternating ABBA/BAAB balances order
+    // while both arms see identical rotating resident weights.
+    for (uint32_t cycle = 0; a_samples.size() < cfg.samples; cycle++) {
+        const uint32_t set0 = (cycle * 2u) % cfg.sets;
+        const uint32_t set1 = (cycle * 2u + 1u) % cfg.sets;
+        if ((cycle & 1u) == 0u) {
+            if (!take(baseline, set0, &a_samples) ||
+                !take(candidate, set0, &b_samples) ||
+                !take(candidate, set1, &b_samples) ||
+                !take(baseline, set1, &a_samples)) return false;
+        } else {
+            if (!take(candidate, set0, &b_samples) ||
+                !take(baseline, set0, &a_samples) ||
+                !take(baseline, set1, &a_samples) ||
+                !take(candidate, set1, &b_samples)) return false;
+        }
+    }
+
+    // samples is a multiple of four, so the final ABBA/BAAB cycle leaves both
+    // output buffers holding weight set samples-1. Check those exact timed
+    // results before oracle_prepare is allowed to poison or reset any guard.
+    const uint32_t timed_set = (cfg.samples - 1u) % cfg.sets;
+    if (!oracle(timed_set)) {
+        std::fprintf(stderr,
+                     "cuda-q4-prefill-bench: %s timed-output oracle failed "
+                     "for weight set %u\n",
+                     case_name, timed_set);
+        return false;
+    }
+
+    // Both arms must still agree bit-for-bit (and with their CPU/guard
+    // oracle) after the measured ABBA/BAAB sequence, for every weight set.
+    if (!validate("post-timing")) return false;
+
+    const stats a = summarize(a_samples);
+    const stats b = summarize(b_samples);
+    std::vector<double> paired_delta;
+    paired_delta.reserve(a_samples.size());
+    for (size_t i = 0; i < a_samples.size(); i++) {
+        paired_delta.push_back((b_samples[i] / a_samples[i] - 1.0) * 100.0);
+    }
+    const double paired_median = percentile(paired_delta, 0.5);
+    const double median_delta = (b.median / a.median - 1.0) * 100.0;
+    const double speedup = (a.median / b.median - 1.0) * 100.0;
+    const double macs = static_cast<double>(n_tokens) *
+                        (focus_macs_per_token + common_macs_per_token);
+    std::printf(
+        "DS4_CUDA_Q4_PREFILL_BENCH case=%s path=%s N=%u K=%u M=%u "
+        "baseline=%s candidate=%s samples=%u sets=%u "
+        "timing=%s "
+        "focus_macs_per_token=%llu common_macs_per_token=%llu "
+        "baseline_ms_p50=%.6f candidate_ms_p50=%.6f "
+        "baseline_ms_min=%.6f candidate_ms_min=%.6f "
+        "baseline_ms_p95=%.6f candidate_ms_p95=%.6f "
+        "baseline_gmac_s=%.3f candidate_gmac_s=%.3f "
+        "candidate_delta_pct=%.3f paired_delta_pct_p50=%.3f "
+        "speedup_pct=%.3f\n",
+        case_name, path_name(cfg.path), n_tokens, in_dim, out_dim,
+        baseline.name, candidate.name, cfg.samples, cfg.sets,
+        timing_scope,
+        static_cast<unsigned long long>(focus_macs_per_token),
+        static_cast<unsigned long long>(common_macs_per_token),
+        a.median, b.median, a.minimum, b.minimum, a.p95, b.p95,
+        macs / (a.median * 1.0e6), macs / (b.median * 1.0e6),
+        median_delta, paired_median, speedup);
+    std::fflush(stdout);
+    return true;
+}
+
+bool run_q4_16warp_kernel(
+        const model_fixture &model, const config &cfg, uint32_t n_tokens,
+        uint32_t in_dim, uint32_t out_dim,
+        uint64_t weight_set::*weight_offset_member, const char *case_name) {
+    uint64_t x_elements = 0;
+    uint64_t out_elements = 0;
+    if (!checked_mul(n_tokens, in_dim, &x_elements) ||
+        !checked_mul(n_tokens, out_dim, &out_elements)) {
+        return false;
+    }
+    const uint64_t x_bytes = x_elements * sizeof(float);
+    const uint64_t out_bytes = out_elements * sizeof(float);
+    const uint64_t guard_bytes = kGuardWords * sizeof(uint32_t);
+    const uint64_t weight_bytes = q4_weight_bytes(in_dim, out_dim);
+    const size_t q8_bytes = ds4_mmq_q4_K_q8_1_scratch_bytes(
+        static_cast<int>(n_tokens), static_cast<int>(in_dim));
+    if (q8_bytes == 0u) {
+        std::fprintf(stderr,
+                     "cuda-q4-prefill-bench: %s kernel-only Q8_1 scratch "
+                     "shape rejected for N=%u K=%u\n",
+                     case_name, n_tokens, in_dim);
+        return false;
+    }
+    int nsm = 0;
+    if (!current_device_nsm(&nsm)) return false;
+    const size_t fixup_bytes =
+        ds4_mmq_q4_K_dense_16warp_streamk_scratch_bytes(
+            static_cast<int>(out_dim), static_cast<int>(n_tokens), nsm);
+
+    tensor_owner x(x_bytes + guard_bytes);
+    tensor_owner reference(out_bytes + guard_bytes);
+    tensor_owner candidate(out_bytes + guard_bytes);
+    cuda_buffer prequant(q8_bytes);
+    guarded_cuda_buffer fixup(fixup_bytes, 0x1a000u);
+    std::vector<float> activation;
+    fill_activation(&activation, n_tokens, in_dim);
+    if (!x.ptr || !reference.ptr || !candidate.ptr || !prequant.ptr ||
+        prequant.status != cudaSuccess || fixup.status != cudaSuccess ||
+        (fixup_bytes != 0u && !fixup.ptr) ||
+        !ds4_gpu_tensor_write(x.ptr, 0, activation.data(), x_bytes) ||
+        !prepare_guard(x.ptr, x_bytes, 0x12000u)) {
+        std::fprintf(stderr,
+                     "cuda-q4-prefill-bench: %s N=%u kernel-only tensor/"
+                     "scratch setup failed (%s)\n",
+                     case_name, n_tokens,
+                     prequant.status != cudaSuccess
+                         ? cudaGetErrorString(prequant.status)
+                         : fixup.status != cudaSuccess
+                               ? cudaGetErrorString(fixup.status)
+                               : "tensor setup");
+        return false;
+    }
+
+    const auto *x_device = static_cast<const float *>(
+        ds4_gpu_tensor_contents(x.ptr));
+    auto *reference_device = static_cast<float *>(
+        ds4_gpu_tensor_contents(reference.ptr));
+    auto *candidate_device = static_cast<float *>(
+        ds4_gpu_tensor_contents(candidate.ptr));
+    if (!x_device || !reference_device || !candidate_device) {
+        std::fprintf(stderr,
+                     "cuda-q4-prefill-bench: %s N=%u device tensor pointer "
+                     "lookup failed\n",
+                     case_name, n_tokens);
+        return false;
+    }
+
+    std::vector<const void *> weight_device(cfg.sets, nullptr);
+    for (uint32_t set = 0; set < cfg.sets; set++) {
+        const uint64_t offset = model.weights[set].*weight_offset_member;
+        if (!ds4_cuda_test_model_range_device_ptr(
+                model.data, model.size, offset, weight_bytes,
+                /*logical_tier=*/0, &weight_device[set]) ||
+            !weight_device[set]) {
+            std::fprintf(stderr,
+                         "cuda-q4-prefill-bench: %s N=%u resident weight "
+                         "pointer lookup failed for set %u\n",
+                         case_name, n_tokens, set);
+            return false;
+        }
+    }
+
+    // Quantize exactly once. Both A/B arms consume this immutable canonical
+    // DS4 Q8_1 buffer. Timed work is restricted to each GEMM's scheduling,
+    // required stream-K scratch clear, producer, and fixup kernels.
+    const int quant_rc = ds4_mmq_q4_K_quantize_q8_1_for_test(
+        x_device, prequant.ptr, q8_bytes, static_cast<int>(n_tokens),
+        static_cast<int>(in_dim), /*stream=*/nullptr);
+    if (quant_rc != 0 || !ds4_gpu_synchronize() ||
+        !check_guard(x.ptr, x_bytes, 0x12000u,
+                     "kernel-only prequant input")) {
+        std::fprintf(stderr,
+                     "cuda-q4-prefill-bench: %s N=%u prequantization "
+                     "failed rc=%d\n",
+                     case_name, n_tokens, quant_rc);
+        return false;
+    }
+
+    const arm baseline = {
+        "canonical_preq_stream_k",
+        [&](uint32_t set) {
+            return ds4_mmq_q4_K_dense_preq_reference_for_test(
+                       weight_device[set], prequant.ptr, q8_bytes,
+                       reference_device, static_cast<int>(out_dim),
+                       static_cast<int>(n_tokens), static_cast<int>(in_dim),
+                       /*use_stream_k=*/1, fixup.ptr, fixup_bytes,
+                       /*stream=*/nullptr) == 0;
+        },
+        {}};
+    const arm candidate_arm = {
+        "q4_16warp_m128n128_stream_k",
+        [&](uint32_t set) {
+            return ds4_mmq_q4_K_dense_16warp_streamk_enqueue(
+                       weight_device[set], prequant.ptr, candidate_device,
+                       fixup.ptr, fixup_bytes,
+                       static_cast<int>(out_dim),
+                       static_cast<int>(n_tokens), static_cast<int>(in_dim),
+                       nsm, /*stream=*/nullptr) == 0;
+        },
+        {},
+        [&](uint32_t set) {
+            return ds4_mmq_q4_K_dense_16warp_streamk_enqueue(
+                       weight_device[set], prequant.ptr, candidate_device,
+                       fixup.ptr, fixup_bytes, static_cast<int>(out_dim),
+                       static_cast<int>(n_tokens), static_cast<int>(in_dim),
+                       nsm, /*stream=*/nullptr) == 0;
+        }};
+    const bool ok = benchmark_pair_arms(
+        case_name, n_tokens, in_dim, out_dim,
+        static_cast<uint64_t>(in_dim) * out_dim, 0u, cfg,
+        baseline, candidate_arm,
+        [&]() {
+            return poison_output(reference.ptr, out_bytes, 0x7fc50005u,
+                                 0x13000u) &&
+                   poison_output(candidate.ptr, out_bytes, 0x7fc60006u,
+                                 0x14000u);
+        },
+        [&](uint32_t set) {
+            const uint64_t offset =
+                model.weights[set].*weight_offset_member;
+            return bitwise_equal(reference.ptr, candidate.ptr, out_bytes,
+                                 "16-warp vs canonical stream-K") &&
+                   output_is_finite(reference.ptr, out_bytes,
+                                    "16-warp canonical output") &&
+                   output_is_finite(candidate.ptr, out_bytes,
+                                    "16-warp candidate output") &&
+                   sampled_cpu_oracle(
+                       reference.ptr, model, offset, activation, n_tokens,
+                       in_dim, out_dim, "16-warp canonical") &&
+                   check_guard(x.ptr, x_bytes, 0x12000u,
+                               "16-warp input") &&
+                   check_guard(reference.ptr, out_bytes, 0x13000u,
+                               "16-warp canonical output") &&
+                   check_guard(candidate.ptr, out_bytes, 0x14000u,
+                               "16-warp candidate output") &&
+                   fixup.intact("16-warp Stream-K scratch");
+        },
+        "kernel_only_prequant");
+    return ok &&
+           check_guard(x.ptr, x_bytes, 0x12000u,
+                       "16-warp input final") &&
+           check_guard(reference.ptr, out_bytes, 0x13000u,
+                       "16-warp canonical output final") &&
+           check_guard(candidate.ptr, out_bytes, 0x14000u,
+                       "16-warp candidate output final") &&
+           fixup.intact("16-warp Stream-K scratch final");
+}
+
+bool run_q4_16warp_pair_kernel(
+        const model_fixture &model, const config &cfg, uint32_t n_tokens) {
+    uint64_t x_elements = 0;
+    uint64_t out0_elements = 0;
+    uint64_t out1_elements = 0;
+    if (!checked_mul(n_tokens, kDenseK, &x_elements) ||
+        !checked_mul(n_tokens, kDenseM, &out0_elements) ||
+        !checked_mul(n_tokens, kKvM, &out1_elements)) {
+        return false;
+    }
+    const uint64_t x_bytes = x_elements * sizeof(float);
+    const uint64_t out0_bytes = out0_elements * sizeof(float);
+    const uint64_t out1_bytes = out1_elements * sizeof(float);
+    const uint64_t guard_bytes = kGuardWords * sizeof(uint32_t);
+    const uint64_t weight0_bytes = q4_weight_bytes(kDenseK, kDenseM);
+    const uint64_t weight1_bytes = q4_weight_bytes(kDenseK, kKvM);
+    const size_t q8_bytes = ds4_mmq_q4_K_q8_1_scratch_bytes(
+        static_cast<int>(n_tokens), static_cast<int>(kDenseK));
+    if (q8_bytes == 0u) {
+        std::fprintf(stderr,
+                     "cuda-q4-prefill-bench: pair kernel-only Q8_1 scratch "
+                     "shape rejected for N=%u K=%u\n",
+                     n_tokens, kDenseK);
+        return false;
+    }
+    int nsm = 0;
+    if (!current_device_nsm(&nsm)) return false;
+    const size_t fixup0_bytes =
+        ds4_mmq_q4_K_dense_16warp_streamk_scratch_bytes(
+            static_cast<int>(kDenseM), static_cast<int>(n_tokens), nsm);
+    const size_t fixup1_bytes =
+        ds4_mmq_q4_K_dense_16warp_streamk_scratch_bytes(
+            static_cast<int>(kKvM), static_cast<int>(n_tokens), nsm);
+    const size_t fixup_bytes = fixup0_bytes > fixup1_bytes
+        ? fixup0_bytes : fixup1_bytes;
+
+    tensor_owner x(x_bytes + guard_bytes);
+    tensor_owner reference0(out0_bytes + guard_bytes);
+    tensor_owner reference1(out1_bytes + guard_bytes);
+    tensor_owner candidate0(out0_bytes + guard_bytes);
+    tensor_owner candidate1(out1_bytes + guard_bytes);
+    cuda_buffer prequant(q8_bytes);
+    guarded_cuda_buffer fixup(fixup_bytes, 0x1b000u);
+    std::vector<float> activation;
+    fill_activation(&activation, n_tokens, kDenseK);
+    if (!x.ptr || !reference0.ptr || !reference1.ptr || !candidate0.ptr ||
+        !candidate1.ptr || !prequant.ptr || prequant.status != cudaSuccess ||
+        fixup.status != cudaSuccess ||
+        (fixup_bytes != 0u && !fixup.ptr) ||
+        !ds4_gpu_tensor_write(x.ptr, 0, activation.data(), x_bytes) ||
+        !prepare_guard(x.ptr, x_bytes, 0x15000u)) {
+        std::fprintf(stderr,
+                     "cuda-q4-prefill-bench: pair N=%u kernel-only tensor/"
+                     "scratch setup failed (%s)\n",
+                     n_tokens,
+                     prequant.status != cudaSuccess
+                         ? cudaGetErrorString(prequant.status)
+                         : fixup.status != cudaSuccess
+                               ? cudaGetErrorString(fixup.status)
+                               : "tensor setup");
+        return false;
+    }
+
+    const auto *x_device = static_cast<const float *>(
+        ds4_gpu_tensor_contents(x.ptr));
+    auto *reference0_device = static_cast<float *>(
+        ds4_gpu_tensor_contents(reference0.ptr));
+    auto *reference1_device = static_cast<float *>(
+        ds4_gpu_tensor_contents(reference1.ptr));
+    auto *candidate0_device = static_cast<float *>(
+        ds4_gpu_tensor_contents(candidate0.ptr));
+    auto *candidate1_device = static_cast<float *>(
+        ds4_gpu_tensor_contents(candidate1.ptr));
+    if (!x_device || !reference0_device || !reference1_device ||
+        !candidate0_device || !candidate1_device) {
+        std::fprintf(stderr,
+                     "cuda-q4-prefill-bench: pair N=%u device tensor pointer "
+                     "lookup failed\n",
+                     n_tokens);
+        return false;
+    }
+
+    std::vector<const void *> weight0_device(cfg.sets, nullptr);
+    std::vector<const void *> weight1_device(cfg.sets, nullptr);
+    for (uint32_t set = 0; set < cfg.sets; set++) {
+        if (!ds4_cuda_test_model_range_device_ptr(
+                model.data, model.size, model.weights[set].dense_offset,
+                weight0_bytes, /*logical_tier=*/0, &weight0_device[set]) ||
+            !weight0_device[set] ||
+            !ds4_cuda_test_model_range_device_ptr(
+                model.data, model.size, model.weights[set].kv_offset,
+                weight1_bytes, /*logical_tier=*/0, &weight1_device[set]) ||
+            !weight1_device[set]) {
+            std::fprintf(stderr,
+                         "cuda-q4-prefill-bench: pair N=%u resident weight "
+                         "pointer lookup failed for set %u\n",
+                         n_tokens, set);
+            return false;
+        }
+    }
+
+    // One immutable activation quantization is shared by both Q-A/KV legs and
+    // is deliberately outside every event interval.
+    const int quant_rc = ds4_mmq_q4_K_quantize_q8_1_for_test(
+        x_device, prequant.ptr, q8_bytes, static_cast<int>(n_tokens),
+        static_cast<int>(kDenseK), /*stream=*/nullptr);
+    if (quant_rc != 0 || !ds4_gpu_synchronize() ||
+        !check_guard(x.ptr, x_bytes, 0x15000u,
+                     "kernel-only pair prequant input")) {
+        std::fprintf(stderr,
+                     "cuda-q4-prefill-bench: pair N=%u prequantization "
+                     "failed rc=%d\n",
+                     n_tokens, quant_rc);
+        return false;
+    }
+
+    const arm baseline = {
+        "two_canonical_preq_stream_k",
+        [&](uint32_t set) {
+            return ds4_mmq_q4_K_dense_preq_reference_for_test(
+                       weight0_device[set], prequant.ptr, q8_bytes,
+                       reference0_device, static_cast<int>(kDenseM),
+                       static_cast<int>(n_tokens), static_cast<int>(kDenseK),
+                       /*use_stream_k=*/1, fixup.ptr, fixup_bytes,
+                       /*stream=*/nullptr) == 0 &&
+                   ds4_mmq_q4_K_dense_preq_reference_for_test(
+                       weight1_device[set], prequant.ptr, q8_bytes,
+                       reference1_device, static_cast<int>(kKvM),
+                       static_cast<int>(n_tokens), static_cast<int>(kDenseK),
+                       /*use_stream_k=*/1, fixup.ptr, fixup_bytes,
+                       /*stream=*/nullptr) == 0;
+        },
+        {}};
+    const arm candidate = {
+        "two_q4_16warp_m128n128_stream_k",
+        [&](uint32_t set) {
+            return ds4_mmq_q4_K_dense_16warp_streamk_enqueue(
+                       weight0_device[set], prequant.ptr, candidate0_device,
+                       fixup.ptr, fixup_bytes,
+                       static_cast<int>(kDenseM),
+                       static_cast<int>(n_tokens), static_cast<int>(kDenseK),
+                       nsm, /*stream=*/nullptr) == 0 &&
+                   ds4_mmq_q4_K_dense_16warp_streamk_enqueue(
+                       weight1_device[set], prequant.ptr, candidate1_device,
+                       fixup.ptr, fixup_bytes,
+                       static_cast<int>(kKvM),
+                       static_cast<int>(n_tokens), static_cast<int>(kDenseK),
+                       nsm, /*stream=*/nullptr) == 0;
+        },
+        {},
+        [&](uint32_t set) {
+            return ds4_mmq_q4_K_dense_16warp_streamk_enqueue(
+                       weight0_device[set], prequant.ptr, candidate0_device,
+                       fixup.ptr, fixup_bytes, static_cast<int>(kDenseM),
+                       static_cast<int>(n_tokens), static_cast<int>(kDenseK),
+                       nsm, /*stream=*/nullptr) == 0 &&
+                   ds4_mmq_q4_K_dense_16warp_streamk_enqueue(
+                       weight1_device[set], prequant.ptr, candidate1_device,
+                       fixup.ptr, fixup_bytes, static_cast<int>(kKvM),
+                       static_cast<int>(n_tokens), static_cast<int>(kDenseK),
+                       nsm, /*stream=*/nullptr) == 0;
+        }};
+    const bool ok = benchmark_pair_arms(
+        "pair", n_tokens, kDenseK, kDenseM + kKvM,
+        static_cast<uint64_t>(kDenseK) * (kDenseM + kKvM), 0u, cfg,
+        baseline, candidate,
+        [&]() {
+            return poison_output(reference0.ptr, out0_bytes, 0x7fc70007u,
+                                 0x16000u) &&
+                   poison_output(reference1.ptr, out1_bytes, 0x7fc80008u,
+                                 0x17000u) &&
+                   poison_output(candidate0.ptr, out0_bytes, 0x7fc90009u,
+                                 0x18000u) &&
+                   poison_output(candidate1.ptr, out1_bytes, 0x7fca000au,
+                                 0x19000u);
+        },
+        [&](uint32_t set) {
+            return bitwise_equal(reference0.ptr, candidate0.ptr, out0_bytes,
+                                 "16-warp pair q_a vs canonical") &&
+                   bitwise_equal(reference1.ptr, candidate1.ptr, out1_bytes,
+                                 "16-warp pair kv vs canonical") &&
+                   output_is_finite(reference0.ptr, out0_bytes,
+                                    "16-warp pair q_a canonical") &&
+                   output_is_finite(reference1.ptr, out1_bytes,
+                                    "16-warp pair kv canonical") &&
+                   output_is_finite(candidate0.ptr, out0_bytes,
+                                    "16-warp pair q_a candidate") &&
+                   output_is_finite(candidate1.ptr, out1_bytes,
+                                    "16-warp pair kv candidate") &&
+                   sampled_cpu_oracle(
+                       reference0.ptr, model,
+                       model.weights[set].dense_offset, activation, n_tokens,
+                       kDenseK, kDenseM, "16-warp pair q_a canonical") &&
+                   sampled_cpu_oracle(
+                       reference1.ptr, model, model.weights[set].kv_offset,
+                       activation, n_tokens, kDenseK, kKvM,
+                       "16-warp pair kv canonical") &&
+                   check_guard(x.ptr, x_bytes, 0x15000u,
+                               "16-warp pair input") &&
+                   check_guard(reference0.ptr, out0_bytes, 0x16000u,
+                               "16-warp pair q_a canonical") &&
+                   check_guard(reference1.ptr, out1_bytes, 0x17000u,
+                               "16-warp pair kv canonical") &&
+                   check_guard(candidate0.ptr, out0_bytes, 0x18000u,
+                               "16-warp pair q_a candidate") &&
+                   check_guard(candidate1.ptr, out1_bytes, 0x19000u,
+                               "16-warp pair kv candidate") &&
+                   fixup.intact("16-warp pair Stream-K scratch");
+        },
+        "kernel_only_prequant");
+    return ok &&
+           check_guard(x.ptr, x_bytes, 0x15000u,
+                       "16-warp pair input final") &&
+           check_guard(reference0.ptr, out0_bytes, 0x16000u,
+                       "16-warp pair q_a canonical final") &&
+           check_guard(reference1.ptr, out1_bytes, 0x17000u,
+                       "16-warp pair kv canonical final") &&
+           check_guard(candidate0.ptr, out0_bytes, 0x18000u,
+                       "16-warp pair q_a candidate final") &&
+           check_guard(candidate1.ptr, out1_bytes, 0x19000u,
+                       "16-warp pair kv candidate final") &&
+           fixup.intact("16-warp pair Stream-K scratch final");
+}
+
+bool run_dense(const model_fixture &model, const config &cfg,
+               uint32_t n_tokens) {
+    uint64_t out_elements = 0;
+    if (!checked_mul(n_tokens, kDenseM, &out_elements)) return false;
+    const uint64_t x_bytes = static_cast<uint64_t>(n_tokens) * kDenseK *
+                             sizeof(float);
+    const uint64_t out_bytes = out_elements * sizeof(float);
+    const uint64_t guard_bytes = kGuardWords * sizeof(uint32_t);
+    tensor_owner x(x_bytes + guard_bytes);
+    tensor_owner output(out_bytes + guard_bytes);
+    std::vector<float> activation;
+    fill_activation(&activation, n_tokens, kDenseK);
+    if (!x.ptr || !output.ptr ||
+        !ds4_gpu_tensor_write(x.ptr, 0, activation.data(), x_bytes) ||
+        !prepare_guard(x.ptr, x_bytes, 0x1000u)) {
+        std::fprintf(stderr, "dense N=%u: tensor setup failed\n", n_tokens);
+        return false;
+    }
+    const arm path = {
+        cfg.path == cuda_path::mmq ? "mmq" : "legacy_q8k",
+        [&](uint32_t set) {
+            return ds4_gpu_matmul_quant_tensor(
+                       output.ptr, model.data, model.size,
+                       model.weights[set].dense_offset, kQ4Type, kDenseK,
+                       kDenseM, x.ptr, n_tokens) != 0;
+        },
+        {}};
+    return benchmark_single_path(
+        "dense", n_tokens, kDenseK, kDenseM, cfg, path,
+        [&]() {
+            return poison_output(output.ptr, out_bytes, 0x7fc10001u,
+                                 0x2000u);
+        },
+        [&](uint32_t set) {
+            return output_is_finite(output.ptr, out_bytes, "dense output") &&
+                   sampled_cpu_oracle(output.ptr, model,
+                                      model.weights[set].dense_offset,
+                                      activation, n_tokens, kDenseK, kDenseM,
+                                      "dense") &&
+                   check_guard(x.ptr, x_bytes, 0x1000u, "dense input") &&
+                   check_guard(output.ptr, out_bytes, 0x2000u,
+                               "dense output");
+        }) &&
+        check_guard(x.ptr, x_bytes, 0x1000u, "dense input final") &&
+        check_guard(output.ptr, out_bytes, 0x2000u, "dense output final");
+}
+
+bool run_pair(const model_fixture &model, const config &cfg,
+              uint32_t n_tokens) {
+    if (cfg.path == cuda_path::legacy) {
+        std::printf(
+            "DS4_CUDA_Q4_PREFILL_SKIP case=pair path=legacy N=%u "
+            "reason=fused_prefill_api_requires_mmq\n",
+            n_tokens);
+        std::fflush(stdout);
+        return true;
+    }
+
+    const uint64_t x_bytes = static_cast<uint64_t>(n_tokens) * kDenseK *
+                             sizeof(float);
+    const uint64_t out0_bytes = static_cast<uint64_t>(n_tokens) * kDenseM *
+                                sizeof(float);
+    const uint64_t out1_bytes = static_cast<uint64_t>(n_tokens) * kKvM *
+                                sizeof(float);
+    const uint64_t guard_bytes = kGuardWords * sizeof(uint32_t);
+    tensor_owner x(x_bytes + guard_bytes);
+    tensor_owner separate0(out0_bytes + guard_bytes);
+    tensor_owner separate1(out1_bytes + guard_bytes);
+    tensor_owner pair0(out0_bytes + guard_bytes);
+    tensor_owner pair1(out1_bytes + guard_bytes);
+    std::vector<float> activation;
+    fill_activation(&activation, n_tokens, kDenseK);
+    if (!x.ptr || !separate0.ptr || !separate1.ptr || !pair0.ptr ||
+        !pair1.ptr ||
+        !ds4_gpu_tensor_write(x.ptr, 0, activation.data(), x_bytes) ||
+        !prepare_guard(x.ptr, x_bytes, 0x3000u)) {
+        std::fprintf(stderr, "pair N=%u: tensor setup failed\n", n_tokens);
+        return false;
+    }
+    const arm baseline = {
+        "two_dense_mmq",
+        [&](uint32_t set) {
+            return ds4_gpu_matmul_quant_tensor(
+                       separate0.ptr, model.data, model.size,
+                       model.weights[set].dense_offset, kQ4Type, kDenseK,
+                       kDenseM, x.ptr, n_tokens) != 0 &&
+                   ds4_gpu_matmul_quant_tensor(
+                       separate1.ptr, model.data, model.size,
+                       model.weights[set].kv_offset, kQ4Type, kDenseK,
+                       kKvM, x.ptr, n_tokens) != 0;
+        },
+        {}};
+    const arm candidate = {
+        "pair_mmq",
+        [&](uint32_t set) {
+            return ds4_gpu_matmul_q4_K_pair_tensor(
+                       pair0.ptr, pair1.ptr, model.data, model.size,
+                       model.weights[set].dense_offset,
+                       model.weights[set].kv_offset, kDenseK, kDenseM, kKvM,
+                       x.ptr, n_tokens) == 1;
+        },
+        {}};
+    return benchmark_pair_arms(
+        "pair", n_tokens, kDenseK, kDenseM + kKvM,
+        static_cast<uint64_t>(kDenseK) * (kDenseM + kKvM), 0u,
+        cfg, baseline, candidate,
+        [&]() {
+            return poison_output(separate0.ptr, out0_bytes, 0x7fc10001u,
+                                 0x4000u) &&
+                   poison_output(separate1.ptr, out1_bytes, 0x7fc20002u,
+                                 0x5000u) &&
+                   poison_output(pair0.ptr, out0_bytes, 0x7fc30003u,
+                                 0x6000u) &&
+                   poison_output(pair1.ptr, out1_bytes, 0x7fc40004u,
+                                 0x7000u);
+        },
+        [&](uint32_t set) {
+            return bitwise_equal(separate0.ptr, pair0.ptr, out0_bytes,
+                                 "pair q_a output") &&
+                   bitwise_equal(separate1.ptr, pair1.ptr, out1_bytes,
+                                 "pair kv output") &&
+                   output_is_finite(separate0.ptr, out0_bytes,
+                                    "pair q_a output") &&
+                   output_is_finite(separate1.ptr, out1_bytes,
+                                    "pair kv output") &&
+                   sampled_cpu_oracle(separate0.ptr, model,
+                                      model.weights[set].dense_offset,
+                                      activation, n_tokens, kDenseK, kDenseM,
+                                      "pair q_a") &&
+                   sampled_cpu_oracle(separate1.ptr, model,
+                                      model.weights[set].kv_offset,
+                                      activation, n_tokens, kDenseK, kKvM,
+                                      "pair kv") &&
+                   check_guard(x.ptr, x_bytes, 0x3000u, "pair input") &&
+                   check_guard(separate0.ptr, out0_bytes, 0x4000u,
+                               "pair separate q_a") &&
+                   check_guard(separate1.ptr, out1_bytes, 0x5000u,
+                               "pair separate kv") &&
+                   check_guard(pair0.ptr, out0_bytes, 0x6000u,
+                               "pair fused q_a") &&
+                   check_guard(pair1.ptr, out1_bytes, 0x7000u,
+                               "pair fused kv");
+        }) &&
+        check_guard(x.ptr, x_bytes, 0x3000u, "pair input final") &&
+        check_guard(separate0.ptr, out0_bytes, 0x4000u,
+                    "pair separate q_a final") &&
+        check_guard(separate1.ptr, out1_bytes, 0x5000u,
+                    "pair separate kv final") &&
+        check_guard(pair0.ptr, out0_bytes, 0x6000u,
+                    "pair fused q_a final") &&
+        check_guard(pair1.ptr, out1_bytes, 0x7000u,
+                    "pair fused kv final");
+}
+
+bool run_qb(const model_fixture &model, const config &cfg,
+            uint32_t n_tokens) {
+    uint64_t out_elements = 0;
+    if (!checked_mul(n_tokens, kQbM, &out_elements)) return false;
+    const uint64_t x_bytes = static_cast<uint64_t>(n_tokens) * kQbK *
+                             sizeof(float);
+    const uint64_t out_bytes = out_elements * sizeof(float);
+    const uint64_t guard_bytes = kGuardWords * sizeof(uint32_t);
+    tensor_owner x(x_bytes + guard_bytes);
+    tensor_owner output(out_bytes + guard_bytes);
+    std::vector<float> activation;
+    fill_activation(&activation, n_tokens, kQbK);
+    if (!x.ptr || !output.ptr ||
+        !ds4_gpu_tensor_write(x.ptr, 0, activation.data(), x_bytes) ||
+        !prepare_guard(x.ptr, x_bytes, 0x8000u)) {
+        std::fprintf(stderr, "q_b N=%u: tensor setup failed\n", n_tokens);
+        return false;
+    }
+    const arm path = {
+        cfg.path == cuda_path::mmq ? "mmq" : "legacy_q8k",
+        [&](uint32_t set) {
+            return ds4_gpu_matmul_quant_tensor(
+                       output.ptr, model.data, model.size,
+                       model.weights[set].qb_offset, kQ4Type, kQbK, kQbM,
+                       x.ptr, n_tokens) != 0;
+        },
+        {}};
+    return benchmark_single_path(
+        "q_b", n_tokens, kQbK, kQbM, cfg, path,
+        [&]() {
+            return poison_output(output.ptr, out_bytes, 0x7fc10001u,
+                                 0x9000u);
+        },
+        [&](uint32_t set) {
+            return output_is_finite(output.ptr, out_bytes, "q_b output") &&
+                   sampled_cpu_oracle(output.ptr, model,
+                                      model.weights[set].qb_offset,
+                                      activation, n_tokens, kQbK, kQbM,
+                                      "q_b") &&
+                   check_guard(x.ptr, x_bytes, 0x8000u, "q_b input") &&
+                   check_guard(output.ptr, out_bytes, 0x9000u,
+                               "q_b output");
+        }) &&
+        check_guard(x.ptr, x_bytes, 0x8000u, "q_b input final") &&
+        check_guard(output.ptr, out_bytes, 0x9000u, "q_b output final");
+}
+
+bool run_output_a(const model_fixture &model, const config &cfg,
+                  uint32_t n_tokens) {
+    if (cfg.path == cuda_path::legacy) {
+        std::printf(
+            "DS4_CUDA_Q4_PREFILL_SKIP case=outa_plus_min_b path=legacy "
+            "N=%u reason=grouped_prefill_requires_mmq\n",
+            n_tokens);
+        std::fflush(stdout);
+        return true;
+    }
+
+    uint64_t heads_elements = 0;
+    uint64_t low_elements = 0;
+    uint64_t out_elements = 0;
+    if (!checked_mul(n_tokens,
+                     static_cast<uint64_t>(kOutputGroups) * kDenseK,
+                     &heads_elements) ||
+        !checked_mul(n_tokens, kOutputLowDim, &low_elements) ||
+        !checked_mul(n_tokens, kOutputMinB, &out_elements)) {
+        return false;
+    }
+    const uint64_t heads_bytes = heads_elements * sizeof(float);
+    const uint64_t low_bytes = low_elements * sizeof(float);
+    const uint64_t out_bytes = out_elements * sizeof(float);
+    const uint64_t group_tmp_bytes =
+        static_cast<uint64_t>(n_tokens) * kDenseK * sizeof(float);
+    const uint64_t low_tmp_bytes =
+        static_cast<uint64_t>(n_tokens) * kOutputRank * sizeof(float);
+    const uint64_t guard_bytes = kGuardWords * sizeof(uint32_t);
+    tensor_owner heads(heads_bytes + guard_bytes);
+    tensor_owner baseline_low(low_bytes + guard_bytes);
+    tensor_owner baseline_out(out_bytes + guard_bytes);
+    tensor_owner candidate_low(low_bytes + guard_bytes);
+    tensor_owner candidate_out(out_bytes + guard_bytes);
+    tensor_owner group_tmp(group_tmp_bytes + guard_bytes);
+    tensor_owner low_tmp(low_tmp_bytes + guard_bytes);
+    std::vector<float> activation;
+    fill_activation(&activation, n_tokens, kOutputGroups * kDenseK);
+    if (!heads.ptr || !baseline_low.ptr || !baseline_out.ptr ||
+        !candidate_low.ptr || !candidate_out.ptr || !group_tmp.ptr ||
+        !low_tmp.ptr ||
+        !ds4_gpu_tensor_write(heads.ptr, 0, activation.data(), heads_bytes) ||
+        !prepare_guard(heads.ptr, heads_bytes, 0xa000u)) {
+        std::fprintf(stderr,
+                     "outa_plus_min_b N=%u: tensor setup failed\n",
+                     n_tokens);
+        return false;
+    }
+
+    const bool compare_single_grid = cfg.grouped_single_grid;
+    const bool compare_q81_kernel = cfg.grouped_q81_kernel;
+    const arm baseline = {
+        compare_q81_kernel
+            ? "grouped_generic_q81"
+            : (compare_single_grid
+                   ? "grouped_8_grids" : "pack8_mmq_unpack"),
+        [&](uint32_t set) {
+            return ds4_gpu_attention_output_q4_K_batch_tensor(
+                       baseline_out.ptr, baseline_low.ptr, group_tmp.ptr,
+                       low_tmp.ptr, model.data, model.size,
+                       model.weights[set].output_a_offset,
+                       model.weights[set].output_b_offset, kQ4Type,
+                       kDenseK, kOutputRank, kOutputGroups, kOutputMinB,
+                       heads.ptr, n_tokens) > 0;
+        },
+        [=]() {
+            return compare_q81_kernel
+                ? select_grouped_q81_reference()
+                : (compare_single_grid
+                       ? select_grouped_prefill_grid8()
+                       : select_grouped_prefill_legacy());
+        }};
+    const arm candidate = {
+        compare_q81_kernel
+            ? "grouped_k4096_g8x2_q81"
+            : (compare_single_grid
+                   ? "grouped_single_grid" : "grouped_8_grids"),
+        [&](uint32_t set) {
+            return ds4_gpu_attention_output_q4_K_batch_tensor(
+                       candidate_out.ptr, candidate_low.ptr, group_tmp.ptr,
+                       low_tmp.ptr, model.data, model.size,
+                       model.weights[set].output_a_offset,
+                       model.weights[set].output_b_offset, kQ4Type,
+                       kDenseK, kOutputRank, kOutputGroups, kOutputMinB,
+                       heads.ptr, n_tokens) > 0;
+        },
+        [=]() {
+            return compare_q81_kernel
+                ? select_grouped_q81_candidate()
+                : (compare_single_grid
+                       ? select_grouped_prefill_single_grid()
+                       : select_grouped_prefill_grid8());
+        }};
+    const uint64_t output_a_macs_per_token =
+        static_cast<uint64_t>(kOutputGroups) * kDenseK * kOutputRank;
+    const uint64_t output_b_macs_per_token =
+        static_cast<uint64_t>(kOutputLowDim) * kOutputMinB;
+    const bool ok = benchmark_pair_arms(
+        "outa_plus_min_b", n_tokens, kDenseK, kOutputLowDim,
+        output_a_macs_per_token, output_b_macs_per_token, cfg,
+        baseline, candidate,
+        [&]() {
+            return poison_output(baseline_low.ptr, low_bytes, 0x7fc10001u,
+                                 0xb000u) &&
+                   poison_output(baseline_out.ptr, out_bytes, 0x7fc20002u,
+                                 0xc000u) &&
+                   poison_output(candidate_low.ptr, low_bytes, 0x7fc30003u,
+                                 0xd000u) &&
+                   poison_output(candidate_out.ptr, out_bytes, 0x7fc40004u,
+                                 0xe000u) &&
+                   prepare_guard(group_tmp.ptr, group_tmp_bytes, 0xf000u) &&
+                   prepare_guard(low_tmp.ptr, low_tmp_bytes, 0x11000u);
+        },
+        [&](uint32_t set) {
+            return bitwise_equal(baseline_low.ptr, candidate_low.ptr,
+                                 low_bytes,
+                                 compare_q81_kernel
+                                     ? "output_a specialized vs generic Q8_1"
+                                     : (compare_single_grid
+                                            ? "output_a single-grid vs eight-grid"
+                                            : "output_a grouped vs pack/unpack")) &&
+                   bitwise_equal(baseline_out.ptr, candidate_out.ptr,
+                                 out_bytes,
+                                 "output_a minimal-B final output") &&
+                   output_is_finite(baseline_low.ptr, low_bytes,
+                                    "output_a low") &&
+                   output_is_finite(baseline_out.ptr, out_bytes,
+                                    "output_a minimal-B output") &&
+                   sampled_grouped_cpu_oracle(
+                       baseline_low.ptr, model,
+                       model.weights[set].output_a_offset, activation,
+                       n_tokens, "output_a") &&
+                   check_guard(heads.ptr, heads_bytes, 0xa000u,
+                               "output_a heads") &&
+                   check_guard(baseline_low.ptr, low_bytes, 0xb000u,
+                               "output_a baseline low") &&
+                   check_guard(baseline_out.ptr, out_bytes, 0xc000u,
+                               "output_a baseline out") &&
+                   check_guard(candidate_low.ptr, low_bytes, 0xd000u,
+                               "output_a candidate low") &&
+                   check_guard(candidate_out.ptr, out_bytes, 0xe000u,
+                               "output_a candidate out") &&
+                   check_guard(group_tmp.ptr, group_tmp_bytes, 0xf000u,
+                               "output_a group scratch") &&
+                   check_guard(low_tmp.ptr, low_tmp_bytes, 0x11000u,
+                               "output_a low scratch");
+        });
+    return ok &&
+           check_guard(heads.ptr, heads_bytes, 0xa000u,
+                       "output_a heads final") &&
+           check_guard(baseline_low.ptr, low_bytes, 0xb000u,
+                       "output_a baseline low final") &&
+           check_guard(baseline_out.ptr, out_bytes, 0xc000u,
+                       "output_a baseline out final") &&
+           check_guard(candidate_low.ptr, low_bytes, 0xd000u,
+                       "output_a candidate low final") &&
+           check_guard(candidate_out.ptr, out_bytes, 0xe000u,
+                       "output_a candidate out final") &&
+           check_guard(group_tmp.ptr, group_tmp_bytes, 0xf000u,
+                       "output_a group scratch final") &&
+           check_guard(low_tmp.ptr, low_tmp_bytes, 0x11000u,
+                       "output_a low scratch final");
+}
+
+void usage(FILE *stream, const char *argv0) {
+    std::fprintf(
+        stream,
+        "usage: %s [options]\n\n"
+        "Resident CUDA Q4_K prefill kernel benchmark (CUDA event timing).\n\n"
+        "  --path mmq|legacy         process-wide path (default: mmq)\n"
+        "  --case all|dense|pair|qb|outa|outb\n"
+        "                             case to run (default: all)\n"
+        "  --tokens N[,N...]         token counts, each 9..8192\n"
+        "  --full                    use 9,16,17,31,32,33,127,128,129,256,"
+        "257,512,1024,2048,2049,4096,6144,8192\n"
+        "  --sets N                  rotating resident weight sets (default: %u)\n"
+        "  --samples N               samples/arm, multiple of 4 (default: %u)\n"
+        "  --warmup N                untimed dispatches/arm (default: %u)\n"
+        "  --grouped-single-grid     compare grouped 8-grid vs grid.z outa\n"
+        "  --grouped-q81-kernel      compare generic vs K4096/G8x2 Q8_1 outa\n"
+        "  --kernel-16warp           prequantized canonical-vs-16-warp A/B\n"
+        "  -h, --help                show this help\n\n"
+        "Dense and q_b measure one immutable process path. Run separate "
+        "legacy/MMQ\nprocesses (preferably ABBA/BAAB) to compare them because "
+        "the CUDA backend\ncaches DS4_CUDA_MMQ on its first dispatch. Pair is "
+        "an in-process ABBA/BAAB\ncomparison of two MMQ projections against "
+        "the fused public pair API. outa\ncompares the rollback eight-group "
+        "pack/MMQ/unpack sequence against the default\ndirect-strided grouped "
+        "path with one canonical MMQ grid per group. Add\n"
+        "--grouped-single-grid to compare that default with the experimental "
+        "grid.z\nsubmission instead. Add\n"
+        "--grouped-q81-kernel to isolate the default fixed-shape Q8_1 "
+        "front-end against\nthe canonical strided quantizer while retaining "
+        "the same eight MMQ grids. Both outa comparisons include a common minimal "
+        "Q4 output-B (M=256) whose MACs are reported separately. "
+        "DS4_CUDA_MMQ_X_MAX\nmay explicitly select an 8..128 multiple-of-8 "
+        "sweep point; the setup line\nattests it, or prints auto when the "
+        "variable is unset. --kernel-16warp requires\n--path mmq and "
+        "--case dense/pair/qb/outb; with --case all it also runs the real "
+        "K=8192,M=4096 output-B. Its "
+        "pair arm\nshares one prequantized activation across M=1024+512. Token "
+        "counts must be\n>=512 and select canonical m128n128 on the active "
+        "device; without --tokens/--full it uses "
+        "512,1024,2048,2049,4096,6144,8192.\n",
+        argv0, kDefaultSets, kDefaultSamples, kDefaultWarmup);
+}
+
+uint32_t parse_u32(const char *text, const char *option, uint32_t minimum,
+                   uint32_t maximum) {
+    char *end = nullptr;
+    errno = 0;
+    const unsigned long value = std::strtoul(text, &end, 10);
+    if (errno != 0 || !text[0] || !end || *end || value < minimum ||
+        value > maximum) {
+        std::fprintf(stderr, "invalid %s: %s\n", option, text);
+        std::exit(2);
+    }
+    return static_cast<uint32_t>(value);
+}
+
+bool env_value_enabled(const char *name) {
+    const char *value = std::getenv(name);
+    return value && value[0] && !(value[0] == '0' && value[1] == '\0');
+}
+
+const char *need_value(int *index, int argc, char **argv) {
+    if (*index + 1 >= argc) {
+        std::fprintf(stderr, "%s requires a value\n", argv[*index]);
+        std::exit(2);
+    }
+    return argv[++*index];
+}
+
+std::vector<uint32_t> parse_tokens(const char *text) {
+    std::vector<uint32_t> result;
+    const char *cursor = text;
+    while (*cursor) {
+        const char *comma = std::strchr(cursor, ',');
+        const std::string item(cursor,
+                               comma ? static_cast<size_t>(comma - cursor)
+                                     : std::strlen(cursor));
+        result.push_back(parse_u32(item.c_str(), "--tokens", 9u, 8192u));
+        if (!comma) break;
+        cursor = comma + 1;
+        if (!*cursor) {
+            std::fprintf(stderr, "invalid --tokens: trailing comma\n");
+            std::exit(2);
+        }
+    }
+    if (result.empty()) {
+        std::fprintf(stderr, "--tokens cannot be empty\n");
+        std::exit(2);
+    }
+    std::sort(result.begin(), result.end());
+    result.erase(std::unique(result.begin(), result.end()), result.end());
+    return result;
+}
+
+config parse_options(int argc, char **argv) {
+    config cfg;
+    bool tokens_explicit = false;
+    for (int i = 1; i < argc; i++) {
+        if (!std::strcmp(argv[i], "-h") || !std::strcmp(argv[i], "--help")) {
+            usage(stdout, argv[0]);
+            std::exit(0);
+        } else if (!std::strcmp(argv[i], "--path")) {
+            const char *value = need_value(&i, argc, argv);
+            if (!std::strcmp(value, "mmq")) cfg.path = cuda_path::mmq;
+            else if (!std::strcmp(value, "legacy")) {
+                cfg.path = cuda_path::legacy;
+            } else {
+                std::fprintf(stderr, "invalid --path: %s\n", value);
+                std::exit(2);
+            }
+        } else if (!std::strcmp(argv[i], "--case")) {
+            const char *value = need_value(&i, argc, argv);
+            if (!std::strcmp(value, "all")) cfg.selected = bench_case::all;
+            else if (!std::strcmp(value, "dense")) {
+                cfg.selected = bench_case::dense;
+            } else if (!std::strcmp(value, "pair")) {
+                cfg.selected = bench_case::pair;
+            } else if (!std::strcmp(value, "qb")) {
+                cfg.selected = bench_case::qb;
+            } else if (!std::strcmp(value, "outa")) {
+                cfg.selected = bench_case::outa;
+            } else if (!std::strcmp(value, "outb")) {
+                cfg.selected = bench_case::outb;
+            } else {
+                std::fprintf(stderr, "invalid --case: %s\n", value);
+                std::exit(2);
+            }
+        } else if (!std::strcmp(argv[i], "--tokens")) {
+            cfg.tokens = parse_tokens(need_value(&i, argc, argv));
+            tokens_explicit = true;
+        } else if (!std::strcmp(argv[i], "--full")) {
+            cfg.tokens = {9u,   16u,  17u,  31u,  32u,  33u, 127u,
+                          128u, 129u, 256u, 257u, 512u, 1024u, 2048u,
+                          2049u, 4096u, 6144u, 8192u};
+            tokens_explicit = true;
+        } else if (!std::strcmp(argv[i], "--sets")) {
+            cfg.sets = parse_u32(need_value(&i, argc, argv), "--sets", 1u,
+                                 32u);
+        } else if (!std::strcmp(argv[i], "--samples")) {
+            cfg.samples = parse_u32(need_value(&i, argc, argv), "--samples",
+                                    4u, 1000u);
+        } else if (!std::strcmp(argv[i], "--warmup")) {
+            cfg.warmup = parse_u32(need_value(&i, argc, argv), "--warmup",
+                                   0u, 100u);
+        } else if (!std::strcmp(argv[i], "--kernel-16warp")) {
+            cfg.kernel_16warp = true;
+        } else if (!std::strcmp(argv[i], "--grouped-single-grid")) {
+            cfg.grouped_single_grid = true;
+        } else if (!std::strcmp(argv[i], "--grouped-q81-kernel")) {
+            cfg.grouped_q81_kernel = true;
+        } else {
+            std::fprintf(stderr, "unknown option: %s\n", argv[i]);
+            usage(stderr, argv[0]);
+            std::exit(2);
+        }
+    }
+    if ((cfg.samples % 4u) != 0u) {
+        std::fprintf(stderr,
+                     "--samples must be a multiple of 4 for balanced runs\n");
+        std::exit(2);
+    }
+    if (cfg.kernel_16warp && !tokens_explicit) {
+        cfg.tokens = {512u, 1024u, 2048u, 2049u, 4096u, 6144u, 8192u};
+    }
+    if (cfg.grouped_q81_kernel && !tokens_explicit) {
+        cfg.tokens = {512u, 1024u, 2048u, 4096u, 6144u, 8192u};
+    }
+    if (cfg.kernel_16warp) {
+        const auto below_minimum = std::find_if(
+            cfg.tokens.begin(), cfg.tokens.end(),
+            [](uint32_t value) { return value < 512u; });
+        if (below_minimum != cfg.tokens.end()) {
+            std::fprintf(stderr,
+                         "--kernel-16warp requires every token count to be "
+                         ">=512 (got %u)\n",
+                         *below_minimum);
+            std::exit(2);
+        }
+    }
+    if (cfg.kernel_16warp && cfg.path != cuda_path::mmq) {
+        std::fprintf(stderr,
+                     "--kernel-16warp requires --path mmq\n");
+        std::exit(2);
+    }
+    if (cfg.grouped_single_grid && cfg.path != cuda_path::mmq) {
+        std::fprintf(stderr,
+                     "--grouped-single-grid requires --path mmq\n");
+        std::exit(2);
+    }
+    if (cfg.grouped_q81_kernel && cfg.path != cuda_path::mmq) {
+        std::fprintf(stderr,
+                     "--grouped-q81-kernel requires --path mmq\n");
+        std::exit(2);
+    }
+    if (cfg.grouped_single_grid && cfg.kernel_16warp) {
+        std::fprintf(stderr,
+                     "--grouped-single-grid cannot be combined with "
+                     "--kernel-16warp\n");
+        std::exit(2);
+    }
+    if (cfg.grouped_q81_kernel &&
+        (cfg.grouped_single_grid || cfg.kernel_16warp)) {
+        std::fprintf(stderr,
+                     "--grouped-q81-kernel cannot be combined with "
+                     "--grouped-single-grid or --kernel-16warp\n");
+        std::exit(2);
+    }
+    if (cfg.grouped_single_grid && cfg.selected != bench_case::all &&
+        cfg.selected != bench_case::outa) {
+        std::fprintf(stderr,
+                     "--grouped-single-grid requires --case outa or all\n");
+        std::exit(2);
+    }
+    if (cfg.grouped_q81_kernel && cfg.selected != bench_case::all &&
+        cfg.selected != bench_case::outa) {
+        std::fprintf(stderr,
+                     "--grouped-q81-kernel requires --case outa or all\n");
+        std::exit(2);
+    }
+    if (cfg.kernel_16warp && cfg.selected == bench_case::outa) {
+        std::fprintf(stderr,
+                     "--kernel-16warp supports only --case dense, pair, qb, "
+                     "outb, or all\n");
+        std::exit(2);
+    }
+    if (!cfg.kernel_16warp && cfg.selected == bench_case::outb) {
+        std::fprintf(stderr,
+                     "--case outb requires --kernel-16warp\n");
+        std::exit(2);
+    }
+    if (cfg.path == cuda_path::legacy &&
+        (cfg.selected == bench_case::pair ||
+         cfg.selected == bench_case::outa)) {
+        std::fprintf(stderr,
+                     "--case pair/outa requires --path mmq for prefill "
+                     "N > 8\n");
+        std::exit(2);
+    }
+    return cfg;
+}
+
+bool includes(bench_case selected, bench_case wanted) {
+    return selected == bench_case::all || selected == wanted;
+}
+
+std::string mmq_x_max_attestation(bool require_m128n128) {
+    const char *value = std::getenv("DS4_CUDA_MMQ_X_MAX");
+    if (!value || !value[0]) return "auto";
+    const uint32_t parsed =
+        parse_u32(value, "DS4_CUDA_MMQ_X_MAX", 8u, 128u);
+    if ((parsed % 8u) != 0u) {
+        std::fprintf(stderr,
+                     "invalid DS4_CUDA_MMQ_X_MAX: %s (must be a multiple "
+                     "of 8)\n",
+                     value);
+        std::exit(2);
+    }
+    if (require_m128n128 && parsed != 128u) {
+        std::fprintf(stderr,
+                     "--kernel-16warp requires DS4_CUDA_MMQ_X_MAX=128 "
+                     "when the variable is set (got %s)\n",
+                     value);
+        std::exit(2);
+    }
+    return std::to_string(parsed);
+}
+
+bool install_resident_model(const model_fixture &model,
+                            size_t *resident_delta,
+                            bool *resident_delta_valid) {
+    size_t free_before = 0, total_before = 0;
+    size_t free_after = 0, total_after = 0;
+    const bool have_before =
+        cudaMemGetInfo(&free_before, &total_before) == cudaSuccess;
+    if (!have_before) (void)cudaGetLastError();
+    if (!ds4_gpu_set_model_map(model.data, model.size) ||
+        !ds4_gpu_synchronize()) {
+        return false;
+    }
+    const bool have_after =
+        cudaMemGetInfo(&free_after, &total_after) == cudaSuccess;
+    if (!have_after) (void)cudaGetLastError();
+    (void)total_before;
+    (void)total_after;
+    *resident_delta_valid = have_before && have_after;
+    *resident_delta = *resident_delta_valid && free_before >= free_after
+        ? free_before - free_after : 0u;
+    return true;
+}
+
+bool verify_resident_weight_ranges(const model_fixture &model) {
+    const uint64_t dense_bytes = q4_weight_bytes(kDenseK, kDenseM);
+    const uint64_t kv_bytes = q4_weight_bytes(kDenseK, kKvM);
+    const uint64_t qb_bytes = q4_weight_bytes(kQbK, kQbM);
+    const uint64_t output_a_bytes =
+        q4_weight_bytes(kDenseK, kOutputLowDim);
+    const uint64_t output_b_bytes =
+        q4_weight_bytes(kOutputLowDim, model.output_b_rows);
+    for (uint32_t set = 0; set < model.weights.size(); set++) {
+        struct range_desc {
+            const char *name;
+            uint64_t offset;
+            uint64_t bytes;
+        };
+        const range_desc ranges[] = {
+            {"dense", model.weights[set].dense_offset, dense_bytes},
+            {"kv", model.weights[set].kv_offset, kv_bytes},
+            {"q_b", model.weights[set].qb_offset, qb_bytes},
+            {"output_a", model.weights[set].output_a_offset, output_a_bytes},
+            {model.output_b_rows == kOutputMinB
+                 ? "output_b_min" : "output_b",
+             model.weights[set].output_b_offset,
+             output_b_bytes},
+        };
+        for (const range_desc &range : ranges) {
+            if (!ds4_cuda_test_model_range_is_device_resident(
+                    model.data, model.size, range.offset, range.bytes, 0)) {
+                std::fprintf(
+                    stderr,
+                    "cuda-q4-prefill-bench: nonresident %s weight range "
+                    "set=%u offset=%llu bytes=%llu\n",
+                    range.name, set,
+                    static_cast<unsigned long long>(range.offset),
+                    static_cast<unsigned long long>(range.bytes));
+                return false;
+            }
+        }
+    }
+    return true;
+}
+
+bool verify_mmq_prefill_dispatch(const model_fixture &model) {
+    // For N > 8 the public pair API succeeds only through MMQ. Non-required
+    // production runs use this small probe to initialize and attest the
+    // process-wide decision. The caller skips it for raw-kernel and required
+    // 16-warp runs, whose measured dispatches provide their own fail-closed
+    // proof at an eligible shape.
+    constexpr uint32_t n_tokens = 9u;
+    const uint64_t x_bytes = static_cast<uint64_t>(n_tokens) * kDenseK *
+                             sizeof(float);
+    const uint64_t out0_bytes = static_cast<uint64_t>(n_tokens) * kDenseM *
+                                sizeof(float);
+    const uint64_t out1_bytes = static_cast<uint64_t>(n_tokens) * kKvM *
+                                sizeof(float);
+    tensor_owner x(x_bytes);
+    tensor_owner out0(out0_bytes);
+    tensor_owner out1(out1_bytes);
+    std::vector<float> activation;
+    fill_activation(&activation, n_tokens, kDenseK);
+    return x.ptr && out0.ptr && out1.ptr &&
+           ds4_gpu_tensor_write(x.ptr, 0, activation.data(), x_bytes) &&
+           ds4_gpu_matmul_q4_K_pair_tensor(
+               out0.ptr, out1.ptr, model.data, model.size,
+               model.weights[0].dense_offset, model.weights[0].kv_offset,
+               kDenseK, kDenseM, kKvM, x.ptr, n_tokens) == 1 &&
+           ds4_gpu_synchronize();
+}
+
+}  // namespace
+
+int main(int argc, char **argv) {
+    const config cfg = parse_options(argc, argv);
+    // get_mmq_x_max_host() caches this process-wide on its first call.  Read
+    // and validate the inherited sweep request before backend initialization,
+    // then attest it in the setup record instead of silently contaminating a
+    // supposedly default run.
+    const std::string mmq_x_max =
+        mmq_x_max_attestation(cfg.kernel_16warp);
+    env_snapshot mmq_guard("DS4_CUDA_MMQ");
+    env_snapshot copy_guard("DS4_CUDA_COPY_MODEL");
+    env_snapshot pair_guard("DS4_CUDA_DISABLE_Q4_DENSE_PAIR");
+    env_snapshot graph_guard("DS4_CUDA_DECODE_GRAPHS");
+    env_snapshot grouped_enable_guard(kGroupedPrefillEnable);
+    env_snapshot grouped_disable_guard(kGroupedPrefillDisable);
+    env_snapshot grouped_require_guard(kGroupedPrefillRequire);
+    env_snapshot grouped_single_grid_enable_guard(kGroupedSingleGridEnable);
+    env_snapshot grouped_single_grid_disable_guard(kGroupedSingleGridDisable);
+    env_snapshot grouped_single_grid_require_guard(kGroupedSingleGridRequire);
+    env_snapshot grouped_q81_disable_guard(kGroupedQ81Disable);
+    env_snapshot grouped_q81_require_guard(kGroupedQ81Require);
+    env_snapshot grouped_global_disable_guard(kGroupedGlobalDisable);
+    env_snapshot gb10_global_disable_guard(kGb10GlobalDisable);
+    if (setenv("DS4_CUDA_MMQ",
+               cfg.path == cuda_path::mmq ? "1" : "0", 1) != 0 ||
+        setenv("DS4_CUDA_COPY_MODEL", "1", 1) != 0 ||
+        setenv("DS4_CUDA_DECODE_GRAPHS", "0", 1) != 0 ||
+        unsetenv("DS4_CUDA_DISABLE_Q4_DENSE_PAIR") != 0 ||
+        unsetenv(kGroupedPrefillEnable) != 0 ||
+        unsetenv(kGroupedPrefillDisable) != 0 ||
+        unsetenv(kGroupedPrefillRequire) != 0 ||
+        unsetenv(kGroupedSingleGridEnable) != 0 ||
+        unsetenv(kGroupedSingleGridDisable) != 0 ||
+        unsetenv(kGroupedSingleGridRequire) != 0 ||
+        unsetenv(kGroupedQ81Disable) != 0 ||
+        unsetenv(kGroupedQ81Require) != 0 ||
+        unsetenv(kGroupedGlobalDisable) != 0 ||
+        unsetenv(kGb10GlobalDisable) != 0) {
+        std::fprintf(stderr,
+                     "cuda-q4-prefill-bench: environment setup failed\n");
+        return 1;
+    }
+
+    int device_count = 0;
+    const cudaError_t count_rc = cudaGetDeviceCount(&device_count);
+    if (count_rc != cudaSuccess || device_count <= 0) {
+        std::fprintf(stderr,
+                     "cuda-q4-prefill-bench: no visible CUDA device (%s)\n",
+                     count_rc == cudaSuccess ? "device count is zero"
+                                             : cudaGetErrorString(count_rc));
+        return 77;
+    }
+    cudaDeviceProp properties{};
+    if (cudaGetDeviceProperties(&properties, 0) != cudaSuccess) {
+        std::fprintf(stderr,
+                     "cuda-q4-prefill-bench: cannot query device properties\n");
+        return 1;
+    }
+    if (cfg.kernel_16warp && device_count != 1) {
+        std::fprintf(stderr,
+                     "cuda-q4-prefill-bench: SKIP (--kernel-16warp "
+                     "requires exactly one visible CUDA device)\n");
+        return 77;
+    }
+    const bool grouped_prefill_supported =
+        device_count == 1 && properties.major == 12 &&
+        properties.minor == 1 && properties.warpSize == 32;
+    if (cfg.selected == bench_case::outa &&
+        !grouped_prefill_supported) {
+        std::fprintf(stderr,
+                     "cuda-q4-prefill-bench: SKIP (outa requires one "
+                     "GB10/sm_121 device)\n");
+        return 77;
+    }
+    if (!ds4_gpu_init()) {
+        std::fprintf(stderr, "cuda-q4-prefill-bench: ds4_gpu_init failed\n");
+        return 1;
+    }
+    if (cfg.kernel_16warp) {
+        const int prepare_rc = ds4_mmq_q4_K_dense_16warp_prepare();
+        if (prepare_rc != 0) {
+            std::fprintf(
+                stderr,
+                "cuda-q4-prefill-bench: --kernel-16warp prepare failed "
+                "rc=%d\n",
+                prepare_rc);
+            ds4_gpu_cleanup();
+            return 1;
+        }
+        for (uint32_t tokens : cfg.tokens) {
+            if (!ds4_mmq_q4_K_dense_preq_reference_m128n128_for_test(
+                    static_cast<int>(tokens))) {
+                std::fprintf(
+                    stderr,
+                    "cuda-q4-prefill-bench: SKIP (--kernel-16warp N=%u "
+                    "does not select canonical m128n128 on this device)\n",
+                    tokens);
+                ds4_gpu_cleanup();
+                return 77;
+            }
+        }
+    }
+    ds4_cuda_test_set_q4_mmq_strict(
+        cfg.path == cuda_path::mmq ? 1 : 0);
+
+    bool ok = true;
+    model_fixture model;
+    const uint32_t output_b_rows =
+        cfg.kernel_16warp && includes(cfg.selected, bench_case::outb)
+            ? kOutputM : kOutputMinB;
+    if (!make_model(&model, cfg.sets, output_b_rows)) {
+        std::fprintf(stderr,
+                     "cuda-q4-prefill-bench: model fixture allocation failed\n");
+        ok = false;
+    }
+    size_t resident_delta = 0;
+    bool resident_delta_valid = false;
+    if (ok) {
+        ds4_gpu_set_quality(false);
+        ds4_gpu_set_ssd_streaming(false);
+        if (!install_resident_model(
+                model, &resident_delta, &resident_delta_valid)) {
+            std::fprintf(
+                stderr,
+                "cuda-q4-prefill-bench: model-map installation failed\n");
+            ok = false;
+        } else if (!verify_resident_weight_ranges(model)) {
+            std::fprintf(
+                stderr,
+                "cuda-q4-prefill-bench: explicit CUDA model provenance "
+                "check failed; refusing PCIe/HMM-contaminated timings\n");
+            ok = false;
+        }
+    }
+    // The N=9 probe is intentionally outside the 16-warp admission envelope.
+    // Kernel-only runs prove their raw path directly; required production runs
+    // fail closed at each measured dispatch, so probing here would be a false
+    // failure before the requested shape is reached.
+    const bool skip_mmq_probe = cfg.kernel_16warp || env_value_enabled(
+        "DS4_CUDA_REQUIRE_Q4_MMQ_16WARP");
+    if (ok && cfg.path == cuda_path::mmq && !skip_mmq_probe &&
+        !verify_mmq_prefill_dispatch(model)) {
+        std::fprintf(stderr,
+                     "cuda-q4-prefill-bench: MMQ prefill proof probe failed; "
+                     "refusing to label fallback timings as MMQ\n");
+        ok = false;
+    }
+
+    if (ok) {
+        std::printf(
+            "DS4_CUDA_Q4_PREFILL_SETUP device=%s cc=%d.%d warp=%d path=%s "
+            "mmq_x_max=%s sets=%u resident_payload_mib=%.2f "
+            "device_free_delta_mib=%.2f "
+            "device_free_delta_valid=%d timing=%s kernel_16warp=%d "
+            "grouped_q81_kernel=%d "
+            "cases=%s "
+            "ssd_streaming=off model_storage=cudaMalloc "
+            "residency=backend_provenance strict_mmq=%d "
+            "grouped_attn_a_prefill=%s grouped_attn_a_ab=%s "
+            "dispatch_stream=legacy_default\n",
+            properties.name, properties.major, properties.minor,
+            properties.warpSize, path_name(cfg.path), mmq_x_max.c_str(),
+            cfg.sets,
+            static_cast<double>(model.payload_bytes) / 1048576.0,
+            static_cast<double>(resident_delta) / 1048576.0,
+            resident_delta_valid ? 1 : 0,
+            cfg.kernel_16warp ? "kernel_only_prequant" : "cuda_events",
+            cfg.kernel_16warp ? 1 : 0,
+            cfg.grouped_q81_kernel ? 1 : 0,
+            case_scope(cfg),
+            cfg.path == cuda_path::mmq ? 1 : 0,
+            grouped_prefill_supported ? "available" : "skipped",
+            cfg.grouped_q81_kernel
+                ? "q81_generic_vs_k4096_g8x2"
+                : (cfg.grouped_single_grid
+                       ? "grid8_vs_single_grid" : "pack8_vs_grid8"));
+        std::fflush(stdout);
+        for (uint32_t n_tokens : cfg.tokens) {
+            if (includes(cfg.selected, bench_case::dense)) {
+                ok = (cfg.kernel_16warp
+                          ? run_q4_16warp_kernel(
+                                model, cfg, n_tokens, kDenseK, kDenseM,
+                                &weight_set::dense_offset, "dense")
+                          : run_dense(model, cfg, n_tokens)) && ok;
+            }
+            if (ok && includes(cfg.selected, bench_case::pair)) {
+                ok = (cfg.kernel_16warp
+                          ? run_q4_16warp_pair_kernel(
+                                model, cfg, n_tokens)
+                          : run_pair(model, cfg, n_tokens)) && ok;
+            }
+            if (ok && includes(cfg.selected, bench_case::qb)) {
+                ok = (cfg.kernel_16warp
+                          ? run_q4_16warp_kernel(
+                                model, cfg, n_tokens, kQbK, kQbM,
+                                &weight_set::qb_offset, "q_b")
+                          : run_qb(model, cfg, n_tokens)) && ok;
+            }
+            if (ok && cfg.kernel_16warp &&
+                includes(cfg.selected, bench_case::outb)) {
+                ok = run_q4_16warp_kernel(
+                         model, cfg, n_tokens, kOutputLowDim, kOutputM,
+                         &weight_set::output_b_offset, "output_b") && ok;
+            }
+            if (ok && !cfg.kernel_16warp && grouped_prefill_supported &&
+                includes(cfg.selected, bench_case::outa)) {
+                ok = run_output_a(model, cfg, n_tokens) && ok;
+            }
+            if (!ok) break;
+        }
+    }
+
+    ds4_gpu_cleanup();
+    std::fprintf(stderr, "cuda-q4-prefill-bench: %s\n", ok ? "PASS" : "FAIL");
+    return ok ? 0 : 1;
+}
diff --git a/speed-bench/ds4_gb10_q2_cuda_port_results.md b/speed-bench/ds4_gb10_q2_cuda_port_results.md
new file mode 100644
index 0000000000..18d0cf4c39
--- /dev/null
+++ b/speed-bench/ds4_gb10_q2_cuda_port_results.md
@@ -0,0 +1,324 @@
+# GB10 Q2 CUDA port results
+
+This records the Q2 CUDA baseline and the exact kernel work promoted for the
+NVIDIA GB10 / DGX Spark path. The starting tree was `b030961` (`main`) and the
+model was:
+
+```text
+gguf/DeepSeek-V4-Flash-IQ2XXS-w2Q2K-AProjQ8-SExpQ8-OutQ8-chat-v2-imatrix-0731.gguf
+```
+
+The test machine was an NVIDIA GB10 (`sm_121`) with nominal 128 GB unified
+memory, driver 580.173.02, and CUDA 13.0. Builds used:
+
+```sh
+make cuda-spark
+```
+
+## Branch audit and selected work
+
+`origin/mxfp4-m3` is 24 commits ahead of the starting tree, but its net CUDA
+backend delta is only four compatibility lines. Its throughput work is in
+Metal. The MXFP4 LUT/tile kernels are not directly applicable to this
+IQ2_XXS/Q2_K model, and CUDA already has equivalents for several exact Metal
+fusions (router top-k/weights, Q/KV RMS+RoPE, compressor pool math, and HC
+split/weighted-sum/norm).
+
+The two useful remaining ideas were removal of intermediate materialization and
+an exact producer/store epilogue:
+
+1. **Direct Q2 prefill.** The existing dormant direct D2R MMQ now has a
+   production dispatch. IQ2 gate/up accumulators remain in registers and
+   weighted SwiGLU is quantized directly to Q8_1 for the Q2_K down MMQ. It
+   avoids materializing F32 gate, up, and mid tensors.
+2. **F16 compressor projection/store.** The established ordered F16 pair
+   matvec now optionally writes the compressor state in lane 0 of the same
+   kernel. This removes 62 dependent state-store launches per Flash decode
+   token (42 ratio-4 attention/indexer calls and 20 ratio-128 attention calls).
+
+Both defaults are limited to GB10. The direct path has shape, residency,
+aligned-artifact, streaming, scratch, and token-count fallbacks. Its public MMQ
+contract returns `DS4_MMQ_NOT_APPLICABLE` only before any enqueue; negative
+results are never retried. It is disabled while graph intermediates are being
+dumped because its scratch intentionally overwrites the otherwise-dead
+gate/up/mid buffers. The compressor fusion is limited to the three validated
+4096-wide Flash/Pro calls: `(ratio,width) = (4,256), (4,1024), (128,512)`.
+
+Rollback switches are:
+
+```text
+DS4_CUDA_NO_DIRECT_Q2_PREFILL=1
+DS4_CUDA_NO_F16_PAIR_COMPRESSOR_STORE=1
+```
+
+The existing ordered-pair rollback switches also dominate the compressor
+fusion.
+
+Two experiments were not promoted: exact score split DIM2 was neutral (808.41
+prefill tok/s in its initial 2K screen), and a QKV RoPE/FP8/cache-store
+prototype preserved output but did not provide a worthwhile speedup.
+
+## Reproducible baseline
+
+The original `b030961` benchmark binary was retained on the author's test
+host at `/tmp/ds4-bench-main-baseline` with SHA-256:
+
+```text
+6cf857252c1dbbc64add2b94857b3be568615c6e6e863961d4df938ad9f3ec98
+```
+
+Four interleaved 2,048-token runs of the ordinary path used
+`speed-bench/promessi_sposi.txt`:
+
+| run | prefill tok/s | decode tok/s | steady decode tok/s |
+|---:|---:|---:|---:|
+| 1 | 810.26 | 17.78 | 17.93 |
+| 2 | 809.01 | 17.78 | 17.94 |
+| 3 | 815.58 | 17.73 | 17.89 |
+| 4 | 807.88 | 17.61 | 17.77 |
+| **mean** | **810.6825** | **17.725** | **17.8825** |
+
+## Performance
+
+The final aggregate was measured in ABBA order with 2,048 prefill tokens and
+512 generated tokens. The control set both rollback variables; the optimized
+arm unset both. The per-run command body was:
+
+```sh
+./ds4-bench -m ds4flash.gguf --cuda \
+  --prompt-file speed-bench/promessi_sposi.txt \
+  --ctx-start 2048 --ctx-max 2048 --step-incr 2048 \
+  --gen-tokens 512 --csv OUT.csv
+```
+
+| arm/run | prefill tok/s | decode tok/s | steady decode tok/s |
+|---|---:|---:|---:|
+| rollback 1 | 808.86 | 17.66 | 17.78 |
+| default 1 | 829.89 | 17.85 | 17.97 |
+| default 2 | 829.70 | 17.84 | 17.96 |
+| rollback 2 | 805.29 | 17.48 | 17.60 |
+| **rollback mean** | **807.075** | **17.570** | **17.690** |
+| **default mean** | **829.795** | **17.845** | **17.965** |
+| **gain** | **+2.82%** | **+1.57%** | **+1.55%** |
+
+Isolated screens agree with the attribution:
+
+- Direct prefill: 830.6275 versus 810.6825 tok/s over four interleaved runs,
+  **+2.46%**; decode was neutral (+0.06%).
+- Compressor/store: 17.935 versus 17.765 decode tok/s and 18.055 versus 17.885
+  steady tok/s over a 512-token ABBA screen, **+0.96% / +0.95%**.
+
+Author-local raw artifacts are under `/tmp/ds4-opt/`, notably `final-abba/`,
+`final-correct/`, `abba/`, and `compstore/abba/`; these paths are provenance
+notes, not required to reproduce the checked-in CSVs.
+
+## Exactness and safety validation
+
+Frontier hashes alone were not used as the decode criterion.
+
+- A 2,782-token prompt with a 128-token greedy decode limit (56 tokens
+  emitted through EOS) produced byte-identical stdout and full per-token
+  logprob JSON with both defaults on versus both rollbacks. SHA-256:
+  `6519499391d81b625344a8335ba10fde02d1a2c4e414eb65e6ee711fa3b37d14`.
+- The isolated compressor A/B generated-token/logprob artifact is
+  byte-identical with SHA-256
+  `9b76a6b7579a192d0aeb04bb48616944313cee805f7cbba158c99655ad657868`.
+- The direct prefill frontier JSON was fully parsed and byte-identical,
+  including every logit and selected ID, with SHA-256
+  `e7b419e8ebbcb6c40a5eccfe8784645d910dbf203df7960500d4d8d133f061a8`.
+- `DS4_MMQ_YIND_VERIFY=1` checked all 43 routed layers at 2,048 tokens; every
+  layer reported `bad=0/393216` Q8 staging values.
+- Small/ragged batches fall back normally; disabling D2R and ordered pair MMQ
+  together also completed without an aligned-path error.
+- `compute-sanitizer --tool memcheck` completed the fused decode path with
+  `ERROR SUMMARY: 0 errors`.
+- `make cuda-regression` passed its long-context/top-k smoke.
+- `make test CUDA_ARCH=sm_121` passed in full. This included a 30,474-token
+  long-context run (direct batches at 4,096 and the final ragged tail), five
+  logprob vectors, exact tensor-equivalence vectors, local golden vectors,
+  server tests, sampling tests, and all ordinary unit tests.
+- Two independent source/safety reviews found no release blocker after the
+  scratch-overflow, tri-state, debug, target-admission, and fallback-contract
+  guards were applied.
+
+Nsight Compute hardware counters were unavailable on this system because
+`RmProfilingAdminOnly=1`; performance conclusions therefore use controlled
+wall-clock ABBA measurements plus exact output and sanitizer validation.
+
+## Decode phase: approximately 20 steady tok/s
+
+A second profiling/optimization pass targeted the remaining decode path. Nsight
+Systems showed that the token was dominated by exact, bandwidth-bound Q8 and
+compressor projections. Four changes were promoted:
+
+1. **No CUDA mid-token split synchronization.** Metal's four-layer split
+   asynchronously commits a command buffer, but the CUDA implementation of the
+   same flush calls `cudaDeviceSynchronize()`. CUDA now defaults the split to
+   zero; Apple, ROCm, and CPU builds retain four. The existing
+   `DS4_METAL_GRAPH_TOKEN_SPLIT_LAYERS` override remains available.
+2. **Coalesced exact F16 compressor pairs.** On first use, each validated
+   compressor KV/score weight pair is repacked into an interleaved chunk-32
+   layout. Every lane still accumulates its original contiguous 128 values and
+   lane 0 performs the same ordered sum, but weights at a given iteration are
+   coalesced. An eight-value load prefetch preserves the arithmetic chain.
+   The transpose is enabled only on the validated single-GB10 path. The cache
+   is device-qualified, released on map change/cleanup, refuses graph capture
+   allocation, and permanently falls back after an allocation failure.
+3. **Aligned Q8 fused consumers.** Decode Q8 pair, HC-expand, and grouped
+   attention-A kernels now consume the already-built aligned Q8 artifacts
+   directly instead of returning to misaligned raw 34-byte blocks. DP4A term
+   order, per-lane accumulation, warp reduction, and all epilogues are
+   unchanged. Admission is limited to one validated GB10, exact aligned
+   artifacts, full tensors, and dimensions divisible by the proven tile sizes;
+   multi-GPU and unsupported shapes fall back to the raw kernels.
+4. **Persistent aligned Q8 projections.** The K=1024 Q-b projection and K=4096
+   vocabulary projection use eight row warps per persistent CTA on one GB10. The
+   K=1024 kernel hoists the immutable activation into registers. Lane/block
+   assignment and the float warp tree remain identical; integer dot regrouping
+   is overflow-safe. The generic aligned kernel remains the fallback.
+
+New CUDA rollback switches are:
+
+```text
+DS4_CUDA_NO_F16_PAIR_COMPRESSOR_TRANSPOSE=1
+DS4_CUDA_NO_F16_PAIR_COMPRESSOR_TRANSPOSE_PREFETCH8=1
+DS4_CUDA_NO_Q8_FUSED_ALIGNED=1
+DS4_CUDA_NO_Q8_ALIGNED_PERSISTENT=1
+```
+
+`DS4_METAL_GRAPH_TOKEN_SPLIT_LAYERS=4` is the existing, Metal-named token-split
+control; setting it to `4` restores the old blocking split behavior on CUDA.
+
+### Initial ABBA result
+
+The final clean-build comparison used the same 2,048-token prompt frontier and
+512 generated tokens. The rollback arm set all five switches above; the default
+arm set none.
+
+| arm/run | prefill tok/s | decode tok/s | steady decode tok/s |
+|---|---:|---:|---:|
+| rollback 1 | 832.25 | 17.98 | 18.10 |
+| default 1 | 830.41 | 19.85 | 20.01 |
+| default 2 | 829.24 | 19.81 | 19.97 |
+| rollback 2 | 832.36 | 17.94 | 18.06 |
+| **rollback mean** | **832.305** | **17.960** | **18.080** |
+| **default mean** | **829.825** | **19.830** | **19.990** |
+| **gain** | **-0.30%** | **+10.41%** | **+10.56%** |
+
+A post-audit clean run measured **19.92 decode / 20.08 steady tok/s**. Using
+the ABBA default means (19.830 decode and 19.990 steady) relative to the
+original `b030961` mean, the complete port is **+11.88% decode** and
+**+11.79% steady decode**, while retaining a **+2.36% prefill** gain. Two
+1,024-generation runs measured 19.73/19.88 and 19.71/19.86 decode/steady as the
+attention context grew from 2,048 to 3,072 tokens.
+
+Profiler attribution matched the wall-clock result. The ordered F16 compressor
+family fell from about 5.14 ms/token to about 2.96 ms/token. The persistent
+K=1024 Q8 kernel fell from roughly 165 us to 150 us per layer, while the aligned
+fused Q8 kernels removed the raw-block penalty from pair, HC-expand, and
+attention-A projections.
+
+### Pre-target-21 exactness and validation
+
+- A 7,000-byte *Promessi sposi* prompt plus up to 256 greedy generated tokens
+  produced byte-identical text and full per-token logprob JSON with all defaults
+  versus all five decode rollbacks. SHA-256:
+  `128362d060d18e38ebadc7649c18ca8db625f53b5ee3b12762db9859d508d174`.
+- A post-audit 64-token default/rollback repeat was also byte-identical, SHA-256
+  `3f3d8890e13d4118b6a8964ed9ab3e2d11c8f0e3f8ab718d85a574bcf8cad6e5`.
+- `compute-sanitizer --tool memcheck` completed the promoted path with
+  `ERROR SUMMARY: 0 errors`.
+- A clean `make cuda-spark`, `make cuda-regression CUDA_ARCH=sm_121`, and
+  `make test CUDA_ARCH=sm_121` all passed after the final
+  safety hardening. The full test again included the 30,474-token context and
+  exact tensor-equivalence suite.
+- Final source audit added single-device derived-artifact admission, active
+  device validation, OOM negative caching, capture and divisibility guards,
+  persistent-grid overflow guards, signed-zero preservation, init/reset
+  hardening, and direct-MMQ scratch-size overflow admission.
+
+Neutral or regressive experiments were not promoted: grouped top-6 MoE CTAs,
+stream-0 graph replay, extending graph island A through Q/KV, small-output F16,
+exact-score split LDG/vector variants, compressor CTA grouping, persistent-grid
+retuning, and HC RMS-fold continuation.
+
+## README speed-table replication
+
+The upstream README's GB10 sweep was repeated with the final target-only build
+(no DSpark support model or speculative flags), using the same 2,048-token
+frontiers and 128 greedy generation tokens:
+
+```sh
+./ds4-bench -m ds4flash.gguf --cuda \
+  --prompt-file speed-bench/promessi_sposi.txt \
+  --ctx-start 2048 --ctx-max 65536 --step-incr 2048 \
+  --gen-tokens 128 \
+  --csv speed-bench/gb10.csv
+```
+
+| Context | README prefill | Final prefill | README generation | Final generation | Final steady |
+|---:|---:|---:|---:|---:|---:|
+| 2,048 | 825.76 | 832.86 | 18.05 | 20.58 | 20.69 |
+| 16,384 | 872.44 | 883.81 | 15.10 | 16.80 | 16.81 |
+| 32,768 | 855.94 | 865.40 | 14.43 | 15.99 | 16.00 |
+| 65,536 | 822.98 | 833.44 | 13.84 | 15.27 | 15.28 |
+
+Generation gains at those four table rows are respectively **+14.02%**,
+**+11.26%**, **+10.81%**, and **+10.33%**. The complete 32-frontier sweep is in
+`speed-bench/gb10.csv`; benchmark binary SHA-256 was
+`04d5321402dc073b1f0300a19063e95262a81cc20831fa4f083cf9147fcc145f`.
+
+## Target-21 follow-up
+
+A review of `eugr/spark-vllm-docker` at commit
+`e5f3cf9e5320d9a424966a801570bf452405d122` led to the referenced B12X SM121
+kernel package at `7cecbb2c4819636ae7f05f8b116f2c45ee2cff7b`. The applicable
+ideas were stable caller-owned scratch (no per-token asynchronous allocation),
+the split/parallel MHC decode structure, and measured GB10 launch geometry.
+Its tensor-core FP4/FP8, sparse-MLA, and MoE scheduler machinery is not a
+drop-in for this IQ2_XXS/Q2_K/F32-exact path.
+
+Two exact changes were promoted:
+
+1. The dense aligned-Q8 wrapper now prefers the already-owned 256 KiB aligned
+   Q8_1 scratch allocation instead of recording a per-token `cudaMallocAsync`
+   pool node. Rollback: `DS4_CUDA_NO_Q8_ALIGNED_DENSE_SCRATCH=1`.
+2. The single-row 4K HC weighted-sum + RMS kernel now uses 16 partial CTAs,
+   a one-CTA exact reduction replay, and 16 store CTAs. The reduction retains
+   the original ascending-column FMA chain and 256-lane shared tree. Rollback:
+   `DS4_CUDA_NO_HC_SPLIT_NORM_SPLIT4096=1`.
+
+Matched 512-generation AB results at 2,048 context:
+
+| Variant | Steady runs | Median steady |
+|---|---:|---:|
+| Both target-21 rollbacks | 19.78, 19.89 | 19.835 t/s |
+| Aligned scratch only | 20.68, 20.69 | 20.685 t/s |
+| Final 16-CTA HC split | 21.01, 21.06 | 21.035 t/s |
+
+A 1,024-generation run measured 20.73 decode and 20.89 steady t/s as context
+grew from 2,048 to 3,072 tokens. The full 32-frontier sweep in
+`gb10.csv` was rerun with this final build.
+
+The 7,000-byte prompt plus 256 greedy generated tokens remained byte-identical
+to the HC rollback, including full per-token logprob JSON. SHA-256:
+`952799babb7f421cb0e2e75e6ede9de73c40304da279ef1d8d99042ef684be62`.
+`make test CUDA_ARCH=sm_121`, `make cuda-regression CUDA_ARCH=sm_121`, and
+`compute-sanitizer --tool memcheck` all passed; sanitizer reported
+`ERROR SUMMARY: 0 errors`.
+
+## Post-rebase validation
+
+The branch was rebased onto `upstream/main` commit
+`84cc882352757baf628a1776badf7cc54d584e28` and retested. The rebased CUDA
+build passed `make cuda-spark`, `make test CUDA_ARCH=sm_121`, and
+`make cuda-regression CUDA_ARCH=sm_121`. Compute Sanitizer reported
+`ERROR SUMMARY: 0 errors`. The 7,000-byte prompt plus 256 greedy
+tokens remained byte-identical to the rollback logprobs, SHA-256
+`952799babb7f421cb0e2e75e6ede9de73c40304da279ef1d8d99042ef684be62`.
+
+Same-day 512-generation runs after the rebase measured 20.69, 20.65, and 20.74
+steady tok/s at 2,048 context. A preserved pre-rebase worktree measured 20.67
+and 20.67 steady tok/s in the same thermal window, so the rebase itself is
+performance-neutral within noise. The full sweep and README table above were
+refreshed with the rebased build.
diff --git a/speed-bench/gb10.csv b/speed-bench/gb10.csv
index a59f1dfe4b..60faa6b37b 100644
--- a/speed-bench/gb10.csv
+++ b/speed-bench/gb10.csv
@@ -1,33 +1,33 @@
 ctx_tokens,prefill_tokens,prefill_tps,gen_tokens,gen_tps,gen_first_ms,gen_steady_tokens,gen_steady_tps,kvcache_bytes
-2048,2048,825.76,128,18.05,71.721,127,18.20,52184460
-4096,2048,899.52,128,15.47,57.061,127,15.54,80373132
-6144,2048,888.62,128,15.37,65.899,127,15.45,108561804
-8192,2048,882.80,128,15.21,65.667,127,15.29,136750476
-10240,2048,874.95,128,15.13,67.430,127,15.21,164939148
-12288,2048,872.74,128,15.08,67.877,127,15.16,193127820
-14336,2048,867.76,128,15.02,67.140,127,15.10,221316492
-16384,2048,872.44,128,15.10,67.128,127,15.18,249505164
-18432,2048,871.75,128,15.04,67.562,127,15.12,277693836
-20480,2048,869.55,128,15.00,67.678,127,15.08,305882508
-22528,2048,866.37,128,14.96,68.248,127,15.04,334071180
-24576,2048,864.65,128,14.91,68.667,127,14.99,362259852
-26624,2048,861.34,128,14.86,68.919,127,14.94,390448524
-28672,2048,859.07,128,14.82,69.044,127,14.90,418637196
-30720,2048,856.88,128,14.80,68.233,127,14.87,446825868
-32768,2048,855.94,128,14.43,68.704,127,14.51,475014540
-34816,2048,850.96,128,14.36,70.661,127,14.43,503203212
-36864,2048,849.83,128,14.34,70.827,127,14.41,531391884
-38912,2048,849.59,128,14.30,70.971,127,14.38,559580556
-40960,2048,847.40,128,14.28,70.881,127,14.36,587769228
-43008,2048,843.24,128,14.24,71.968,127,14.32,615957900
-45056,2048,840.41,128,14.19,71.406,127,14.26,644146572
-47104,2048,839.95,128,14.16,71.523,127,14.23,672335244
-49152,2048,836.64,128,14.12,72.153,127,14.19,700523916
-51200,2048,837.72,128,14.08,71.579,127,14.15,728712588
-53248,2048,837.01,128,14.04,72.138,127,14.11,756901260
-55296,2048,829.92,128,14.00,73.733,127,14.08,785089932
-57344,2048,830.16,128,13.98,72.862,127,14.05,813278604
-59392,2048,829.11,128,13.94,73.157,127,14.01,841467276
-61440,2048,827.26,128,13.90,73.924,127,13.97,869655948
-63488,2048,825.04,128,13.86,73.787,127,13.93,897844620
-65536,2048,822.98,128,13.84,73.966,127,13.91,0
+2048,2048,832.86,128,20.58,77.503,127,20.69,52184460
+4096,2048,909.55,128,17.28,50.694,127,17.28,80373132
+6144,2048,901.83,128,17.15,59.791,127,17.16,108561804
+8192,2048,894.75,128,16.96,60.946,127,16.97,136750476
+10240,2048,884.88,128,16.85,60.291,127,16.86,164939148
+12288,2048,879.46,128,16.78,61.277,127,16.79,193127820
+14336,2048,876.32,128,16.71,61.579,127,16.72,221316492
+16384,2048,883.81,128,16.80,60.258,127,16.81,249505164
+18432,2048,881.73,128,16.73,61.953,127,16.74,277693836
+20480,2048,879.01,128,16.68,59.971,127,16.69,305882508
+22528,2048,875.57,128,16.64,61.660,127,16.65,334071180
+24576,2048,872.93,128,16.60,61.166,127,16.61,362259852
+26624,2048,871.46,128,16.54,62.080,127,16.55,390448524
+28672,2048,869.00,128,16.49,60.657,127,16.50,418637196
+30720,2048,865.56,128,16.46,62.215,127,16.47,446825868
+32768,2048,865.40,128,15.99,63.022,127,16.00,475014540
+34816,2048,859.60,128,15.91,64.303,127,15.92,503203212
+36864,2048,860.68,128,15.87,65.082,127,15.88,531391884
+38912,2048,857.03,128,15.83,63.978,127,15.84,559580556
+40960,2048,854.89,128,15.79,64.615,127,15.80,587769228
+43008,2048,853.18,128,15.76,64.846,127,15.76,615957900
+45056,2048,850.05,128,15.70,65.960,127,15.71,644146572
+47104,2048,847.88,128,15.66,65.232,127,15.67,672335244
+49152,2048,845.96,128,15.63,65.233,127,15.64,700523916
+51200,2048,846.48,128,15.58,65.126,127,15.59,728712588
+53248,2048,844.07,128,15.53,65.666,127,15.54,756901260
+55296,2048,839.85,128,15.48,66.767,127,15.49,785089932
+57344,2048,839.90,128,15.45,66.645,127,15.46,813278604
+59392,2048,835.70,128,15.41,65.679,127,15.42,841467276
+61440,2048,837.29,128,15.36,66.754,127,15.37,869655948
+63488,2048,832.94,128,15.32,67.365,127,15.33,897844620
+65536,2048,833.44,128,15.27,67.665,127,15.28,0
diff --git a/speed-bench/gpu_iq2_moe_prefill_bench.c b/speed-bench/gpu_iq2_moe_prefill_bench.c
new file mode 100644
index 0000000000..261f870863
--- /dev/null
+++ b/speed-bench/gpu_iq2_moe_prefill_bench.c
@@ -0,0 +1,1058 @@
+#define _POSIX_C_SOURCE 200809L
+
+#include "ds4_gpu.h"
+
+#include <errno.h>
+#include <limits.h>
+#include <math.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#if defined(DS4_BENCH_ROCM) && defined(DS4_BENCH_CUDA)
+#error "define exactly one of DS4_BENCH_ROCM or DS4_BENCH_CUDA"
+#elif !defined(DS4_BENCH_ROCM) && !defined(DS4_BENCH_CUDA)
+#error "define exactly one of DS4_BENCH_ROCM or DS4_BENCH_CUDA"
+#endif
+
+/*
+ * This source intentionally has no implicit backend selection.  Compile it
+ * once with DS4_BENCH_ROCM and link the ROCm implementation, or once with
+ * DS4_BENCH_CUDA and link the CUDA implementation.
+ *
+ * ROCm exposes a real in-process A/B policy switch.  CUDA currently has no
+ * independent tail-cull candidate selected by this harness; its build is a
+ * measurement-only run of the current production path.  In particular, the
+ * CUDA markers never label two executions of the same path as an A/B result.
+ */
+#if defined(DS4_BENCH_ROCM)
+#define BENCH_BACKEND "rocm"
+#define TAIL_ENABLE_ENV  "DS4_ROCM_ENABLE_IQ2_MOE_WMMA_TAIL_CULL"
+#define TAIL_DISABLE_ENV "DS4_ROCM_DISABLE_IQ2_MOE_WMMA_TAIL_CULL"
+#define ROCM_PROFILE_ENV "DS4_ROCM_IQ2_MOE_WMMA_PROFILE"
+#else
+#define BENCH_BACKEND "cuda"
+#define CUDA_PROFILE_ENV "DS4_CUDA_MOE_PROFILE"
+#endif
+
+#define IQ2_XXS_TYPE 16u
+#define Q2_K_TYPE 10u
+#define QK_K 256u
+#define IN_DIM 4096u
+#define MID_DIM 2048u
+#define OUT_DIM 4096u
+#define N_TOKENS 4096u
+#define N_TOTAL_EXPERT 256u
+#define N_EXPERT 6u
+#define CLAMP 4.0f
+#define GUARD_WORDS 64u
+#define GUARD_BYTES ((uint64_t)GUARD_WORDS * sizeof(uint32_t))
+#define GUARD_BITS 0x51a7c3e9u
+#define DEFAULT_SAMPLES 8u
+#define DEFAULT_WARMUPS 2u
+#define IO_CHUNK_BYTES (8u * 1024u * 1024u)
+#define GIB (1024ull * 1024ull * 1024ull)
+
+typedef struct {
+    uint16_t d;
+    uint16_t qs[QK_K / 8u];
+} block_iq2_xxs;
+
+typedef struct {
+    uint8_t scales[QK_K / 16u];
+    uint8_t qs[QK_K / 4u];
+    uint16_t d;
+    uint16_t dmin;
+} block_q2_K;
+
+_Static_assert(sizeof(block_iq2_xxs) == 66u,
+               "IQ2_XXS block layout changed");
+_Static_assert(sizeof(block_q2_K) == 84u,
+               "Q2_K block layout changed");
+
+typedef struct {
+    uint32_t samples;
+    uint32_t warmups;
+} bench_config;
+
+typedef enum {
+    ARM_BASELINE,
+    ARM_CANDIDATE,
+    ARM_CURRENT,
+} bench_arm;
+
+typedef struct {
+    void *model;
+    uint64_t model_size;
+    uint64_t gate_offset;
+    uint64_t up_offset;
+    uint64_t down_offset;
+    uint64_t gate_row_bytes;
+    uint64_t gate_expert_bytes;
+    uint64_t down_row_bytes;
+    uint64_t down_expert_bytes;
+
+    uint64_t x_bytes;
+    uint64_t route_count;
+    uint64_t route_i32_bytes;
+    uint64_t route_f32_bytes;
+    uint64_t pair_count;
+    uint64_t pair_f32_bytes;
+    uint64_t expert_count;
+    uint64_t expert_bytes;
+    uint64_t out_count;
+    uint64_t out_bytes;
+
+    ds4_gpu_tensor *x;
+    ds4_gpu_tensor *selected;
+    ds4_gpu_tensor *weights;
+    ds4_gpu_tensor *gate;
+    ds4_gpu_tensor *up;
+    ds4_gpu_tensor *mid;
+    ds4_gpu_tensor *experts;
+    ds4_gpu_tensor *out;
+} fixture;
+
+#if defined(DS4_BENCH_ROCM)
+typedef struct {
+    uint8_t *storage;
+    uint8_t *gate;
+    uint8_t *up;
+    uint8_t *mid;
+    uint8_t *experts;
+    uint8_t *out;
+    uint64_t bytes;
+} oracle_snapshot;
+#endif
+
+bool ds4_log_is_tty(FILE *fp) {
+    (void)fp;
+    return false;
+}
+
+#if defined(DS4_BENCH_CUDA)
+extern uint64_t ds4_cuda_test_moe_fast_profile_report_count(void);
+#endif
+
+static void usage(FILE *fp, const char *argv0) {
+    fprintf(fp,
+            "usage: %s [options]\n"
+            "\n"
+            "Resident production-geometry IQ2_XXS/Q2_K routed-MoE prefill "
+            "benchmark (%s build).\n",
+            argv0, BENCH_BACKEND);
+#if defined(DS4_BENCH_ROCM)
+    fprintf(fp,
+            "ROCm performs a real balanced A/B: DISABLE=1 is baseline and "
+            "ENABLE=1 is candidate. The harness enables the GPU-only "
+            "IQ2/Q2 WMMA profiler.\n"
+            "  --samples N    samples per arm, even (default: %u)\n"
+            "  --warmups N    warmups per arm, even (default: %u)\n",
+            DEFAULT_SAMPLES, DEFAULT_WARMUPS);
+#else
+    fprintf(fp,
+            "CUDA is measurement-only: every marker is variant=current and "
+            "DS4_CUDA_MOE_PROFILE is enabled.\n"
+            "No reliable baseline/candidate selector exists in this process; "
+            "this harness does not report a false A/B.\n"
+            "  --samples N    current-path measured runs (default: %u)\n"
+            "  --warmups N    current-path warmup runs (default: %u)\n",
+            DEFAULT_SAMPLES, DEFAULT_WARMUPS);
+#endif
+    fprintf(fp, "  -h, --help     show this help\n");
+}
+
+static uint32_t parse_u32(const char *text, const char *option) {
+    char *end = NULL;
+    errno = 0;
+    const unsigned long long value = strtoull(text, &end, 10);
+    if (errno != 0 || !text[0] || !end || *end || value > UINT32_MAX) {
+        fprintf(stderr, "gpu-iq2-moe-prefill-bench: invalid %s: %s\n",
+                option, text);
+        exit(2);
+    }
+    return (uint32_t)value;
+}
+
+static const char *need_arg(int *index, int argc, char **argv) {
+    if (*index + 1 >= argc) {
+        fprintf(stderr, "gpu-iq2-moe-prefill-bench: %s needs a value\n",
+                argv[*index]);
+        exit(2);
+    }
+    return argv[++*index];
+}
+
+static bench_config parse_options(int argc, char **argv) {
+    bench_config config = {
+        .samples = DEFAULT_SAMPLES,
+        .warmups = DEFAULT_WARMUPS,
+    };
+    for (int i = 1; i < argc; i++) {
+        if (!strcmp(argv[i], "-h") || !strcmp(argv[i], "--help")) {
+            usage(stdout, argv[0]);
+            exit(0);
+        } else if (!strcmp(argv[i], "--samples")) {
+            const char *option = argv[i];
+            config.samples = parse_u32(need_arg(&i, argc, argv), option);
+        } else if (!strcmp(argv[i], "--warmups")) {
+            const char *option = argv[i];
+            config.warmups = parse_u32(need_arg(&i, argc, argv), option);
+        } else {
+            fprintf(stderr, "gpu-iq2-moe-prefill-bench: unknown option: %s\n",
+                    argv[i]);
+            usage(stderr, argv[0]);
+            exit(2);
+        }
+    }
+    if (config.samples == 0u) {
+        fprintf(stderr, "gpu-iq2-moe-prefill-bench: --samples must be nonzero\n");
+        exit(2);
+    }
+#if defined(DS4_BENCH_ROCM)
+    if ((config.samples & 1u) != 0u || (config.warmups & 1u) != 0u) {
+        fprintf(stderr,
+                "gpu-iq2-moe-prefill-bench: ROCm --samples and --warmups "
+                "must be even\n");
+        exit(2);
+    }
+#endif
+    return config;
+}
+
+static uint64_t align_up(uint64_t value, uint64_t alignment) {
+    return (value + alignment - 1u) / alignment * alignment;
+}
+
+static uint32_t mix32(uint32_t value) {
+    value ^= value >> 16u;
+    value *= 0x7feb352du;
+    value ^= value >> 15u;
+    value *= 0x846ca68bu;
+    value ^= value >> 16u;
+    return value;
+}
+
+static void fill_iq2(block_iq2_xxs *matrix, uint32_t salt) {
+    const uint32_t blocks_per_row = IN_DIM / QK_K;
+    for (uint32_t expert = 0; expert < N_TOTAL_EXPERT; expert++) {
+        for (uint32_t row = 0; row < MID_DIM; row++) {
+            for (uint32_t block = 0; block < blocks_per_row; block++) {
+                block_iq2_xxs *b = matrix +
+                    ((uint64_t)expert * MID_DIM + row) * blocks_per_row + block;
+                const uint32_t key = salt * 977u + expert * 431u +
+                                     row * 37u + block * 811u;
+                b->d = (uint16_t)(0x1800u + ((key & 1u) ? 0x0200u : 0u));
+                for (uint32_t i = 0; i < QK_K / 8u; i++) {
+                    b->qs[i] = (uint16_t)(key + i * 509u +
+                                          (i >> 2u) * 131u);
+                }
+            }
+        }
+    }
+}
+
+static void fill_q2(block_q2_K *matrix) {
+    const uint32_t blocks_per_row = MID_DIM / QK_K;
+    for (uint32_t expert = 0; expert < N_TOTAL_EXPERT; expert++) {
+        for (uint32_t row = 0; row < OUT_DIM; row++) {
+            for (uint32_t block = 0; block < blocks_per_row; block++) {
+                block_q2_K *b = matrix +
+                    ((uint64_t)expert * OUT_DIM + row) * blocks_per_row + block;
+                const uint32_t key = expert * 617u + row * 73u + block * 991u;
+                for (uint32_t group = 0; group < QK_K / 16u; group++) {
+                    const uint8_t scale =
+                        (uint8_t)(1u + (key + 3u * group) % 7u);
+                    const uint8_t min =
+                        (uint8_t)((key / 5u + group) % 4u);
+                    b->scales[group] =
+                        (uint8_t)(scale | (uint8_t)(min << 4u));
+                }
+                for (uint32_t i = 0; i < QK_K / 4u; i++) {
+                    b->qs[i] = (uint8_t)(key + 29u * i + (i >> 1u) * 7u);
+                }
+                b->d = 0x1800u;
+                b->dmin = 0x1400u;
+            }
+        }
+    }
+}
+
+static uint64_t touch_model_pages(const void *model, uint64_t bytes,
+                                  uint64_t page) {
+    const volatile uint8_t *data = model;
+    uint64_t checksum = 0xcbf29ce484222325ull;
+    for (uint64_t offset = 0; offset < bytes; offset += page) {
+        checksum ^= data[offset];
+        checksum *= 0x100000001b3ull;
+    }
+    checksum ^= data[bytes - 1u];
+    return checksum;
+}
+
+/* Every 32-row tail size 1..31 is represented at production N=4096/top-6. */
+static int build_routes(int32_t *selected, float *weights) {
+    static const uint8_t final_remainders[8] = {1, 2, 3, 4, 5, 6, 7, 4};
+    uint32_t target[N_TOTAL_EXPERT];
+    uint32_t remaining[N_TOTAL_EXPERT];
+    uint32_t actual[N_TOTAL_EXPERT] = {0};
+    uint64_t target_sum = 0;
+
+    for (uint32_t expert = 0; expert < N_TOTAL_EXPERT; expert++) {
+        const uint32_t tail = expert < 248u ?
+            1u + (expert * 17u) % 31u : final_remainders[expert - 248u];
+        const uint32_t full_tiles =
+            ((expert * 73u) & 255u) < 131u ? 3u : 2u;
+        target[expert] = full_tiles * 32u + tail;
+        remaining[expert] = target[expert];
+        target_sum += target[expert];
+    }
+    if (target_sum != (uint64_t)N_TOKENS * N_EXPERT) {
+        fprintf(stderr, "gpu-iq2-moe-prefill-bench: route target sum=%llu\n",
+                (unsigned long long)target_sum);
+        return 0;
+    }
+
+    for (uint32_t token = 0; token < N_TOKENS; token++) {
+        uint8_t used[N_TOTAL_EXPERT] = {0};
+        float raw_weight[N_EXPERT];
+        float weight_sum = 0.0f;
+        for (uint32_t slot = 0; slot < N_EXPERT; slot++) {
+            uint32_t best = UINT32_MAX;
+            uint32_t best_remaining = 0;
+            uint32_t best_hash = 0;
+            for (uint32_t expert = 0; expert < N_TOTAL_EXPERT; expert++) {
+                if (used[expert] || remaining[expert] == 0u) continue;
+                const uint32_t hash = mix32(
+                    token * 0x9e3779b9u ^ slot * 0x85ebca6bu ^
+                    expert * 0xc2b2ae35u);
+                if (best == UINT32_MAX || remaining[expert] > best_remaining ||
+                    (remaining[expert] == best_remaining && hash > best_hash)) {
+                    best = expert;
+                    best_remaining = remaining[expert];
+                    best_hash = hash;
+                }
+            }
+            if (best == UINT32_MAX) {
+                fprintf(stderr,
+                        "gpu-iq2-moe-prefill-bench: route scheduler exhausted "
+                        "at token=%u slot=%u\n", token, slot);
+                return 0;
+            }
+            const uint64_t route = (uint64_t)token * N_EXPERT + slot;
+            selected[route] = (int32_t)best;
+            used[best] = 1u;
+            remaining[best]--;
+            actual[best]++;
+            raw_weight[slot] = 1.0f + (float)(mix32(
+                token * 0x27d4eb2du ^ slot * 0x165667b1u) % 17u);
+            weight_sum += raw_weight[slot];
+        }
+        for (uint32_t slot = 0; slot < N_EXPERT; slot++) {
+            weights[(uint64_t)token * N_EXPERT + slot] =
+                raw_weight[slot] / weight_sum;
+        }
+    }
+
+    bool tails_seen[32] = {false};
+    uint32_t min_count = UINT32_MAX;
+    uint32_t max_count = 0;
+    for (uint32_t expert = 0; expert < N_TOTAL_EXPERT; expert++) {
+        if (remaining[expert] != 0u || actual[expert] != target[expert]) {
+            fprintf(stderr,
+                    "gpu-iq2-moe-prefill-bench: expert=%u target=%u "
+                    "actual=%u remaining=%u\n",
+                    expert, target[expert], actual[expert], remaining[expert]);
+            return 0;
+        }
+        tails_seen[actual[expert] & 31u] = true;
+        if (actual[expert] < min_count) min_count = actual[expert];
+        if (actual[expert] > max_count) max_count = actual[expert];
+    }
+    for (uint32_t tail = 1; tail < 32u; tail++) {
+        if (!tails_seen[tail]) {
+            fprintf(stderr,
+                    "gpu-iq2-moe-prefill-bench: missing tail=%u\n", tail);
+            return 0;
+        }
+    }
+    if (tails_seen[0]) {
+        fprintf(stderr,
+                "gpu-iq2-moe-prefill-bench: unexpected full-only expert\n");
+        return 0;
+    }
+    fprintf(stderr,
+            "DS4_GPU_IQ2_MOE_PREFILL_SETUP backend=%s routes=%u experts=%u "
+            "topk=%u count_min=%u count_max=%u tail_coverage=1..31 "
+            "unique_per_token=yes\n",
+            BENCH_BACKEND, N_TOKENS * N_EXPERT, N_TOTAL_EXPERT, N_EXPERT,
+            min_count, max_count);
+    return 1;
+}
+
+static void fill_input(float *x) {
+    for (uint32_t token = 0; token < N_TOKENS; token++) {
+        for (uint32_t column = 0; column < IN_DIM; column++) {
+            const uint32_t bits = mix32(
+                token * 0x9e3779b9u ^ column * 0x85ebca6bu);
+            const int32_t centered = (int32_t)(bits & 511u) - 256;
+            x[(uint64_t)token * IN_DIM + column] =
+                (float)centered / 1024.0f;
+        }
+    }
+}
+
+static void make_guard(uint32_t guard[GUARD_WORDS]) {
+    for (uint32_t i = 0; i < GUARD_WORDS; i++) {
+        guard[i] = GUARD_BITS ^ (i * 0x9e3779b9u);
+    }
+}
+
+static int write_guard(ds4_gpu_tensor *tensor, uint64_t offset) {
+    uint32_t guard[GUARD_WORDS];
+    make_guard(guard);
+    return ds4_gpu_tensor_write(tensor, offset, guard, sizeof(guard));
+}
+
+static int check_guard(const char *name, const ds4_gpu_tensor *tensor,
+                       uint64_t offset) {
+    uint32_t expected[GUARD_WORDS];
+    uint32_t actual[GUARD_WORDS];
+    make_guard(expected);
+    if (!ds4_gpu_tensor_read(tensor, offset, actual, sizeof(actual))) {
+        fprintf(stderr,
+                "DS4_GPU_IQ2_MOE_PREFILL_CANARY backend=%s name=%s "
+                "result=READ_FAIL\n", BENCH_BACKEND, name);
+        return 0;
+    }
+    for (uint32_t i = 0; i < GUARD_WORDS; i++) {
+        if (actual[i] != expected[i]) {
+            fprintf(stderr,
+                    "DS4_GPU_IQ2_MOE_PREFILL_CANARY backend=%s name=%s "
+                    "result=FAIL word=%u expected=0x%08x actual=0x%08x\n",
+                    BENCH_BACKEND, name, i, expected[i], actual[i]);
+            return 0;
+        }
+    }
+    return 1;
+}
+
+static int poison_outputs(fixture *f) {
+    int ok = ds4_gpu_tensor_fill_f32(
+        f->gate, -101.0f, (f->pair_f32_bytes + GUARD_BYTES) / sizeof(float));
+    ok = ds4_gpu_tensor_fill_f32(
+        f->up, -102.0f,
+        (f->pair_f32_bytes + GUARD_BYTES) / sizeof(float)) && ok;
+    ok = ds4_gpu_tensor_fill_f32(
+        f->mid, -103.0f,
+        (f->pair_f32_bytes + GUARD_BYTES) / sizeof(float)) && ok;
+    ok = ds4_gpu_tensor_fill_f32(
+        f->experts, -104.0f,
+        (f->expert_bytes + GUARD_BYTES) / sizeof(float)) && ok;
+    ok = ds4_gpu_tensor_fill_f32(
+        f->out, -105.0f,
+        (f->out_bytes + GUARD_BYTES) / sizeof(float)) && ok;
+    ok = write_guard(f->gate, f->pair_f32_bytes) && ok;
+    ok = write_guard(f->up, f->pair_f32_bytes) && ok;
+    ok = write_guard(f->mid, f->pair_f32_bytes) && ok;
+    ok = write_guard(f->experts, f->expert_bytes) && ok;
+    ok = write_guard(f->out, f->out_bytes) && ok;
+    return ok;
+}
+
+static int check_all_canaries(const fixture *f) {
+    int ok = check_guard("x", f->x, f->x_bytes);
+    ok = check_guard("selected", f->selected, f->route_i32_bytes) && ok;
+    ok = check_guard("weights", f->weights, f->route_f32_bytes) && ok;
+    ok = check_guard("gate", f->gate, f->pair_f32_bytes) && ok;
+    ok = check_guard("up", f->up, f->pair_f32_bytes) && ok;
+    ok = check_guard("mid-f32", f->mid, f->pair_f32_bytes) && ok;
+    ok = check_guard("experts-f32", f->experts, f->expert_bytes) && ok;
+    ok = check_guard("out-f32", f->out, f->out_bytes) && ok;
+    return ok;
+}
+
+static const char *variant_name(bench_arm arm) {
+    switch (arm) {
+        case ARM_BASELINE: return "baseline";
+        case ARM_CANDIDATE: return "candidate";
+        case ARM_CURRENT: return "current";
+    }
+    return "invalid";
+}
+
+static int select_variant(bench_arm arm) {
+#if defined(DS4_BENCH_ROCM)
+    if (arm != ARM_BASELINE && arm != ARM_CANDIDATE) return 0;
+    if (unsetenv(TAIL_ENABLE_ENV) != 0 || unsetenv(TAIL_DISABLE_ENV) != 0) {
+        return 0;
+    }
+    return setenv(arm == ARM_BASELINE ? TAIL_DISABLE_ENV : TAIL_ENABLE_ENV,
+                  "1", 1) == 0;
+#else
+    return arm == ARM_CURRENT;
+#endif
+}
+
+static int run_once(fixture *f, bench_arm arm,
+                    const char *phase, const char *order,
+                    uint32_t sample, uint32_t cycle, uint32_t position,
+                    bool poison, bool check_canaries) {
+    if (!select_variant(arm)) {
+        fprintf(stderr,
+                "gpu-iq2-moe-prefill-bench: backend policy setup failed\n");
+        return 0;
+    }
+    if (poison && !poison_outputs(f)) {
+        fprintf(stderr, "gpu-iq2-moe-prefill-bench: output poison failed\n");
+        return 0;
+    }
+
+    fprintf(stderr,
+            "DS4_GPU_IQ2_MOE_PREFILL_BENCH backend=%s phase=%s "
+            "variant=%s sample=%u cycle=%u position=%u order=%s "
+            "force_resident=1 ssd_streaming=0\n",
+            BENCH_BACKEND, phase, variant_name(arm), sample,
+            cycle, position, order);
+    fflush(stderr);
+
+    if (!ds4_gpu_begin_commands()) {
+        fprintf(stderr, "gpu-iq2-moe-prefill-bench: begin commands failed\n");
+        return 0;
+    }
+#if defined(DS4_BENCH_CUDA)
+    const uint64_t profile_reports_before =
+        ds4_cuda_test_moe_fast_profile_report_count();
+#endif
+    bool mid_is_f16 = false;
+    const int call_ok = ds4_gpu_routed_moe_batch_tensor(
+        f->out, f->gate, f->up, f->mid, f->experts,
+        f->model, f->model_size,
+        f->gate_offset, f->up_offset, f->down_offset,
+        IQ2_XXS_TYPE, Q2_K_TYPE,
+        f->gate_expert_bytes, f->gate_row_bytes,
+        f->down_expert_bytes, f->down_row_bytes,
+        IN_DIM, MID_DIM, OUT_DIM,
+        f->selected, f->weights, N_TOTAL_EXPERT, N_EXPERT, CLAMP, f->x,
+        0u, N_TOKENS, &mid_is_f16, true);
+    const int end_ok = ds4_gpu_end_commands();
+#if defined(DS4_BENCH_CUDA)
+    const uint64_t profile_reports_after =
+        ds4_cuda_test_moe_fast_profile_report_count();
+    const int profile_ok =
+        profile_reports_after == profile_reports_before + 1u;
+    if (!profile_ok) {
+        fprintf(stderr,
+                "DS4_GPU_IQ2_MOE_PREFILL_BENCH backend=cuda result=FAIL "
+                "variant=%s fast_profile_reports_before=%llu "
+                "fast_profile_reports_after=%llu expected_delta=1\n",
+                variant_name(arm),
+                (unsigned long long)profile_reports_before,
+                (unsigned long long)profile_reports_after);
+    }
+#else
+    const int profile_ok = 1;
+#endif
+    int ok = call_ok && end_ok && profile_ok;
+    if (!ok) {
+        fprintf(stderr,
+                "DS4_GPU_IQ2_MOE_PREFILL_BENCH backend=%s result=FAIL "
+                "variant=%s call=%d end=%d reported_mid_format=%s\n",
+                BENCH_BACKEND, variant_name(arm), call_ok, end_ok,
+                mid_is_f16 ? "f16" : "f32-or-opaque");
+    }
+    if (check_canaries) ok = check_all_canaries(f) && ok;
+    return ok;
+}
+
+#if defined(DS4_BENCH_ROCM)
+static int snapshot_alloc(oracle_snapshot *snapshot, const fixture *f) {
+    memset(snapshot, 0, sizeof(*snapshot));
+    snapshot->bytes = 3u * f->pair_f32_bytes +
+                      f->expert_bytes + f->out_bytes;
+    if (snapshot->bytes > SIZE_MAX) return 0;
+    snapshot->storage = malloc((size_t)snapshot->bytes);
+    if (!snapshot->storage) return 0;
+    snapshot->gate = snapshot->storage;
+    snapshot->up = snapshot->gate + f->pair_f32_bytes;
+    snapshot->mid = snapshot->up + f->pair_f32_bytes;
+    snapshot->experts = snapshot->mid + f->pair_f32_bytes;
+    snapshot->out = snapshot->experts + f->expert_bytes;
+    return 1;
+}
+
+static int capture_snapshot(oracle_snapshot *snapshot, const fixture *f) {
+    return ds4_gpu_tensor_read(
+               f->gate, 0, snapshot->gate, f->pair_f32_bytes) &&
+           ds4_gpu_tensor_read(
+               f->up, 0, snapshot->up, f->pair_f32_bytes) &&
+           ds4_gpu_tensor_read(
+               f->mid, 0, snapshot->mid, f->pair_f32_bytes) &&
+           ds4_gpu_tensor_read(
+               f->experts, 0, snapshot->experts, f->expert_bytes) &&
+           ds4_gpu_tensor_read(
+               f->out, 0, snapshot->out, f->out_bytes);
+}
+
+static int tensor_matches(const char *name, const ds4_gpu_tensor *tensor,
+                          const uint8_t *expected, uint64_t bytes,
+                          uint8_t *scratch) {
+    uint64_t offset = 0;
+    while (offset < bytes) {
+        const size_t chunk = bytes - offset > IO_CHUNK_BYTES ?
+            IO_CHUNK_BYTES : (size_t)(bytes - offset);
+        if (!ds4_gpu_tensor_read(tensor, offset, scratch, chunk)) {
+            fprintf(stderr,
+                    "DS4_GPU_IQ2_MOE_PREFILL_ORACLE backend=rocm "
+                    "candidate=tail-cull tensor=%s result=READ_FAIL "
+                    "offset=%llu\n", name, (unsigned long long)offset);
+            return 0;
+        }
+        if (memcmp(scratch, expected + offset, chunk) != 0) {
+            size_t mismatch = 0;
+            while (mismatch < chunk &&
+                   scratch[mismatch] == expected[offset + mismatch]) {
+                mismatch++;
+            }
+            fprintf(stderr,
+                    "DS4_GPU_IQ2_MOE_PREFILL_ORACLE backend=rocm "
+                    "candidate=tail-cull tensor=%s result=MISMATCH byte=%llu "
+                    "expected=0x%02x actual=0x%02x\n",
+                    name, (unsigned long long)(offset + mismatch),
+                    expected[offset + mismatch], scratch[mismatch]);
+            return 0;
+        }
+        offset += chunk;
+    }
+    return 1;
+}
+
+static int run_correctness(fixture *f) {
+    oracle_snapshot baseline;
+    if (!snapshot_alloc(&baseline, f)) {
+        fprintf(stderr,
+                "gpu-iq2-moe-prefill-bench: oracle snapshot allocation failed\n");
+        return 0;
+    }
+    uint8_t *scratch = malloc(IO_CHUNK_BYTES);
+    int ok = scratch != NULL;
+    if (ok) {
+        ok = run_once(f, ARM_BASELINE, "oracle", "BASELINE",
+                      0u, 0u, 0u, true, true);
+    }
+    if (ok) ok = capture_snapshot(&baseline, f);
+    if (ok) {
+        ok = run_once(f, ARM_CANDIDATE, "oracle", "CANDIDATE",
+                      0u, 0u, 0u, true, true);
+    }
+    const int gate_ok = ok && tensor_matches(
+        "gate_scratch", f->gate, baseline.gate, f->pair_f32_bytes, scratch);
+    const int up_ok = ok && tensor_matches(
+        "up_scratch", f->up, baseline.up, f->pair_f32_bytes, scratch);
+    const int mid_ok = ok && tensor_matches(
+        "mid_scratch", f->mid, baseline.mid, f->pair_f32_bytes, scratch);
+    const int experts_ok = ok && tensor_matches(
+        "down_scratch", f->experts, baseline.experts, f->expert_bytes, scratch);
+    const int out_ok = ok && tensor_matches(
+        "out_f32", f->out, baseline.out, f->out_bytes, scratch);
+    ok = ok && gate_ok && up_ok && mid_ok && experts_ok && out_ok;
+    fprintf(stderr,
+            "DS4_GPU_IQ2_MOE_PREFILL_ORACLE backend=rocm "
+            "candidate=tail-cull result=%s gate_scratch=%s up_scratch=%s "
+            "mid_scratch=%s down_scratch=%s out_f32=%s canaries=%s\n",
+            ok ? "PASS" : "FAIL",
+            gate_ok ? "exact" : "mismatch",
+            up_ok ? "exact" : "mismatch",
+            mid_ok ? "exact" : "mismatch",
+            experts_ok ? "exact" : "mismatch",
+            out_ok ? "exact" : "mismatch",
+            ok ? "PASS" : "FAIL");
+    free(scratch);
+    free(baseline.storage);
+    return ok;
+}
+
+static int run_balanced_block(fixture *f, const char *phase,
+                              uint32_t samples_per_arm) {
+    uint32_t arm_samples[2] = {0, 0};
+    const uint32_t cycles = samples_per_arm / 2u;
+    for (uint32_t cycle = 0; cycle < cycles; cycle++) {
+        static const bench_arm abba[4] = {
+            ARM_BASELINE, ARM_CANDIDATE, ARM_CANDIDATE, ARM_BASELINE,
+        };
+        static const bench_arm baab[4] = {
+            ARM_CANDIDATE, ARM_BASELINE, ARM_BASELINE, ARM_CANDIDATE,
+        };
+        const bench_arm *order = (cycle & 1u) ? baab : abba;
+        const char *order_name = (cycle & 1u) ? "BAAB" : "ABBA";
+        for (uint32_t position = 0; position < 4u; position++) {
+            const bench_arm arm = order[position];
+            if (!run_once(f, arm, phase, order_name,
+                          arm_samples[arm]++, cycle, position,
+                          false, false)) {
+                return 0;
+            }
+        }
+    }
+    return arm_samples[ARM_BASELINE] == samples_per_arm &&
+           arm_samples[ARM_CANDIDATE] == samples_per_arm;
+}
+
+static int run_experiment(fixture *f, const bench_config *config) {
+    if (config->warmups != 0u &&
+        !run_balanced_block(f, "warmup", config->warmups)) {
+        return 0;
+    }
+    if (!run_balanced_block(f, "sample", config->samples)) return 0;
+    fprintf(stderr,
+            "DS4_GPU_IQ2_MOE_PREFILL_BENCH backend=rocm phase=complete "
+            "mode=real-ab samples_per_variant=%u warmups_per_variant=%u "
+            "result=PASS\n", config->samples, config->warmups);
+    return 1;
+}
+#else
+static int validate_current_tensor(const char *name,
+                                   const ds4_gpu_tensor *tensor,
+                                   uint64_t bytes, float poison,
+                                   bool dense_f32) {
+    uint8_t *scratch = malloc(IO_CHUNK_BYTES);
+    if (!scratch) return 0;
+    uint64_t hash = 0xcbf29ce484222325ull;
+    uint64_t changed = 0;
+    uint64_t unchanged = 0;
+    uint64_t offset = 0;
+    int ok = 1;
+    uint8_t poison_bytes[sizeof(float)];
+    memcpy(poison_bytes, &poison, sizeof(poison_bytes));
+    while (offset < bytes && ok) {
+        const size_t chunk = bytes - offset > IO_CHUNK_BYTES ?
+            IO_CHUNK_BYTES : (size_t)(bytes - offset);
+        if (!ds4_gpu_tensor_read(tensor, offset, scratch, chunk)) {
+            ok = 0;
+            break;
+        }
+        if (dense_f32) {
+            const float *values = (const float *)scratch;
+            const size_t count = chunk / sizeof(float);
+            for (size_t i = 0; i < count; i++) {
+                if (!isfinite(values[i])) {
+                    fprintf(stderr,
+                            "DS4_GPU_IQ2_MOE_PREFILL_ORACLE backend=cuda "
+                            "mode=current-single-run tensor=%s "
+                            "result=NONFINITE element=%llu\n", name,
+                            (unsigned long long)(offset / sizeof(float) + i));
+                    ok = 0;
+                    break;
+                }
+                if (values[i] != poison) {
+                    changed++;
+                } else {
+                    unchanged++;
+                }
+            }
+        } else {
+            for (size_t i = 0; i < chunk; i++) {
+                const uint8_t expected =
+                    poison_bytes[(size_t)((offset + i) % sizeof(float))];
+                if (scratch[i] != expected) {
+                    changed++;
+                } else {
+                    unchanged++;
+                }
+            }
+        }
+        for (size_t i = 0; i < chunk; i++) {
+            hash ^= scratch[i];
+            hash *= 0x100000001b3ull;
+        }
+        offset += chunk;
+    }
+    if (ok && (dense_f32 ? unchanged != 0u : changed == 0u)) ok = 0;
+    fprintf(stderr,
+            "DS4_GPU_IQ2_MOE_PREFILL_ORACLE backend=cuda "
+            "mode=current-single-run scope=structural tensor=%s layout=%s "
+            "result=%s "
+            "written=%llu unchanged_poison=%llu units=%s hash=0x%016llx\n",
+            name, dense_f32 ? "dense_f32" : "opaque_scratch",
+            ok ? "PASS" : "FAIL",
+            (unsigned long long)changed, (unsigned long long)unchanged,
+            dense_f32 ? "elements" : "bytes",
+            (unsigned long long)hash);
+    free(scratch);
+    return ok;
+}
+
+static int run_correctness(fixture *f) {
+    int ok = run_once(f, ARM_CURRENT, "oracle", "CURRENT",
+                      0u, 0u, 0u, true, true);
+    const int gate_ok = ok && validate_current_tensor(
+        "gate_scratch", f->gate, f->pair_f32_bytes, -101.0f, false);
+    const int up_ok = ok && validate_current_tensor(
+        "up_scratch", f->up, f->pair_f32_bytes, -102.0f, false);
+    const int mid_ok = ok && validate_current_tensor(
+        "mid_scratch", f->mid, f->pair_f32_bytes, -103.0f, false);
+    const int experts_ok = ok && validate_current_tensor(
+        "down_f32", f->experts, f->expert_bytes, -104.0f, true);
+    const int out_ok = ok && validate_current_tensor(
+        "out_f32", f->out, f->out_bytes, -105.0f, true);
+    ok = ok && gate_ok && up_ok && mid_ok && experts_ok && out_ok;
+    fprintf(stderr,
+            "DS4_GPU_IQ2_MOE_PREFILL_ORACLE backend=cuda "
+            "mode=current-single-run scope=structural result=%s "
+            "canaries=%s numerical_ab=NOT_AVAILABLE\n",
+            ok ? "PASS" : "FAIL", ok ? "PASS" : "FAIL");
+    return ok;
+}
+
+static int run_experiment(fixture *f, const bench_config *config) {
+    for (uint32_t i = 0; i < config->warmups; i++) {
+        if (!run_once(f, ARM_CURRENT, "warmup", "CURRENT",
+                      i, 0u, 0u, false, false)) {
+            return 0;
+        }
+    }
+    for (uint32_t i = 0; i < config->samples; i++) {
+        if (!run_once(f, ARM_CURRENT, "sample", "CURRENT",
+                      i, 0u, 0u, false, false)) {
+            return 0;
+        }
+    }
+    fprintf(stderr,
+            "DS4_GPU_IQ2_MOE_PREFILL_BENCH backend=cuda phase=complete "
+            "mode=measurement-only variant=current samples=%u warmups=%u "
+            "profiler=%s numerical_ab=NOT_AVAILABLE result=PASS\n",
+            config->samples, config->warmups, CUDA_PROFILE_ENV);
+    return 1;
+}
+#endif
+
+static void free_tensors(fixture *f) {
+    ds4_gpu_tensor_free(f->out);
+    ds4_gpu_tensor_free(f->experts);
+    ds4_gpu_tensor_free(f->mid);
+    ds4_gpu_tensor_free(f->up);
+    ds4_gpu_tensor_free(f->gate);
+    ds4_gpu_tensor_free(f->weights);
+    ds4_gpu_tensor_free(f->selected);
+    ds4_gpu_tensor_free(f->x);
+    f->out = NULL;
+    f->experts = NULL;
+    f->mid = NULL;
+    f->up = NULL;
+    f->gate = NULL;
+    f->weights = NULL;
+    f->selected = NULL;
+    f->x = NULL;
+}
+
+static int init_fixture(fixture *f) {
+    memset(f, 0, sizeof(*f));
+    const long page_long = sysconf(_SC_PAGESIZE);
+    if (page_long <= 0) {
+        fprintf(stderr, "gpu-iq2-moe-prefill-bench: page size unavailable\n");
+        return 0;
+    }
+    const uint64_t page = (uint64_t)page_long;
+    f->gate_row_bytes =
+        (uint64_t)(IN_DIM / QK_K) * sizeof(block_iq2_xxs);
+    f->gate_expert_bytes = (uint64_t)MID_DIM * f->gate_row_bytes;
+    const uint64_t gate_tensor_bytes =
+        (uint64_t)N_TOTAL_EXPERT * f->gate_expert_bytes;
+    f->down_row_bytes =
+        (uint64_t)(MID_DIM / QK_K) * sizeof(block_q2_K);
+    f->down_expert_bytes = (uint64_t)OUT_DIM * f->down_row_bytes;
+    const uint64_t down_tensor_bytes =
+        (uint64_t)N_TOTAL_EXPERT * f->down_expert_bytes;
+    f->gate_offset = 0;
+    f->up_offset = align_up(gate_tensor_bytes, page);
+    f->down_offset = align_up(f->up_offset + gate_tensor_bytes, page);
+    f->model_size = align_up(f->down_offset + down_tensor_bytes, page);
+
+    f->x_bytes = (uint64_t)N_TOKENS * IN_DIM * sizeof(float);
+    f->route_count = (uint64_t)N_TOKENS * N_EXPERT;
+    f->route_i32_bytes = f->route_count * sizeof(int32_t);
+    f->route_f32_bytes = f->route_count * sizeof(float);
+    f->pair_count = f->route_count * MID_DIM;
+    f->pair_f32_bytes = f->pair_count * sizeof(float);
+    f->expert_count = (uint64_t)N_TOKENS * N_EXPERT * OUT_DIM;
+    f->expert_bytes = f->expert_count * sizeof(float);
+    f->out_count = (uint64_t)N_TOKENS * OUT_DIM;
+    f->out_bytes = f->out_count * sizeof(float);
+
+    if (f->gate_row_bytes != 1056u ||
+        f->gate_expert_bytes != 2162688u ||
+        f->down_row_bytes != 672u ||
+        f->down_expert_bytes != 2752512u) {
+        fprintf(stderr,
+                "gpu-iq2-moe-prefill-bench: production layout mismatch "
+                "gate_row=%llu gate_expert=%llu down_row=%llu "
+                "down_expert=%llu\n",
+                (unsigned long long)f->gate_row_bytes,
+                (unsigned long long)f->gate_expert_bytes,
+                (unsigned long long)f->down_row_bytes,
+                (unsigned long long)f->down_expert_bytes);
+        return 0;
+    }
+
+    const uint64_t tensor_bytes =
+        f->x_bytes + f->route_i32_bytes + f->route_f32_bytes +
+        3u * (f->pair_f32_bytes + GUARD_BYTES) +
+        f->expert_bytes + GUARD_BYTES + f->out_bytes + GUARD_BYTES +
+        3u * GUARD_BYTES;
+#if defined(DS4_BENCH_ROCM)
+    const uint64_t oracle_bytes =
+        3u * f->pair_f32_bytes + f->expert_bytes +
+        f->out_bytes + IO_CHUNK_BYTES;
+#else
+    const uint64_t oracle_bytes = IO_CHUNK_BYTES;
+#endif
+    const uint64_t setup_host_bytes =
+        f->x_bytes + f->route_i32_bytes + f->route_f32_bytes;
+    const uint64_t explicit_peak =
+        f->model_size + tensor_bytes + oracle_bytes + setup_host_bytes;
+    fprintf(stderr,
+            "DS4_GPU_IQ2_MOE_PREFILL_SETUP backend=%s "
+            "geometry=N%u,d%u,mid%u,out%u,experts%u,top%u "
+            "model=%.3f_GiB tensors=%.3f_GiB oracle=%.3f_GiB "
+            "explicit_peak=%.3f_GiB resident=1\n",
+            BENCH_BACKEND, N_TOKENS, IN_DIM, MID_DIM, OUT_DIM,
+            N_TOTAL_EXPERT, N_EXPERT,
+            (double)f->model_size / (double)GIB,
+            (double)tensor_bytes / (double)GIB,
+            (double)oracle_bytes / (double)GIB,
+            (double)explicit_peak / (double)GIB);
+    if (explicit_peak >= 5u * GIB) {
+        fprintf(stderr,
+                "gpu-iq2-moe-prefill-bench: explicit peak exceeds 5 GiB\n");
+        return 0;
+    }
+
+    if (posix_memalign(&f->model, (size_t)page,
+                       (size_t)f->model_size) != 0) {
+        fprintf(stderr,
+                "gpu-iq2-moe-prefill-bench: model allocation failed\n");
+        return 0;
+    }
+    fprintf(stderr,
+            "DS4_GPU_IQ2_MOE_PREFILL_SETUP backend=%s phase=fill_weights "
+            "tensor=gate\n", BENCH_BACKEND);
+    fill_iq2((block_iq2_xxs *)((uint8_t *)f->model + f->gate_offset), 19u);
+    fprintf(stderr,
+            "DS4_GPU_IQ2_MOE_PREFILL_SETUP backend=%s phase=fill_weights "
+            "tensor=up\n", BENCH_BACKEND);
+    fill_iq2((block_iq2_xxs *)((uint8_t *)f->model + f->up_offset), 47u);
+    fprintf(stderr,
+            "DS4_GPU_IQ2_MOE_PREFILL_SETUP backend=%s phase=fill_weights "
+            "tensor=down\n", BENCH_BACKEND);
+    fill_q2((block_q2_K *)((uint8_t *)f->model + f->down_offset));
+    const uint64_t checksum = touch_model_pages(f->model, f->model_size, page);
+    fprintf(stderr,
+            "DS4_GPU_IQ2_MOE_PREFILL_SETUP backend=%s phase=touch_weights "
+            "pages=%llu checksum=0x%016llx resident=1\n",
+            BENCH_BACKEND,
+            (unsigned long long)((f->model_size + page - 1u) / page),
+            (unsigned long long)checksum);
+
+    float *x_host = malloc((size_t)f->x_bytes);
+    int32_t *selected_host = malloc((size_t)f->route_i32_bytes);
+    float *weights_host = malloc((size_t)f->route_f32_bytes);
+    int ok = x_host && selected_host && weights_host;
+    if (ok) fill_input(x_host);
+    if (ok) ok = build_routes(selected_host, weights_host);
+    if (ok) ok = ds4_gpu_set_model_map(f->model, f->model_size);
+
+    if (ok) f->x = ds4_gpu_tensor_alloc(f->x_bytes + GUARD_BYTES);
+    if (ok) f->selected =
+        ds4_gpu_tensor_alloc(f->route_i32_bytes + GUARD_BYTES);
+    if (ok) f->weights =
+        ds4_gpu_tensor_alloc(f->route_f32_bytes + GUARD_BYTES);
+    if (ok) f->gate =
+        ds4_gpu_tensor_alloc(f->pair_f32_bytes + GUARD_BYTES);
+    if (ok) f->up =
+        ds4_gpu_tensor_alloc(f->pair_f32_bytes + GUARD_BYTES);
+    if (ok) f->mid =
+        ds4_gpu_tensor_alloc(f->pair_f32_bytes + GUARD_BYTES);
+    if (ok) f->experts =
+        ds4_gpu_tensor_alloc(f->expert_bytes + GUARD_BYTES);
+    if (ok) f->out = ds4_gpu_tensor_alloc(f->out_bytes + GUARD_BYTES);
+    ok = ok && f->x && f->selected && f->weights && f->gate && f->up &&
+         f->mid && f->experts && f->out;
+
+    if (ok) ok = ds4_gpu_tensor_write(f->x, 0, x_host, f->x_bytes);
+    if (ok) ok = ds4_gpu_tensor_write(
+        f->selected, 0, selected_host, f->route_i32_bytes);
+    if (ok) ok = ds4_gpu_tensor_write(
+        f->weights, 0, weights_host, f->route_f32_bytes);
+    if (ok) ok = write_guard(f->x, f->x_bytes);
+    if (ok) ok = write_guard(f->selected, f->route_i32_bytes);
+    if (ok) ok = write_guard(f->weights, f->route_f32_bytes);
+    if (ok) ok = poison_outputs(f);
+
+    free(weights_host);
+    free(selected_host);
+    free(x_host);
+    if (!ok) {
+        fprintf(stderr,
+                "gpu-iq2-moe-prefill-bench: fixture initialization failed\n");
+    }
+    return ok;
+}
+
+static int configure_backend(void) {
+#if defined(DS4_BENCH_ROCM)
+    return unsetenv(TAIL_ENABLE_ENV) == 0 &&
+           unsetenv(TAIL_DISABLE_ENV) == 0 &&
+           setenv(ROCM_PROFILE_ENV, "1", 1) == 0;
+#else
+    /* Presence, including value zero, enables the existing CUDA profiler. */
+    return setenv(CUDA_PROFILE_ENV, "1", 1) == 0;
+#endif
+}
+
+int main(int argc, char **argv) {
+    const bench_config config = parse_options(argc, argv);
+    if (!configure_backend()) {
+        fprintf(stderr,
+                "gpu-iq2-moe-prefill-bench: backend environment setup failed\n");
+        return 1;
+    }
+
+    if (!ds4_gpu_init()) {
+        fprintf(stderr,
+                "gpu-iq2-moe-prefill-bench: %s initialization failed\n",
+                BENCH_BACKEND);
+        /* Both GPU backends make cleanup idempotent for partially initialized
+         * state; do not strand a stream, handle, or allocation on init error. */
+        ds4_gpu_cleanup();
+        return 1;
+    }
+    ds4_gpu_set_quality(false);
+    ds4_gpu_set_ssd_streaming(false);
+
+    fixture f;
+    const int fixture_ready = init_fixture(&f);
+    int ok = fixture_ready;
+    if (ok) ok = run_correctness(&f);
+    if (ok) ok = run_experiment(&f, &config);
+    /* A failing dispatch must not suppress the final overrun check. */
+    if (fixture_ready) {
+        const int canaries_ok = check_all_canaries(&f);
+        ok = canaries_ok && ok;
+    }
+
+#if defined(DS4_BENCH_ROCM)
+    (void)unsetenv(TAIL_ENABLE_ENV);
+    (void)unsetenv(TAIL_DISABLE_ENV);
+    (void)unsetenv(ROCM_PROFILE_ENV);
+#endif
+    free_tensors(&f);
+    ds4_gpu_cleanup();
+    free(f.model);
+    fprintf(stderr,
+            "DS4_GPU_IQ2_MOE_PREFILL_BENCH backend=%s result=%s\n",
+            BENCH_BACKEND, ok ? "PASS" : "FAIL");
+    return ok ? 0 : 1;
+}
diff --git a/speed-bench/m1_max_q4_raw_gathered_ab.md b/speed-bench/m1_max_q4_raw_gathered_ab.md
new file mode 100644
index 0000000000..21e9f34c39
--- /dev/null
+++ b/speed-bench/m1_max_q4_raw_gathered_ab.md
@@ -0,0 +1,47 @@
+# M1 Max Q4 raw-gathered attention A/B
+
+Date: 2026-08-22
+
+Hardware: Apple M1 Max, 32 GiB RAM. Backend: Metal with SSD streaming.
+Model: `DeepSeek-V4-Flash-IQ2XXS-w2Q2K-AProjQ4-SExpQ8-OutQ8-chat-v2-imatrix-0731.gguf`.
+
+The control is the new default raw-gathered path. The candidate sets
+`DS4_METAL_DISABLE_DECODE_RAW_GATHERED_ATTN=1` and restores the legacy raw-only
+attention path. Both use the same decode split schedule.
+
+## Correctness
+
+- 16-step greedy top-20 logprob dumps are byte-identical.
+- Both files have SHA-256
+  `7ee7b8a119f8f93c4ea91fb27867eb4563037e34fc8c61ca795a0953ace61738`.
+- Each alternating run compared 145 full-vocabulary frontiers: 18,745,600
+  float logits and 144 non-EOS selections were bit-identical.
+
+## Alternating same-process results
+
+| Prefix | Variant | Steady tokens/s | Delta |
+|---:|---|---:|---:|
+| 128 | raw-gathered | 6.7360 | +1.52% |
+| 128 | legacy raw | 6.6350 | baseline |
+| 2048 | raw-gathered | 5.4325 | +0.66% |
+| 2048 | legacy raw | 5.3967 | baseline |
+
+Command shape:
+
+```sh
+speed-bench/metal_decode_schedule_bench \
+  -m /path/to/model.gguf \
+  --prompt-file speed-bench/promessi_sposi.txt \
+  --prefix-tokens 128 --ctx 300 \
+  --warmup 16 --tokens 128 \
+  --candidate-env DS4_METAL_DISABLE_DECODE_RAW_GATHERED_ATTN \
+  --include-selection --ssd-streaming
+```
+
+Forcing `DS4_METAL_ENABLE_GATHERED_KV_STAGE=1` on the M1 was neutral:
+6.4763 tokens/s versus 6.4769 tokens/s control (-0.01%), with exact logits.
+It therefore remains automatic only on its existing device policy.
+
+The packed32 raw extension is retained for eligible devices, but the M1 does
+not arm the inverse-RoPE fusion required by that kernel, so it was not selected
+in these measurements.
diff --git a/speed-bench/metal_decode_schedule_bench.c b/speed-bench/metal_decode_schedule_bench.c
index 5866b49db4..3b3852c7ef 100644
--- a/speed-bench/metal_decode_schedule_bench.c
+++ b/speed-bench/metal_decode_schedule_bench.c
@@ -33,7 +33,11 @@ typedef struct {
     int ctx;
     int warmup;
     int measured;
+    uint32_t ssd_cache_experts;
+    uint32_t ssd_preload_experts;
     bool include_selection;
+    bool ssd_streaming;
+    bool ssd_streaming_cold;
     decode_schedule control;
     decode_schedule candidate;
 } bench_config;
@@ -53,6 +57,12 @@ static void usage(FILE *fp, const char *argv0) {
             "  --candidate-first N    candidate first split (default: 1; control with --candidate-env)\n"
             "  --candidate-second N   candidate second split (default: 32; control with --candidate-env)\n"
             "  --candidate-env NAME   unset NAME for control, set NAME=1 for candidate\n"
+            "  --ssd-streaming       use the SSD-backed model path instead of full residency\n"
+            "  --ssd-streaming-cold  skip the default expert-cache preload\n"
+            "  --ssd-streaming-cache-experts N\n"
+            "                        dynamic expert-cache entry count\n"
+            "  --ssd-streaming-preload-experts N\n"
+            "                        popularity preload count\n"
             "  --include-selection    include one non-EOS argmax in each timed step\n",
             argv0);
 }
@@ -91,7 +101,11 @@ static bench_config parse_options(int argc, char **argv) {
         .ctx = DEFAULT_CTX,
         .warmup = DEFAULT_WARMUP,
         .measured = DEFAULT_MEASURED,
+        .ssd_cache_experts = 0,
+        .ssd_preload_experts = 0,
         .include_selection = false,
+        .ssd_streaming = false,
+        .ssd_streaming_cold = false,
         .control = {.first = 2, .second = 32},
         .candidate = {.first = 1, .second = 32},
     };
@@ -111,6 +125,16 @@ static bench_config parse_options(int argc, char **argv) {
             cfg.candidate_env = need_arg(&i, argc, argv, arg);
         } else if (!strcmp(arg, "--include-selection")) {
             cfg.include_selection = true;
+        } else if (!strcmp(arg, "--ssd-streaming")) {
+            cfg.ssd_streaming = true;
+        } else if (!strcmp(arg, "--ssd-streaming-cold")) {
+            cfg.ssd_streaming_cold = true;
+        } else if (!strcmp(arg, "--ssd-streaming-cache-experts")) {
+            cfg.ssd_cache_experts = (uint32_t)parse_int_arg(
+                need_arg(&i, argc, argv, arg), arg, 1);
+        } else if (!strcmp(arg, "--ssd-streaming-preload-experts")) {
+            cfg.ssd_preload_experts = (uint32_t)parse_int_arg(
+                need_arg(&i, argc, argv, arg), arg, 1);
         } else if (!strcmp(arg, "--prefix-tokens")) {
             cfg.prefix_tokens =
                 parse_int_arg(need_arg(&i, argc, argv, arg), arg, 1);
@@ -159,6 +183,15 @@ static bench_config parse_options(int argc, char **argv) {
         }
     }
 
+    if (!cfg.ssd_streaming &&
+        (cfg.ssd_streaming_cold || cfg.ssd_cache_experts != 0 ||
+         cfg.ssd_preload_experts != 0)) {
+        fprintf(stderr,
+                "metal-decode-schedule-bench: SSD cache options require "
+                "--ssd-streaming\n");
+        exit(2);
+    }
+
     const int64_t needed =
         (int64_t)cfg.prefix_tokens + cfg.warmup + cfg.measured + 1;
     if (needed > cfg.ctx) {
@@ -372,7 +405,11 @@ int main(int argc, char **argv) {
         .backend = DS4_BACKEND_METAL,
         .context_size = cfg.ctx,
         .power_percent = 100,
-        .warm_weights = true,
+        .warm_weights = !cfg.ssd_streaming,
+        .ssd_streaming = cfg.ssd_streaming,
+        .ssd_streaming_cold = cfg.ssd_streaming_cold,
+        .ssd_streaming_cache_experts = cfg.ssd_cache_experts,
+        .ssd_streaming_preload_experts = cfg.ssd_preload_experts,
     };
     ds4_engine *engine = NULL;
     ds4_session *sessions[VARIANT_COUNT] = {0};
@@ -438,7 +475,8 @@ int main(int argc, char **argv) {
     fprintf(stderr,
             "metal-decode-schedule-bench: model=%s prompt=%s prefix=%d "
             "ctx=%d warmup=%d measured=%d control=%d/%d candidate=%d/%d "
-            "candidate_env=%s include_selection=%s\n",
+            "candidate_env=%s include_selection=%s ssd_streaming=%s "
+            "ssd_cold=%s cache_experts=%u preload_experts=%u\n",
             cfg.model_path,
             cfg.prompt_path,
             cfg.prefix_tokens,
@@ -450,7 +488,11 @@ int main(int argc, char **argv) {
             cfg.candidate.first,
             cfg.candidate.second,
             cfg.candidate_env ? cfg.candidate_env : "(none)",
-            cfg.include_selection ? "yes" : "no");
+            cfg.include_selection ? "yes" : "no",
+            cfg.ssd_streaming ? "yes" : "no",
+            cfg.ssd_streaming_cold ? "yes" : "no",
+            cfg.ssd_cache_experts,
+            cfg.ssd_preload_experts);
 
     const int eos = ds4_token_eos(engine);
     const int total_steps = cfg.warmup + cfg.measured;
diff --git a/speed-bench/metal_iq2_moe_tail_cull_bench.c b/speed-bench/metal_iq2_moe_tail_cull_bench.c
new file mode 100644
index 0000000000..f6e0188a26
--- /dev/null
+++ b/speed-bench/metal_iq2_moe_tail_cull_bench.c
@@ -0,0 +1,935 @@
+#define _DARWIN_C_SOURCE
+
+#include "ds4_gpu.h"
+
+#include <errno.h>
+#include <limits.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#define IQ2_XXS_TYPE 16u
+#define Q2_K_TYPE 10u
+#define QK_K 256u
+#define IN_DIM 4096u
+#define MID_DIM 2048u
+#define OUT_DIM 4096u
+#define N_TOKENS 4096u
+#ifdef DS4_METAL_IQ2_MOE_TOP8_PAIR_BENCH
+#define N_TOTAL_EXPERT 288u
+#define N_EXPERT 8u
+#define CLAMP 0.0f
+#define BENCH_DESCRIPTION \
+    "Resident GLM-geometry IQ2_XXS routed-MoE top-8 pair-fusion benchmark."
+#define EXPERIMENT_NAME "top8-pair-fusion"
+#else
+#define N_TOTAL_EXPERT 256u
+#define N_EXPERT 6u
+#define CLAMP 4.0f
+#define BENCH_DESCRIPTION \
+    "Resident production-geometry IQ2_XXS pair routed-MoE tail-cull benchmark."
+#define EXPERIMENT_NAME "pair-tail-cull"
+#endif
+#define GUARD_WORDS 64u
+#define GUARD_BYTES ((uint64_t)GUARD_WORDS * sizeof(uint32_t))
+#define GUARD_BITS 0x51a7c3e9u
+#define DEFAULT_SAMPLES 8u
+#define DEFAULT_WARMUP_CYCLES 1u
+#define COMPARE_CHUNK_BYTES (8u * 1024u * 1024u)
+#define GIB (1024ull * 1024ull * 1024ull)
+
+#define PAIR_TAIL_ENABLE_ENV \
+    "DS4_METAL_ENABLE_IQ2_XXS_MOE_MM_ID_PAIR_TAIL_SIMDGROUP_CULL"
+#define PAIR_TAIL_DISABLE_ENV \
+    "DS4_METAL_DISABLE_IQ2_XXS_MOE_MM_ID_PAIR_TAIL_SIMDGROUP_CULL"
+#define PAIR_FUSION_DISABLE_ENV \
+    "DS4_METAL_DISABLE_MOE_MM_ID_PAIR_SWIGLU"
+
+typedef struct {
+    uint16_t d;
+    uint16_t qs[QK_K / 8u];
+} block_iq2_xxs;
+
+typedef struct {
+    uint8_t scales[QK_K / 16u];
+    uint8_t qs[QK_K / 4u];
+    uint16_t d;
+    uint16_t dmin;
+} block_q2_K;
+
+_Static_assert(sizeof(block_iq2_xxs) == 66u,
+               "IQ2_XXS block layout changed");
+_Static_assert(sizeof(block_q2_K) == 84u,
+               "Q2_K block layout changed");
+
+typedef struct {
+    uint32_t samples;
+    uint32_t warmup_cycles;
+} bench_config;
+
+typedef enum {
+    ARM_BASELINE,
+    ARM_CANDIDATE,
+} bench_arm;
+
+typedef struct {
+    void *model;
+    uint64_t model_size;
+    uint64_t gate_offset;
+    uint64_t up_offset;
+    uint64_t down_offset;
+    uint64_t gate_row_bytes;
+    uint64_t gate_expert_bytes;
+    uint64_t down_row_bytes;
+    uint64_t down_expert_bytes;
+
+    uint64_t x_bytes;
+    uint64_t route_count;
+    uint64_t route_i32_bytes;
+    uint64_t route_f32_bytes;
+    uint64_t pair_count;
+    uint64_t pair_f16_bytes;
+    uint64_t pair_f32_bytes;
+    uint64_t expert_count;
+    uint64_t expert_bytes;
+    uint64_t out_count;
+    uint64_t out_bytes;
+
+    ds4_gpu_tensor *x;
+    ds4_gpu_tensor *selected;
+    ds4_gpu_tensor *weights;
+    ds4_gpu_tensor *gate;
+    ds4_gpu_tensor *up;
+    ds4_gpu_tensor *mid;
+    ds4_gpu_tensor *experts;
+    ds4_gpu_tensor *out;
+} fixture;
+
+typedef struct {
+    uint8_t *storage;
+    uint8_t *mid;
+    uint8_t *experts;
+    uint8_t *out;
+    uint64_t bytes;
+} oracle_snapshot;
+
+bool ds4_log_is_tty(FILE *fp) {
+    (void)fp;
+    return false;
+}
+
+static void usage(FILE *fp, const char *argv0) {
+    fprintf(fp,
+            "usage: %s [options]\n"
+            "\n"
+            BENCH_DESCRIPTION "\n"
+            "Metal stage profiling prints the kernel GPU timestamps; marker "
+            "lines identify each arm.\n"
+            "\n"
+            "  --samples N         samples per arm, even (default: %u)\n"
+            "  --warmup-cycles N   four-run balanced warmup cycles "
+            "(default: %u)\n"
+            "  -h, --help          show this help\n",
+            argv0, DEFAULT_SAMPLES, DEFAULT_WARMUP_CYCLES);
+}
+
+static uint32_t parse_u32(const char *text, const char *option,
+                          uint32_t minimum) {
+    char *end = NULL;
+    errno = 0;
+    const unsigned long long value = strtoull(text, &end, 10);
+    if (errno != 0 || !text[0] || !end || *end ||
+        value < minimum || value > UINT32_MAX) {
+        fprintf(stderr,
+                "metal-iq2-moe-tail-cull-bench: invalid %s: %s\n",
+                option, text);
+        exit(2);
+    }
+    return (uint32_t)value;
+}
+
+static const char *need_arg(int *index, int argc, char **argv) {
+    if (*index + 1 >= argc) {
+        fprintf(stderr,
+                "metal-iq2-moe-tail-cull-bench: %s needs a value\n",
+                argv[*index]);
+        exit(2);
+    }
+    return argv[++*index];
+}
+
+static bench_config parse_options(int argc, char **argv) {
+    bench_config config = {
+        .samples = DEFAULT_SAMPLES,
+        .warmup_cycles = DEFAULT_WARMUP_CYCLES,
+    };
+    for (int i = 1; i < argc; i++) {
+        if (!strcmp(argv[i], "-h") || !strcmp(argv[i], "--help")) {
+            usage(stdout, argv[0]);
+            exit(0);
+        } else if (!strcmp(argv[i], "--samples")) {
+            const char *option = argv[i];
+            const char *value = need_arg(&i, argc, argv);
+            config.samples = parse_u32(value, option, 2u);
+        } else if (!strcmp(argv[i], "--warmup-cycles")) {
+            const char *option = argv[i];
+            const char *value = need_arg(&i, argc, argv);
+            config.warmup_cycles = parse_u32(value, option, 0u);
+        } else {
+            fprintf(stderr,
+                    "metal-iq2-moe-tail-cull-bench: unknown option: %s\n",
+                    argv[i]);
+            usage(stderr, argv[0]);
+            exit(2);
+        }
+    }
+    if ((config.samples & 1u) != 0u) {
+        fprintf(stderr,
+                "metal-iq2-moe-tail-cull-bench: --samples must be even\n");
+        exit(2);
+    }
+    return config;
+}
+
+static uint64_t align_up(uint64_t value, uint64_t alignment) {
+    return (value + alignment - 1u) / alignment * alignment;
+}
+
+static uint32_t mix32(uint32_t value) {
+    value ^= value >> 16u;
+    value *= 0x7feb352du;
+    value ^= value >> 15u;
+    value *= 0x846ca68bu;
+    value ^= value >> 16u;
+    return value;
+}
+
+static void fill_iq2(block_iq2_xxs *matrix, uint32_t salt) {
+    const uint32_t blocks_per_row = IN_DIM / QK_K;
+    for (uint32_t expert = 0; expert < N_TOTAL_EXPERT; expert++) {
+        for (uint32_t row = 0; row < MID_DIM; row++) {
+            for (uint32_t block = 0; block < blocks_per_row; block++) {
+                block_iq2_xxs *b = matrix +
+                    ((uint64_t)expert * MID_DIM + row) * blocks_per_row +
+                    block;
+                const uint32_t key = salt * 977u + expert * 431u +
+                                     row * 37u + block * 811u;
+                b->d = (uint16_t)(0x1800u +
+                                  ((key & 1u) ? 0x0200u : 0u));
+                for (uint32_t i = 0; i < QK_K / 8u; i++) {
+                    b->qs[i] = (uint16_t)(key + i * 509u +
+                                          (i >> 2u) * 131u);
+                }
+            }
+        }
+    }
+}
+
+static void fill_q2(block_q2_K *matrix) {
+    const uint32_t blocks_per_row = MID_DIM / QK_K;
+    for (uint32_t expert = 0; expert < N_TOTAL_EXPERT; expert++) {
+        for (uint32_t row = 0; row < OUT_DIM; row++) {
+            for (uint32_t block = 0; block < blocks_per_row; block++) {
+                block_q2_K *b = matrix +
+                    ((uint64_t)expert * OUT_DIM + row) * blocks_per_row +
+                    block;
+                const uint32_t key = expert * 617u + row * 73u +
+                                     block * 991u;
+                for (uint32_t group = 0; group < QK_K / 16u; group++) {
+                    const uint8_t scale =
+                        (uint8_t)(1u + (key + 3u * group) % 7u);
+                    const uint8_t min =
+                        (uint8_t)((key / 5u + group) % 4u);
+                    b->scales[group] =
+                        (uint8_t)(scale | (uint8_t)(min << 4u));
+                }
+                for (uint32_t i = 0; i < QK_K / 4u; i++) {
+                    b->qs[i] =
+                        (uint8_t)(key + 29u * i + (i >> 1u) * 7u);
+                }
+                b->d = 0x1800u;
+                b->dmin = 0x1400u;
+            }
+        }
+    }
+}
+
+static uint64_t touch_model_pages(const void *model, uint64_t bytes,
+                                  uint64_t page) {
+    const volatile uint8_t *data = model;
+    uint64_t checksum = 0xcbf29ce484222325ull;
+    for (uint64_t offset = 0; offset < bytes; offset += page) {
+        checksum ^= data[offset];
+        checksum *= 0x100000001b3ull;
+    }
+    checksum ^= data[bytes - 1u];
+    return checksum;
+}
+
+/* Construct non-uniform expert counts with every final tile size 1..31.
+ * A per-token hash breaks ties in the remaining-count scheduler and keeps all
+ * routed experts unique without changing the target counts. */
+static int build_routes(int32_t *selected, float *weights) {
+#ifndef DS4_METAL_IQ2_MOE_TOP8_PAIR_BENCH
+    static const uint8_t final_remainders[8] = {1, 2, 3, 4, 5, 6, 7, 4};
+#endif
+    uint32_t target[N_TOTAL_EXPERT];
+    uint32_t remaining[N_TOTAL_EXPERT];
+    uint32_t actual[N_TOTAL_EXPERT] = {0};
+    uint64_t target_sum = 0;
+
+    for (uint32_t expert = 0; expert < N_TOTAL_EXPERT; expert++) {
+#ifdef DS4_METAL_IQ2_MOE_TOP8_PAIR_BENCH
+        const uint32_t tail = 1u + (expert * 17u) % 31u;
+        target[expert] = 3u * 32u + tail;
+#else
+        const uint32_t tail = expert < 248u ?
+            1u + (expert * 17u) % 31u :
+            final_remainders[expert - 248u];
+        const uint32_t full_tiles =
+            ((expert * 73u) & 255u) < 131u ? 3u : 2u;
+        target[expert] = full_tiles * 32u + tail;
+#endif
+        target_sum += target[expert];
+    }
+#ifdef DS4_METAL_IQ2_MOE_TOP8_PAIR_BENCH
+    uint64_t deficit = (uint64_t)N_TOKENS * N_EXPERT - target_sum;
+    while (deficit != 0u) {
+        bool progressed = false;
+        /* Preserve experts 0..30 as one complete permutation of tails. */
+        for (uint32_t expert = 31u;
+             expert < N_TOTAL_EXPERT && deficit != 0u;
+             expert++) {
+            if ((target[expert] & 31u) == 31u) continue;
+            target[expert]++;
+            target_sum++;
+            deficit--;
+            progressed = true;
+        }
+        if (!progressed) {
+            fprintf(stderr,
+                    "metal-iq2-moe-tail-cull-bench: route target "
+                    "distribution exhausted\n");
+            return 0;
+        }
+    }
+#endif
+    for (uint32_t expert = 0; expert < N_TOTAL_EXPERT; expert++) {
+        remaining[expert] = target[expert];
+    }
+    if (target_sum != (uint64_t)N_TOKENS * N_EXPERT) {
+        fprintf(stderr,
+                "metal-iq2-moe-tail-cull-bench: route target sum=%llu\n",
+                (unsigned long long)target_sum);
+        return 0;
+    }
+
+    for (uint32_t token = 0; token < N_TOKENS; token++) {
+        uint8_t used[N_TOTAL_EXPERT] = {0};
+        float raw_weight[N_EXPERT];
+        float weight_sum = 0.0f;
+        for (uint32_t slot = 0; slot < N_EXPERT; slot++) {
+            uint32_t best = UINT32_MAX;
+            uint32_t best_remaining = 0;
+            uint32_t best_hash = 0;
+            for (uint32_t expert = 0; expert < N_TOTAL_EXPERT; expert++) {
+                if (used[expert] || remaining[expert] == 0u) continue;
+                const uint32_t hash = mix32(
+                    token * 0x9e3779b9u ^ slot * 0x85ebca6bu ^
+                    expert * 0xc2b2ae35u);
+                if (best == UINT32_MAX ||
+                    remaining[expert] > best_remaining ||
+                    (remaining[expert] == best_remaining &&
+                     hash > best_hash)) {
+                    best = expert;
+                    best_remaining = remaining[expert];
+                    best_hash = hash;
+                }
+            }
+            if (best == UINT32_MAX) {
+                fprintf(stderr,
+                        "metal-iq2-moe-tail-cull-bench: route scheduler "
+                        "exhausted at token=%u slot=%u\n",
+                        token, slot);
+                return 0;
+            }
+            const uint64_t route = (uint64_t)token * N_EXPERT + slot;
+            selected[route] = (int32_t)best;
+            used[best] = 1u;
+            remaining[best]--;
+            actual[best]++;
+            raw_weight[slot] = 1.0f + (float)(mix32(
+                token * 0x27d4eb2du ^ slot * 0x165667b1u) % 17u);
+            weight_sum += raw_weight[slot];
+        }
+        for (uint32_t slot = 0; slot < N_EXPERT; slot++) {
+            weights[(uint64_t)token * N_EXPERT + slot] =
+                raw_weight[slot] / weight_sum;
+        }
+    }
+
+    bool tails_seen[32] = {false};
+    uint32_t min_count = UINT32_MAX;
+    uint32_t max_count = 0;
+    for (uint32_t expert = 0; expert < N_TOTAL_EXPERT; expert++) {
+        if (remaining[expert] != 0u || actual[expert] != target[expert]) {
+            fprintf(stderr,
+                    "metal-iq2-moe-tail-cull-bench: expert=%u target=%u "
+                    "actual=%u remaining=%u\n",
+                    expert, target[expert], actual[expert],
+                    remaining[expert]);
+            return 0;
+        }
+        tails_seen[actual[expert] & 31u] = true;
+        if (actual[expert] < min_count) min_count = actual[expert];
+        if (actual[expert] > max_count) max_count = actual[expert];
+    }
+    for (uint32_t tail = 1; tail < 32u; tail++) {
+        if (!tails_seen[tail]) {
+            fprintf(stderr,
+                    "metal-iq2-moe-tail-cull-bench: missing tail=%u\n",
+                    tail);
+            return 0;
+        }
+    }
+    if (tails_seen[0]) {
+        fprintf(stderr,
+                "metal-iq2-moe-tail-cull-bench: unexpected full-only expert\n");
+        return 0;
+    }
+    fprintf(stderr,
+            "DS4_IQ2_MOE_TAIL_SETUP routes=%u experts=%u topk=%u "
+            "count_min=%u count_max=%u tail_coverage=1..31 "
+            "unique_per_token=yes\n",
+            N_TOKENS * N_EXPERT, N_TOTAL_EXPERT, N_EXPERT,
+            min_count, max_count);
+    return 1;
+}
+
+static void fill_input(float *x) {
+    for (uint32_t token = 0; token < N_TOKENS; token++) {
+        for (uint32_t column = 0; column < IN_DIM; column++) {
+            const uint32_t bits = mix32(
+                token * 0x9e3779b9u ^ column * 0x85ebca6bu);
+            const int32_t centered = (int32_t)(bits & 511u) - 256;
+            x[(uint64_t)token * IN_DIM + column] =
+                (float)centered / 1024.0f;
+        }
+    }
+}
+
+static void make_guard(uint32_t guard[GUARD_WORDS]) {
+    for (uint32_t i = 0; i < GUARD_WORDS; i++) {
+        guard[i] = GUARD_BITS ^ (i * 0x9e3779b9u);
+    }
+}
+
+static int write_guard(ds4_gpu_tensor *tensor, uint64_t offset) {
+    uint32_t guard[GUARD_WORDS];
+    make_guard(guard);
+    return ds4_gpu_tensor_write(tensor, offset, guard, sizeof(guard));
+}
+
+static int check_guard(const char *name, const ds4_gpu_tensor *tensor,
+                       uint64_t offset) {
+    uint32_t expected[GUARD_WORDS];
+    uint32_t actual[GUARD_WORDS];
+    make_guard(expected);
+    if (!ds4_gpu_tensor_read(tensor, offset, actual, sizeof(actual))) {
+        fprintf(stderr,
+                "DS4_IQ2_MOE_TAIL_CANARY name=%s result=READ_FAIL\n",
+                name);
+        return 0;
+    }
+    for (uint32_t i = 0; i < GUARD_WORDS; i++) {
+        if (actual[i] != expected[i]) {
+            fprintf(stderr,
+                    "DS4_IQ2_MOE_TAIL_CANARY name=%s result=FAIL "
+                    "word=%u expected=0x%08x actual=0x%08x\n",
+                    name, i, expected[i], actual[i]);
+            return 0;
+        }
+    }
+    return 1;
+}
+
+static int poison_outputs(fixture *f) {
+    int ok = ds4_gpu_tensor_fill_f32(
+        f->gate, -101.0f, (f->pair_f32_bytes + GUARD_BYTES) / sizeof(float));
+    ok = ds4_gpu_tensor_fill_f32(
+        f->up, -102.0f, (f->pair_f32_bytes + GUARD_BYTES) / sizeof(float)) && ok;
+    ok = ds4_gpu_tensor_fill_f32(
+        f->mid, -103.0f, (f->pair_f32_bytes + GUARD_BYTES) / sizeof(float)) && ok;
+    ok = ds4_gpu_tensor_fill_f32(
+        f->experts, -104.0f,
+        (f->expert_bytes + GUARD_BYTES) / sizeof(float)) && ok;
+    ok = ds4_gpu_tensor_fill_f32(
+        f->out, -105.0f,
+        (f->out_bytes + GUARD_BYTES) / sizeof(float)) && ok;
+    ok = write_guard(f->gate, f->pair_f32_bytes) && ok;
+    ok = write_guard(f->up, f->pair_f32_bytes) && ok;
+    /* The first guard proves the production F16 mid contract. Full F32
+     * capacity remains allocated so a wrong fallback is caught safely. */
+    ok = write_guard(f->mid, f->pair_f16_bytes) && ok;
+    ok = write_guard(f->mid, f->pair_f32_bytes) && ok;
+    ok = write_guard(f->experts, f->expert_bytes) && ok;
+    ok = write_guard(f->out, f->out_bytes) && ok;
+    return ok;
+}
+
+static int check_all_canaries(const fixture *f) {
+    int ok = check_guard("x", f->x, f->x_bytes);
+    ok = check_guard("selected", f->selected, f->route_i32_bytes) && ok;
+    ok = check_guard("weights", f->weights, f->route_f32_bytes) && ok;
+    ok = check_guard("gate", f->gate, f->pair_f32_bytes) && ok;
+    ok = check_guard("up", f->up, f->pair_f32_bytes) && ok;
+    ok = check_guard("mid-f16-boundary", f->mid, f->pair_f16_bytes) && ok;
+    ok = check_guard("mid-allocation-end", f->mid, f->pair_f32_bytes) && ok;
+    ok = check_guard("experts", f->experts, f->expert_bytes) && ok;
+    ok = check_guard("out", f->out, f->out_bytes) && ok;
+    return ok;
+}
+
+static const char *variant_name(bench_arm arm) {
+#ifdef DS4_METAL_IQ2_MOE_TOP8_PAIR_BENCH
+    return arm == ARM_BASELINE ? "separate" : "fused";
+#else
+    return arm == ARM_BASELINE ? "baseline" : "candidate";
+#endif
+}
+
+static int select_variant(bench_arm arm) {
+#ifdef DS4_METAL_IQ2_MOE_TOP8_PAIR_BENCH
+    ds4_gpu_test_set_flags(arm == ARM_CANDIDATE
+        ? DS4_GPU_TEST_REQUIRE_IQ2_TOP8_PAIR_SWIGLU : 0u);
+    return arm == ARM_BASELINE
+        ? setenv(PAIR_FUSION_DISABLE_ENV, "1", 1) == 0
+        : unsetenv(PAIR_FUSION_DISABLE_ENV) == 0;
+#else
+    if (unsetenv(PAIR_TAIL_ENABLE_ENV) != 0 ||
+        unsetenv(PAIR_TAIL_DISABLE_ENV) != 0) {
+        return 0;
+    }
+    return setenv(arm == ARM_BASELINE ? PAIR_TAIL_DISABLE_ENV :
+                  PAIR_TAIL_ENABLE_ENV, "1", 1) == 0;
+#endif
+}
+
+static int run_once(fixture *f, bench_arm arm,
+                    const char *phase, const char *order,
+                    uint32_t sample, uint32_t cycle, uint32_t position,
+                    bool poison, bool check_canaries) {
+    if (!select_variant(arm)) {
+        fprintf(stderr,
+                "metal-iq2-moe-tail-cull-bench: environment setup failed\n");
+        return 0;
+    }
+    if (poison && !poison_outputs(f)) {
+        fprintf(stderr,
+                "metal-iq2-moe-tail-cull-bench: output poison failed\n");
+        return 0;
+    }
+
+    fprintf(stderr,
+            "DS4_IQ2_MOE_TAIL_BENCH phase=%s experiment=%s variant=%s "
+            "sample=%u cycle=%u position=%u order=%s force_resident=1\n",
+            phase, EXPERIMENT_NAME, variant_name(arm), sample,
+            cycle, position, order);
+    fflush(stderr);
+
+    if (!ds4_gpu_begin_commands()) {
+        fprintf(stderr,
+                "metal-iq2-moe-tail-cull-bench: begin commands failed\n");
+        return 0;
+    }
+    bool mid_is_f16 = false;
+    const int call_ok = ds4_gpu_routed_moe_batch_tensor(
+        f->out, f->gate, f->up, f->mid, f->experts,
+        f->model, f->model_size,
+        f->gate_offset, f->up_offset, f->down_offset,
+        IQ2_XXS_TYPE, Q2_K_TYPE,
+        f->gate_expert_bytes, f->gate_row_bytes,
+        f->down_expert_bytes, f->down_row_bytes,
+        IN_DIM, MID_DIM, OUT_DIM,
+        f->selected, f->weights, N_TOTAL_EXPERT, N_EXPERT, CLAMP, f->x,
+        0u, N_TOKENS, &mid_is_f16, true);
+    const int end_ok = ds4_gpu_end_commands();
+    int ok = call_ok && end_ok && mid_is_f16;
+    if (!ok) {
+        fprintf(stderr,
+                "DS4_IQ2_MOE_TAIL_BENCH result=FAIL experiment=%s "
+                "variant=%s call=%d end=%d mid_f16=%d\n",
+                EXPERIMENT_NAME, variant_name(arm),
+                call_ok, end_ok, mid_is_f16 ? 1 : 0);
+    }
+    if (check_canaries) ok = check_all_canaries(f) && ok;
+    return ok;
+}
+
+static int snapshot_alloc(oracle_snapshot *snapshot, const fixture *f) {
+    memset(snapshot, 0, sizeof(*snapshot));
+    snapshot->bytes = f->pair_f16_bytes + f->expert_bytes + f->out_bytes;
+    if (snapshot->bytes > SIZE_MAX) return 0;
+    snapshot->storage = malloc((size_t)snapshot->bytes);
+    if (!snapshot->storage) return 0;
+    snapshot->mid = snapshot->storage;
+    snapshot->experts = snapshot->mid + f->pair_f16_bytes;
+    snapshot->out = snapshot->experts + f->expert_bytes;
+    return 1;
+}
+
+static int capture_snapshot(oracle_snapshot *snapshot, const fixture *f) {
+    return ds4_gpu_tensor_read(
+               f->mid, 0, snapshot->mid, f->pair_f16_bytes) &&
+           ds4_gpu_tensor_read(
+               f->experts, 0, snapshot->experts, f->expert_bytes) &&
+           ds4_gpu_tensor_read(
+               f->out, 0, snapshot->out, f->out_bytes);
+}
+
+static int tensor_matches(const char *candidate, const char *name,
+                          const ds4_gpu_tensor *tensor,
+                          const uint8_t *expected, uint64_t bytes,
+                          uint8_t *scratch, size_t scratch_bytes) {
+    uint64_t offset = 0;
+    while (offset < bytes) {
+        const size_t chunk = bytes - offset > scratch_bytes ?
+            scratch_bytes : (size_t)(bytes - offset);
+        if (!ds4_gpu_tensor_read(tensor, offset, scratch, chunk)) {
+            fprintf(stderr,
+                    "DS4_IQ2_MOE_TAIL_ORACLE candidate=%s tensor=%s "
+                    "result=READ_FAIL offset=%llu\n",
+                    candidate, name, (unsigned long long)offset);
+            return 0;
+        }
+        if (memcmp(scratch, expected + offset, chunk) != 0) {
+            size_t mismatch = 0;
+            while (mismatch < chunk &&
+                   scratch[mismatch] == expected[offset + mismatch]) {
+                mismatch++;
+            }
+            fprintf(stderr,
+                    "DS4_IQ2_MOE_TAIL_ORACLE candidate=%s tensor=%s "
+                    "result=MISMATCH byte=%llu expected=0x%02x actual=0x%02x\n",
+                    candidate, name,
+                    (unsigned long long)(offset + mismatch),
+                    expected[offset + mismatch], scratch[mismatch]);
+            return 0;
+        }
+        offset += chunk;
+    }
+    return 1;
+}
+
+static int compare_candidate(const char *candidate, const fixture *f,
+                             const oracle_snapshot *baseline,
+                             uint8_t *scratch) {
+    const int mid_ok = tensor_matches(
+        candidate, "mid_f16", f->mid, baseline->mid,
+        f->pair_f16_bytes, scratch, COMPARE_CHUNK_BYTES);
+    const int experts_ok = tensor_matches(
+        candidate, "experts_f32", f->experts, baseline->experts,
+        f->expert_bytes, scratch, COMPARE_CHUNK_BYTES);
+    const int out_ok = tensor_matches(
+        candidate, "out_f32", f->out, baseline->out,
+        f->out_bytes, scratch, COMPARE_CHUNK_BYTES);
+    const int ok = mid_ok && experts_ok && out_ok;
+    fprintf(stderr,
+            "DS4_IQ2_MOE_TAIL_ORACLE candidate=%s result=%s "
+            "mid_f16=%s experts_f32=%s out_f32=%s canaries=PASS\n",
+            candidate, ok ? "PASS" : "FAIL",
+            mid_ok ? "exact" : "mismatch",
+            experts_ok ? "exact" : "mismatch",
+            out_ok ? "exact" : "mismatch");
+    return ok;
+}
+
+static int run_oracle(fixture *f) {
+    oracle_snapshot baseline;
+    if (!snapshot_alloc(&baseline, f)) {
+        fprintf(stderr,
+                "metal-iq2-moe-tail-cull-bench: oracle snapshot allocation "
+                "failed\n");
+        return 0;
+    }
+    uint8_t *scratch = malloc(COMPARE_CHUNK_BYTES);
+    int ok = scratch != NULL;
+
+    if (ok) {
+        ok = run_once(f, ARM_BASELINE,
+                      "oracle", "baseline", 0u, 0u, 0u, true, true);
+    }
+    if (ok) ok = capture_snapshot(&baseline, f);
+
+    if (ok) {
+        ok = run_once(f, ARM_CANDIDATE,
+                      "oracle", "pair", 0u, 0u, 0u, true, true);
+    }
+    if (ok) ok = compare_candidate(EXPERIMENT_NAME, f, &baseline, scratch);
+
+    free(scratch);
+    free(baseline.storage);
+    return ok;
+}
+
+static int run_balanced_block(fixture *f, const char *phase, uint32_t cycles,
+                              uint32_t sample_limit) {
+    uint32_t arm_samples[2] = {0, 0};
+    for (uint32_t cycle = 0; cycle < cycles; cycle++) {
+        static const bench_arm abba[4] = {
+            ARM_BASELINE, ARM_CANDIDATE,
+            ARM_CANDIDATE, ARM_BASELINE,
+        };
+        static const bench_arm baab[4] = {
+            ARM_CANDIDATE, ARM_BASELINE,
+            ARM_BASELINE, ARM_CANDIDATE,
+        };
+        const bench_arm *order = (cycle & 1u) ? baab : abba;
+        const char *order_name = (cycle & 1u) ? "BAAB" : "ABBA";
+        for (uint32_t position = 0; position < 4u; position++) {
+            const bench_arm arm = order[position];
+            if (!run_once(f, arm, phase, order_name,
+                          arm_samples[arm]++, cycle, position,
+                          false, false)) {
+                return 0;
+            }
+        }
+    }
+    if (arm_samples[ARM_BASELINE] != sample_limit ||
+        arm_samples[ARM_CANDIDATE] != sample_limit) {
+        fprintf(stderr,
+                "metal-iq2-moe-tail-cull-bench: %s %s arm count "
+                "baseline=%u candidate=%u expected=%u\n",
+                EXPERIMENT_NAME, phase,
+                arm_samples[ARM_BASELINE], arm_samples[ARM_CANDIDATE],
+                sample_limit);
+        return 0;
+    }
+    return 1;
+}
+
+static int run_experiment(fixture *f, const bench_config *config) {
+    if (config->warmup_cycles != 0u &&
+        !run_balanced_block(f, "warmup",
+                            config->warmup_cycles,
+                            config->warmup_cycles * 2u)) {
+        return 0;
+    }
+    if (!run_balanced_block(f, "sample",
+                            config->samples / 2u, config->samples)) {
+        return 0;
+    }
+    fprintf(stderr,
+            "DS4_IQ2_MOE_TAIL_BENCH phase=complete experiment=%s "
+            "samples_per_variant=%u warmup_per_variant=%u result=PASS\n",
+            EXPERIMENT_NAME, config->samples,
+            config->warmup_cycles * 2u);
+    return 1;
+}
+
+static void free_tensors(fixture *f) {
+    ds4_gpu_tensor_free(f->out);
+    ds4_gpu_tensor_free(f->experts);
+    ds4_gpu_tensor_free(f->mid);
+    ds4_gpu_tensor_free(f->up);
+    ds4_gpu_tensor_free(f->gate);
+    ds4_gpu_tensor_free(f->weights);
+    ds4_gpu_tensor_free(f->selected);
+    ds4_gpu_tensor_free(f->x);
+    f->out = NULL;
+    f->experts = NULL;
+    f->mid = NULL;
+    f->up = NULL;
+    f->gate = NULL;
+    f->weights = NULL;
+    f->selected = NULL;
+    f->x = NULL;
+}
+
+static int init_fixture(fixture *f) {
+    memset(f, 0, sizeof(*f));
+    const uint64_t page = (uint64_t)getpagesize();
+    f->gate_row_bytes =
+        (uint64_t)(IN_DIM / QK_K) * sizeof(block_iq2_xxs);
+    f->gate_expert_bytes = (uint64_t)MID_DIM * f->gate_row_bytes;
+    const uint64_t gate_tensor_bytes =
+        (uint64_t)N_TOTAL_EXPERT * f->gate_expert_bytes;
+    f->down_row_bytes =
+        (uint64_t)(MID_DIM / QK_K) * sizeof(block_q2_K);
+    f->down_expert_bytes = (uint64_t)OUT_DIM * f->down_row_bytes;
+    const uint64_t down_tensor_bytes =
+        (uint64_t)N_TOTAL_EXPERT * f->down_expert_bytes;
+    f->gate_offset = 0;
+    f->up_offset = align_up(gate_tensor_bytes, page);
+    f->down_offset = align_up(f->up_offset + gate_tensor_bytes, page);
+    f->model_size = align_up(f->down_offset + down_tensor_bytes, page);
+
+    f->x_bytes = (uint64_t)N_TOKENS * IN_DIM * sizeof(float);
+    f->route_count = (uint64_t)N_TOKENS * N_EXPERT;
+    f->route_i32_bytes = f->route_count * sizeof(int32_t);
+    f->route_f32_bytes = f->route_count * sizeof(float);
+    f->pair_count = f->route_count * MID_DIM;
+    f->pair_f16_bytes = f->pair_count * sizeof(_Float16);
+    f->pair_f32_bytes = f->pair_count * sizeof(float);
+    f->expert_count = (uint64_t)N_TOKENS * N_EXPERT * OUT_DIM;
+    f->expert_bytes = f->expert_count * sizeof(float);
+    f->out_count = (uint64_t)N_TOKENS * OUT_DIM;
+    f->out_bytes = f->out_count * sizeof(float);
+
+    if (f->gate_row_bytes != 1056u ||
+        f->gate_expert_bytes != 2162688u ||
+        f->down_row_bytes != 672u ||
+        f->down_expert_bytes != 2752512u) {
+        fprintf(stderr,
+                "metal-iq2-moe-tail-cull-bench: production layout mismatch "
+                "gate_row=%llu gate_expert=%llu down_row=%llu "
+                "down_expert=%llu\n",
+                (unsigned long long)f->gate_row_bytes,
+                (unsigned long long)f->gate_expert_bytes,
+                (unsigned long long)f->down_row_bytes,
+                (unsigned long long)f->down_expert_bytes);
+        return 0;
+    }
+
+    const uint64_t tensor_bytes =
+        f->x_bytes + f->route_i32_bytes + f->route_f32_bytes +
+        3u * (f->pair_f32_bytes + GUARD_BYTES) +
+        f->expert_bytes + GUARD_BYTES + f->out_bytes + GUARD_BYTES +
+        3u * GUARD_BYTES;
+    const uint64_t oracle_bytes =
+        f->pair_f16_bytes + f->expert_bytes + f->out_bytes +
+        COMPARE_CHUNK_BYTES;
+    const uint64_t setup_host_bytes =
+        f->x_bytes + f->route_i32_bytes + f->route_f32_bytes;
+    const uint64_t explicit_peak =
+        f->model_size + tensor_bytes + oracle_bytes + setup_host_bytes;
+    fprintf(stderr,
+            "DS4_IQ2_MOE_TAIL_SETUP geometry=N%u,d%u,mid%u,out%u,"
+            "experts%u,top%u model=%.3f_GiB tensors=%.3f_GiB "
+            "oracle=%.3f_GiB explicit_peak=%.3f_GiB\n",
+            N_TOKENS, IN_DIM, MID_DIM, OUT_DIM,
+            N_TOTAL_EXPERT, N_EXPERT,
+            (double)f->model_size / (double)GIB,
+            (double)tensor_bytes / (double)GIB,
+            (double)oracle_bytes / (double)GIB,
+            (double)explicit_peak / (double)GIB);
+    if (explicit_peak >= 5u * GIB) {
+        fprintf(stderr,
+                "metal-iq2-moe-tail-cull-bench: explicit peak exceeds "
+                "5 GiB\n");
+        return 0;
+    }
+
+    if (posix_memalign(&f->model, (size_t)page,
+                       (size_t)f->model_size) != 0) {
+        fprintf(stderr,
+                "metal-iq2-moe-tail-cull-bench: model allocation failed\n");
+        return 0;
+    }
+    fprintf(stderr,
+            "DS4_IQ2_MOE_TAIL_SETUP phase=fill_weights tensor=gate\n");
+    fill_iq2((block_iq2_xxs *)((uint8_t *)f->model + f->gate_offset),
+             19u);
+    fprintf(stderr,
+            "DS4_IQ2_MOE_TAIL_SETUP phase=fill_weights tensor=up\n");
+    fill_iq2((block_iq2_xxs *)((uint8_t *)f->model + f->up_offset),
+             47u);
+    fprintf(stderr,
+            "DS4_IQ2_MOE_TAIL_SETUP phase=fill_weights tensor=down\n");
+    fill_q2((block_q2_K *)((uint8_t *)f->model + f->down_offset));
+    const uint64_t checksum =
+        touch_model_pages(f->model, f->model_size, page);
+    fprintf(stderr,
+            "DS4_IQ2_MOE_TAIL_SETUP phase=touch_weights pages=%llu "
+            "checksum=0x%016llx\n",
+            (unsigned long long)((f->model_size + page - 1u) / page),
+            (unsigned long long)checksum);
+
+    float *x_host = malloc((size_t)f->x_bytes);
+    int32_t *selected_host = malloc((size_t)f->route_i32_bytes);
+    float *weights_host = malloc((size_t)f->route_f32_bytes);
+    int ok = x_host && selected_host && weights_host;
+    if (ok) fill_input(x_host);
+    if (ok) ok = build_routes(selected_host, weights_host);
+    if (ok) ok = ds4_gpu_set_model_map(f->model, f->model_size);
+
+    if (ok) f->x = ds4_gpu_tensor_alloc(f->x_bytes + GUARD_BYTES);
+    if (ok) f->selected =
+        ds4_gpu_tensor_alloc(f->route_i32_bytes + GUARD_BYTES);
+    if (ok) f->weights =
+        ds4_gpu_tensor_alloc(f->route_f32_bytes + GUARD_BYTES);
+    if (ok) f->gate =
+        ds4_gpu_tensor_alloc(f->pair_f32_bytes + GUARD_BYTES);
+    if (ok) f->up =
+        ds4_gpu_tensor_alloc(f->pair_f32_bytes + GUARD_BYTES);
+    if (ok) f->mid =
+        ds4_gpu_tensor_alloc(f->pair_f32_bytes + GUARD_BYTES);
+    if (ok) f->experts =
+        ds4_gpu_tensor_alloc(f->expert_bytes + GUARD_BYTES);
+    if (ok) f->out = ds4_gpu_tensor_alloc(f->out_bytes + GUARD_BYTES);
+    ok = ok && f->x && f->selected && f->weights && f->gate && f->up &&
+         f->mid && f->experts && f->out;
+
+    if (ok) ok = ds4_gpu_tensor_write(f->x, 0, x_host, f->x_bytes);
+    if (ok) ok = ds4_gpu_tensor_write(
+        f->selected, 0, selected_host, f->route_i32_bytes);
+    if (ok) ok = ds4_gpu_tensor_write(
+        f->weights, 0, weights_host, f->route_f32_bytes);
+    if (ok) ok = write_guard(f->x, f->x_bytes);
+    if (ok) ok = write_guard(f->selected, f->route_i32_bytes);
+    if (ok) ok = write_guard(f->weights, f->route_f32_bytes);
+    if (ok) ok = poison_outputs(f);
+
+    free(weights_host);
+    free(selected_host);
+    free(x_host);
+    if (!ok) {
+        fprintf(stderr,
+                "metal-iq2-moe-tail-cull-bench: fixture initialization "
+                "failed\n");
+    }
+    return ok;
+}
+
+int main(int argc, char **argv) {
+    const bench_config config = parse_options(argc, argv);
+
+    /* This benchmark compares the legacy grouped pair kernels directly.
+     * Prevent Metal 4 TensorOps/MPP from bypassing either A/B variant on
+     * M5+ hosts; production defaults remain independently hardware-gated. */
+    setenv("DS4_METAL_DISABLE_METAL4", "1", 1);
+    setenv("DS4_METAL_MOE_STAGE_PROFILE", "1", 1);
+    setenv("DS4_METAL_MOE_STAGE_PROFILE_LAYER", "0", 1);
+    unsetenv("DS4_METAL_MOE_STAGE_PROFILE_FILTER");
+    unsetenv(PAIR_FUSION_DISABLE_ENV);
+    unsetenv("DS4_METAL_MOE_WRITE_CLAMPED_ACT");
+    unsetenv("DS4_METAL_GRAPH_DUMP_PREFIX");
+    unsetenv(PAIR_TAIL_ENABLE_ENV);
+    unsetenv(PAIR_TAIL_DISABLE_ENV);
+
+    if (!ds4_gpu_init()) {
+        fprintf(stderr,
+                "metal-iq2-moe-tail-cull-bench: Metal initialization failed\n");
+        return 1;
+    }
+    ds4_gpu_set_quality(false);
+    ds4_gpu_set_ssd_streaming(false);
+
+    fixture f;
+    int ok = init_fixture(&f);
+    if (ok) ok = run_oracle(&f);
+    if (ok) ok = run_experiment(&f, &config);
+    if (ok) ok = check_all_canaries(&f);
+    (void)select_variant(ARM_BASELINE);
+
+    free_tensors(&f);
+    ds4_gpu_cleanup();
+    free(f.model);
+    fprintf(stderr,
+            "DS4_IQ2_MOE_TAIL_BENCH result=%s\n",
+            ok ? "PASS" : "FAIL");
+    return ok ? 0 : 1;
+}
diff --git a/speed-bench/metal_iq2_moe_top8_pair_bench.c b/speed-bench/metal_iq2_moe_top8_pair_bench.c
new file mode 100644
index 0000000000..e53ec3916e
--- /dev/null
+++ b/speed-bench/metal_iq2_moe_top8_pair_bench.c
@@ -0,0 +1,2 @@
+#define DS4_METAL_IQ2_MOE_TOP8_PAIR_BENCH 1
+#include "metal_iq2_moe_tail_cull_bench.c"
diff --git a/speed-bench/metal_prefill_variant_bench.c b/speed-bench/metal_prefill_variant_bench.c
index 82f2184140..f2f1c8ed38 100644
--- a/speed-bench/metal_prefill_variant_bench.c
+++ b/speed-bench/metal_prefill_variant_bench.c
@@ -27,6 +27,10 @@ typedef struct {
     int warmup_tokens;
     int ctx;
     int repeats;
+    uint32_t ssd_cache_experts;
+    uint32_t ssd_preload_experts;
+    bool ssd_streaming;
+    bool ssd_streaming_cold;
 } bench_config;
 
 typedef struct {
@@ -45,7 +49,13 @@ static void usage(FILE *fp, const char *argv0) {
             "  --prefix-tokens N      timed prefill length (default: 8192)\n"
             "  --warmup-tokens N      untimed tokens per variant (default: 32; min: 32)\n"
             "  --ctx N                session allocation (default: max lengths + 1)\n"
-            "  --repeats N            alternating ABBA/BAAB pairs (default: 2)\n",
+            "  --repeats N            alternating ABBA/BAAB pairs (default: 2)\n"
+            "  --ssd-streaming        use the SSD-backed model path\n"
+            "  --ssd-streaming-cold   skip the default expert-cache preload\n"
+            "  --ssd-streaming-cache-experts N\n"
+            "                         dynamic expert-cache entry count\n"
+            "  --ssd-streaming-preload-experts N\n"
+            "                         popularity preload count\n",
             argv0);
 }
 
@@ -82,6 +92,10 @@ static bench_config parse_options(int argc, char **argv) {
         .warmup_tokens = DEFAULT_WARMUP_TOKENS,
         .ctx = 0,
         .repeats = DEFAULT_REPEATS,
+        .ssd_cache_experts = 0,
+        .ssd_preload_experts = 0,
+        .ssd_streaming = false,
+        .ssd_streaming_cold = false,
     };
 
     for (int i = 1; i < argc; i++) {
@@ -106,6 +120,16 @@ static bench_config parse_options(int argc, char **argv) {
         } else if (!strcmp(arg, "--repeats")) {
             cfg.repeats =
                 parse_int_arg(need_arg(&i, argc, argv, arg), arg, 1);
+        } else if (!strcmp(arg, "--ssd-streaming")) {
+            cfg.ssd_streaming = true;
+        } else if (!strcmp(arg, "--ssd-streaming-cold")) {
+            cfg.ssd_streaming_cold = true;
+        } else if (!strcmp(arg, "--ssd-streaming-cache-experts")) {
+            cfg.ssd_cache_experts = (uint32_t)parse_int_arg(
+                need_arg(&i, argc, argv, arg), arg, 1);
+        } else if (!strcmp(arg, "--ssd-streaming-preload-experts")) {
+            cfg.ssd_preload_experts = (uint32_t)parse_int_arg(
+                need_arg(&i, argc, argv, arg), arg, 1);
         } else {
             fprintf(stderr, "%s: unknown option: %s\n", BENCH_NAME, arg);
             usage(stderr, argv[0]);
@@ -118,6 +142,14 @@ static bench_config parse_options(int argc, char **argv) {
         fprintf(stderr, "%s: --candidate-env requires a valid name\n", BENCH_NAME);
         exit(2);
     }
+    if (!cfg.ssd_streaming &&
+        (cfg.ssd_streaming_cold || cfg.ssd_cache_experts != 0 ||
+         cfg.ssd_preload_experts != 0)) {
+        fprintf(stderr,
+                "%s: SSD cache options require --ssd-streaming\n",
+                BENCH_NAME);
+        exit(2);
+    }
     const int longest =
         cfg.prefix_tokens > cfg.warmup_tokens
             ? cfg.prefix_tokens
@@ -293,7 +325,11 @@ int main(int argc, char **argv) {
         .context_size = cfg.ctx,
         .prefill_chunk = 4096,
         .power_percent = 100,
-        .warm_weights = true,
+        .warm_weights = !cfg.ssd_streaming,
+        .ssd_streaming = cfg.ssd_streaming,
+        .ssd_streaming_cold = cfg.ssd_streaming_cold,
+        .ssd_streaming_cache_experts = cfg.ssd_cache_experts,
+        .ssd_streaming_preload_experts = cfg.ssd_preload_experts,
     };
     ds4_engine *engine = NULL;
     ds4_tokens tokens = {0};
@@ -333,7 +369,8 @@ int main(int argc, char **argv) {
 
     fprintf(stderr,
             "%s: model=%s prompt=%s prefix=%d warmup=%d ctx=%d repeats=%d "
-            "candidate_env=%s\n",
+            "candidate_env=%s ssd_streaming=%s ssd_cold=%s "
+            "cache_experts=%u preload_experts=%u\n",
             BENCH_NAME,
             cfg.model_path,
             cfg.prompt_path,
@@ -341,7 +378,11 @@ int main(int argc, char **argv) {
             cfg.warmup_tokens,
             cfg.ctx,
             cfg.repeats,
-            cfg.candidate_env);
+            cfg.candidate_env,
+            cfg.ssd_streaming ? "yes" : "no",
+            cfg.ssd_streaming_cold ? "yes" : "no",
+            cfg.ssd_cache_experts,
+            cfg.ssd_preload_experts);
 
     for (int variant = 0; variant < VARIANT_COUNT; variant++) {
         err[0] = '\0';
diff --git a/speed-bench/metal_q4_attn_out_a_direct_bench b/speed-bench/metal_q4_attn_out_a_direct_bench
new file mode 100755
index 0000000000..21d3e78e1e
Binary files /dev/null and b/speed-bench/metal_q4_attn_out_a_direct_bench differ
diff --git a/speed-bench/metal_q4_attn_out_a_direct_bench.m b/speed-bench/metal_q4_attn_out_a_direct_bench.m
new file mode 100644
index 0000000000..3f13528bb2
--- /dev/null
+++ b/speed-bench/metal_q4_attn_out_a_direct_bench.m
@@ -0,0 +1,995 @@
+#import <Foundation/Foundation.h>
+#import <Metal/Metal.h>
+
+#include <errno.h>
+#include <math.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+/* Resident, production-shape comparison for Q4 attention output-A.  The
+ * measured command buffers contain only Metal dispatches; fixture creation,
+ * map construction for the routed-only arm, and all validation stay outside
+ * the timed region. */
+enum {
+    K_DIM = 4096,
+    M_DIM = 1024,
+    GROUPS = 8,
+    QK_K = 256,
+    Q4_BLOCK_BYTES = 144,
+    THREADS_PER_GROUP = 128,
+    THREADGROUP_MEMORY_BYTES = 8192,
+    GUARD_BYTES = 4096,
+    MIN_TOKENS = 512,
+    MAX_TOKENS = 4096,
+    MAX_TOKEN_CASES = 16,
+    MAX_SAMPLES = 64,
+    MAX_WARMUP = 32,
+    DEFAULT_SAMPLES = 8,
+    DEFAULT_WARMUP = 2,
+};
+
+static const uint32_t k_guard = 0x7fc12345u;
+static const uint32_t k_poison_current = 0x7fc0a001u;
+static const uint32_t k_poison_routed = 0x7fc0b001u;
+static const uint32_t k_poison_direct = 0x7fc0c001u;
+static const uint32_t k_default_tokens[] = {512u, 1024u, 2048u, 4096u};
+
+typedef struct {
+    uint16_t d;
+    uint16_t dmin;
+    uint8_t scales[12];
+    uint8_t qs[QK_K / 2];
+} block_q4_K_host;
+
+typedef struct {
+    int32_t ne02;
+    int32_t ne10;
+    int32_t ne11;
+    uint64_t nb11;
+    uint64_t nb12;
+    int32_t ne21;
+    int32_t ne20;
+    uint64_t nb21;
+} map_args;
+
+typedef struct {
+    int32_t ne00;
+    int32_t ne02;
+    uint64_t nb01;
+    uint64_t nb02;
+    uint64_t nb03;
+    int32_t ne11;
+    uint64_t nb10;
+    uint64_t nb11;
+    uint64_t nb12;
+    uint64_t nb13;
+    int32_t ne20;
+    int32_t ne21;
+    int32_t ne0;
+    int32_t ne1;
+    int16_t r2;
+    int16_t r3;
+    int32_t tp_rank;
+    int32_t tp_world;
+    int32_t tp_expert_base;
+} mm_args;
+
+_Static_assert(sizeof(block_q4_K_host) == Q4_BLOCK_BYTES,
+               "Q4_K host fixture must match the Metal ABI");
+_Static_assert(sizeof(map_args) == 48, "map argument ABI changed");
+_Static_assert(sizeof(mm_args) == 104, "routed-MM argument ABI changed");
+
+typedef struct {
+    NSUInteger tpe_bytes;
+    NSUInteger hids_bytes;
+    NSUInteger work_offset;
+    NSUInteger total_bytes;
+    NSUInteger work_cap;
+} map_layout;
+
+typedef struct {
+    uint32_t tokens[MAX_TOKEN_CASES];
+    uint32_t token_count;
+    uint32_t samples;
+    uint32_t warmup;
+} bench_config;
+
+typedef enum {
+    ARM_CURRENT,
+    ARM_ROUTED_ONLY,
+    ARM_DIRECT,
+} bench_arm;
+
+typedef struct {
+    __strong id<MTLDevice> device;
+    __strong id<MTLCommandQueue> queue;
+    __strong id<MTLComputePipelineState> map_pipeline;
+    __strong id<MTLComputePipelineState> routed_pipeline;
+    __strong id<MTLComputePipelineState> direct_pipeline;
+    __strong id<MTLBuffer> weights;
+    NSUInteger weights_bytes;
+    uint64_t weights_hash;
+} fixture;
+
+typedef struct {
+    __strong id<MTLBuffer> heads;
+    __strong id<MTLBuffer> current_out;
+    __strong id<MTLBuffer> routed_out;
+    __strong id<MTLBuffer> direct_out;
+    __strong id<MTLBuffer> ids;
+    __strong id<MTLBuffer> current_map;
+    __strong id<MTLBuffer> prebuilt_map;
+    NSUInteger heads_bytes;
+    NSUInteger output_bytes;
+    NSUInteger ids_bytes;
+    map_layout layout;
+    map_args map;
+    mm_args mm;
+    uint64_t heads_hash;
+    uint64_t ids_hash;
+    uint64_t prebuilt_map_hash;
+    uint32_t tokens;
+} case_buffers;
+
+typedef struct {
+    double first[MAX_SAMPLES];
+    double second[MAX_SAMPLES];
+} pair_samples;
+
+static void usage(FILE *fp, const char *argv0) {
+    fprintf(fp,
+            "usage: %s [options]\n"
+            "\n"
+            "Resident Metal Q4_K attention output-A kernel comparison.\n"
+            "\n"
+            "  --n LIST       comma-separated token counts, %u..%u\n"
+            "                 (default: 512,1024,2048,4096)\n"
+            "  --samples N    samples per arm, multiple of 4, max %u\n"
+            "                 (default: %u)\n"
+            "  --warmup N     warmup dispatches per arm, max %u\n"
+            "                 (default: %u)\n"
+            "  -h, --help     show this help\n"
+            "\n"
+            "The fixed production geometry is 4096 -> 1024 across 8 groups.\n"
+            "Set DS4_SOURCE_ROOT when running outside the repository root.\n",
+            argv0, MIN_TOKENS, MAX_TOKENS, MAX_SAMPLES, DEFAULT_SAMPLES,
+            MAX_WARMUP, DEFAULT_WARMUP);
+}
+
+static const char *need_arg(int *index, int argc, char **argv) {
+    if (*index + 1 >= argc) {
+        fprintf(stderr,
+                "metal-q4-attn-out-a-direct-bench: %s needs a value\n",
+                argv[*index]);
+        exit(2);
+    }
+    return argv[++*index];
+}
+
+static uint32_t parse_u32(const char *text, const char *option,
+                          uint32_t minimum, uint32_t maximum) {
+    char *end = NULL;
+    errno = 0;
+    const unsigned long long value = strtoull(text, &end, 10);
+    if (errno != 0 || !text[0] || !end || *end ||
+        value < minimum || value > maximum) {
+        fprintf(stderr,
+                "metal-q4-attn-out-a-direct-bench: invalid %s: %s\n",
+                option, text);
+        exit(2);
+    }
+    return (uint32_t)value;
+}
+
+static void parse_token_list(bench_config *config, const char *text) {
+    const char *cursor = text;
+    config->token_count = 0u;
+    while (*cursor) {
+        char *end = NULL;
+        errno = 0;
+        const unsigned long long value = strtoull(cursor, &end, 10);
+        if (errno != 0 || end == cursor || value < MIN_TOKENS ||
+            value > MAX_TOKENS || (*end != ',' && *end != '\0') ||
+            config->token_count == MAX_TOKEN_CASES) {
+            fprintf(stderr,
+                    "metal-q4-attn-out-a-direct-bench: invalid --n list: %s\n",
+                    text);
+            exit(2);
+        }
+        for (uint32_t i = 0; i < config->token_count; i++) {
+            if (config->tokens[i] == (uint32_t)value) {
+                fprintf(stderr,
+                        "metal-q4-attn-out-a-direct-bench: duplicate N=%llu\n",
+                        value);
+                exit(2);
+            }
+        }
+        config->tokens[config->token_count++] = (uint32_t)value;
+        if (*end == '\0') break;
+        cursor = end + 1;
+        if (!*cursor) {
+            fprintf(stderr,
+                    "metal-q4-attn-out-a-direct-bench: invalid --n list: %s\n",
+                    text);
+            exit(2);
+        }
+    }
+    if (config->token_count == 0u) {
+        fprintf(stderr,
+                "metal-q4-attn-out-a-direct-bench: --n list is empty\n");
+        exit(2);
+    }
+}
+
+static bench_config parse_options(int argc, char **argv) {
+    bench_config config = {
+        .samples = DEFAULT_SAMPLES,
+        .warmup = DEFAULT_WARMUP,
+    };
+    config.token_count = (uint32_t)(sizeof(k_default_tokens) /
+                                     sizeof(k_default_tokens[0]));
+    memcpy(config.tokens, k_default_tokens, sizeof(k_default_tokens));
+
+    for (int i = 1; i < argc; i++) {
+        if (!strcmp(argv[i], "-h") || !strcmp(argv[i], "--help")) {
+            usage(stdout, argv[0]);
+            exit(0);
+        } else if (!strcmp(argv[i], "--n")) {
+            parse_token_list(&config, need_arg(&i, argc, argv));
+        } else if (!strcmp(argv[i], "--samples")) {
+            const char *option = argv[i];
+            config.samples = parse_u32(need_arg(&i, argc, argv), option,
+                                       4u, MAX_SAMPLES);
+        } else if (!strcmp(argv[i], "--warmup")) {
+            const char *option = argv[i];
+            config.warmup = parse_u32(need_arg(&i, argc, argv), option,
+                                      0u, MAX_WARMUP);
+        } else {
+            fprintf(stderr,
+                    "metal-q4-attn-out-a-direct-bench: unknown option: %s\n",
+                    argv[i]);
+            usage(stderr, argv[0]);
+            exit(2);
+        }
+    }
+    if ((config.samples % 4u) != 0u) {
+        fprintf(stderr,
+                "metal-q4-attn-out-a-direct-bench: --samples must be "
+                "divisible by 4 for balanced ABBA/BAAB blocks\n");
+        exit(2);
+    }
+    return config;
+}
+
+static bool checked_add(NSUInteger a, NSUInteger b, NSUInteger *out) {
+    if (b > NSUIntegerMax - a) return false;
+    *out = a + b;
+    return true;
+}
+
+static bool checked_mul(NSUInteger a, NSUInteger b, NSUInteger *out) {
+    if (a != 0u && b > NSUIntegerMax / a) return false;
+    *out = a * b;
+    return true;
+}
+
+static NSUInteger align_up(NSUInteger value, NSUInteger alignment) {
+    return (value + alignment - 1u) / alignment * alignment;
+}
+
+static uint32_t lcg_next(uint32_t *state) {
+    *state = *state * 1664525u + 1013904223u;
+    return *state;
+}
+
+static uint64_t hash_bytes(const void *raw, NSUInteger bytes) {
+    const uint8_t *data = raw;
+    uint64_t hash = UINT64_C(1469598103934665603);
+    for (NSUInteger i = 0; i < bytes; i++) {
+        hash ^= data[i];
+        hash *= UINT64_C(1099511628211);
+    }
+    return hash;
+}
+
+static bool is_pre_m5_apple_silicon_name(const char *name) {
+    return name && !strncmp(name, "Apple M", 7) &&
+           name[7] >= '1' && name[7] <= '4' &&
+           (name[8] == '\0' || name[8] == ' ');
+}
+
+static NSString *metal_prelude(void) {
+    return @"#include <metal_stdlib>\n"
+            "using namespace metal;\n"
+            "#define MAX(x, y) ((x) > (y) ? (x) : (y))\n"
+            "#define MIN(x, y) ((x) < (y) ? (x) : (y))\n"
+            "#define SWAP(x, y) { auto tmp = (x); (x) = (y); (y) = tmp; }\n"
+            "#define QK8_0 32\n"
+            "#ifndef QK_K\n#define QK_K 256\n#endif\n"
+            "#define N_SIMDWIDTH 32\n"
+            "#define N_R0_Q8_0 2\n"
+            "#define N_SG_Q8_0 4\n"
+            "#define FC_MUL_MV 600\n"
+            "#define FC_MUL_MM 700\n"
+            "#define FC_BIN 1300\n"
+            "#define FOR_UNROLL(x) _Pragma(\"clang loop unroll(full)\") for (x)\n"
+            "#define M_PI_F 3.14159265358979323846f\n"
+            "enum ds4_sort_order { DS4_SORT_ORDER_ASC, DS4_SORT_ORDER_DESC };\n"
+            "struct block_q8_0 { half d; int8_t qs[QK8_0]; };\n"
+            "struct block_q8_K { float d; int8_t qs[QK_K]; "
+            "int16_t bsums[QK_K / 16]; };\n";
+}
+
+static NSString *load_metal_source(void) {
+    static const char *paths[] = {
+        "metal/activations.metal",
+        "metal/flash_attn.metal",
+        "metal/dense.metal",
+        "metal/moe.metal",
+        "metal/dsv4_hc.metal",
+        "metal/unary.metal",
+        "metal/dsv4_kv.metal",
+        "metal/dsv4_rope.metal",
+        "metal/dsv4_misc.metal",
+        "metal/argsort.metal",
+        "metal/cpy.metal",
+        "metal/concat.metal",
+        "metal/get_rows.metal",
+        "metal/sum_rows.metal",
+        "metal/softmax.metal",
+        "metal/repeat.metal",
+        "metal/glu.metal",
+        "metal/norm.metal",
+        "metal/bin.metal",
+        "metal/set_rows.metal",
+    };
+    const char *root_env = getenv("DS4_SOURCE_ROOT");
+    NSString *root = root_env && root_env[0]
+        ? [NSString stringWithUTF8String:root_env]
+        : [[NSFileManager defaultManager] currentDirectoryPath];
+    NSMutableString *source = [NSMutableString stringWithString:metal_prelude()];
+    for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); i++) {
+        NSString *relative = [NSString stringWithUTF8String:paths[i]];
+        NSString *path = [root stringByAppendingPathComponent:relative];
+        NSError *error = nil;
+        NSString *part = [NSString stringWithContentsOfFile:path
+                                                   encoding:NSUTF8StringEncoding
+                                                      error:&error];
+        if (!part) {
+            fprintf(stderr,
+                    "metal-q4-attn-out-a-direct-bench: cannot read %s: %s\n",
+                    [path fileSystemRepresentation],
+                    [[error localizedDescription] UTF8String]);
+            return nil;
+        }
+        [source appendFormat:@"\n// appended %@\n%@\n", relative, part];
+    }
+    return source;
+}
+
+static id<MTLComputePipelineState> make_pipeline(
+        id<MTLDevice> device, id<MTLLibrary> library, NSString *name,
+        bool routed_constants) {
+    NSError *error = nil;
+    id<MTLFunction> function = nil;
+    if (routed_constants) {
+        bool bc_inp = false;
+        MTLFunctionConstantValues *values = [MTLFunctionConstantValues new];
+        [values setConstantValue:&bc_inp
+                            type:MTLDataTypeBool
+                         atIndex:700];
+        function = [library newFunctionWithName:name
+                                  constantValues:values
+                                           error:&error];
+    } else {
+        function = [library newFunctionWithName:name];
+    }
+    if (!function) {
+        fprintf(stderr,
+                "metal-q4-attn-out-a-direct-bench: function %s: %s\n",
+                [name UTF8String],
+                error ? [[error localizedDescription] UTF8String] : "missing");
+        return nil;
+    }
+    id<MTLComputePipelineState> pipeline =
+        [device newComputePipelineStateWithFunction:function error:&error];
+    if (!pipeline) {
+        fprintf(stderr,
+                "metal-q4-attn-out-a-direct-bench: pipeline %s: %s\n",
+                [name UTF8String], [[error localizedDescription] UTF8String]);
+    }
+    return pipeline;
+}
+
+static id<MTLBuffer> guarded_buffer(id<MTLDevice> device,
+                                    NSUInteger payload_bytes,
+                                    NSString *label) {
+    NSUInteger total = 0u;
+    if (!checked_add(payload_bytes, 2u * GUARD_BYTES, &total)) return nil;
+    id<MTLBuffer> buffer =
+        [device newBufferWithLength:total options:MTLResourceStorageModeShared];
+    if (!buffer) {
+        fprintf(stderr,
+                "metal-q4-attn-out-a-direct-bench: allocation failed for "
+                "%s (%llu bytes)\n",
+                [label UTF8String], (unsigned long long)total);
+        return nil;
+    }
+    buffer.label = label;
+    uint32_t *words = buffer.contents;
+    for (NSUInteger i = 0; i < total / sizeof(*words); i++) {
+        words[i] = k_guard;
+    }
+    return buffer;
+}
+
+static void *payload(id<MTLBuffer> buffer) {
+    return (uint8_t *)buffer.contents + GUARD_BYTES;
+}
+
+static bool check_canary(id<MTLBuffer> buffer, NSUInteger payload_bytes,
+                         const char *label, uint32_t tokens) {
+    const uint32_t *words = buffer.contents;
+    const NSUInteger suffix = (GUARD_BYTES + payload_bytes) / sizeof(*words);
+    for (NSUInteger i = 0; i < GUARD_BYTES / sizeof(*words); i++) {
+        if (words[i] != k_guard || words[suffix + i] != k_guard) {
+            fprintf(stderr,
+                    "metal-q4-attn-out-a-direct-bench: %s canary changed "
+                    "N=%u word=%llu\n",
+                    label, tokens, (unsigned long long)i);
+            return false;
+        }
+    }
+    return true;
+}
+
+static void fill_weights(fixture *f) {
+    block_q4_K_host *blocks = payload(f->weights);
+    const NSUInteger count = f->weights_bytes / sizeof(*blocks);
+    uint32_t state = 0x31415926u;
+    for (NSUInteger block = 0; block < count; block++) {
+        blocks[block].d =
+            (uint16_t)(0x2800u | (lcg_next(&state) & 0x01ffu));
+        blocks[block].dmin =
+            (uint16_t)(0x2000u | (lcg_next(&state) & 0x01ffu));
+        for (size_t i = 0; i < sizeof(blocks[block].scales); i++) {
+            blocks[block].scales[i] = (uint8_t)(lcg_next(&state) >> 24);
+        }
+        for (size_t i = 0; i < sizeof(blocks[block].qs); i++) {
+            blocks[block].qs[i] = (uint8_t)(lcg_next(&state) >> 24);
+        }
+    }
+}
+
+static bool init_fixture(fixture *f) {
+    f->device = MTLCreateSystemDefaultDevice();
+    if (!f->device) {
+        fprintf(stderr,
+                "metal-q4-attn-out-a-direct-bench: no Metal device\n");
+        return false;
+    }
+    if (!is_pre_m5_apple_silicon_name([f->device.name UTF8String])) {
+        fprintf(stderr,
+                "metal-q4-attn-out-a-direct-bench: production route requires "
+                "Apple M1-M4 (device: %s)\n",
+                [f->device.name UTF8String]);
+        return false;
+    }
+    f->queue = [f->device newCommandQueue];
+    NSString *source = load_metal_source();
+    if (!f->queue || !source) return false;
+
+    NSError *error = nil;
+    MTLCompileOptions *options = [MTLCompileOptions new];
+    id<MTLLibrary> library =
+        [f->device newLibraryWithSource:source options:options error:&error];
+    if (!library) {
+        fprintf(stderr,
+                "metal-q4-attn-out-a-direct-bench: Metal compile failed: %s\n",
+                [[error localizedDescription] UTF8String]);
+        return false;
+    }
+    f->map_pipeline = make_pipeline(
+        f->device, library, @"kernel_mul_mm_id_map0_ne20_8", false);
+    f->routed_pipeline = make_pipeline(
+        f->device, library, @"kernel_mul_mm_id_q4_K_f32", true);
+    f->direct_pipeline = make_pipeline(
+        f->device, library,
+        @"kernel_attn_out_low_q4_K_legacy_direct", false);
+    if (!f->map_pipeline || !f->routed_pipeline || !f->direct_pipeline ||
+        f->map_pipeline.maxTotalThreadsPerThreadgroup < GROUPS ||
+        f->routed_pipeline.threadExecutionWidth != 32u ||
+        f->direct_pipeline.threadExecutionWidth != 32u ||
+        f->routed_pipeline.maxTotalThreadsPerThreadgroup < THREADS_PER_GROUP ||
+        f->direct_pipeline.maxTotalThreadsPerThreadgroup < THREADS_PER_GROUP ||
+        f->routed_pipeline.staticThreadgroupMemoryLength +
+                THREADGROUP_MEMORY_BYTES > f->device.maxThreadgroupMemoryLength ||
+        f->direct_pipeline.staticThreadgroupMemoryLength +
+                THREADGROUP_MEMORY_BYTES > f->device.maxThreadgroupMemoryLength) {
+        fprintf(stderr,
+                "metal-q4-attn-out-a-direct-bench: unexpected pipeline "
+                "geometry or threadgroup memory limit\n");
+        return false;
+    }
+
+    const NSUInteger row_bytes =
+        (NSUInteger)(K_DIM / QK_K) * Q4_BLOCK_BYTES;
+    NSUInteger rows = 0u;
+    if (!checked_mul((NSUInteger)GROUPS, M_DIM, &rows) ||
+        !checked_mul(rows, row_bytes, &f->weights_bytes)) {
+        fprintf(stderr,
+                "metal-q4-attn-out-a-direct-bench: weight size overflow\n");
+        return false;
+    }
+    f->weights = guarded_buffer(f->device, f->weights_bytes,
+                                @"Q4 output-A weights");
+    if (!f->weights) return false;
+    fill_weights(f);
+    f->weights_hash = hash_bytes(payload(f->weights), f->weights_bytes);
+    return true;
+}
+
+static map_layout make_map_layout(uint32_t tokens) {
+    const NSUInteger pair_rows = (NSUInteger)tokens * GROUPS;
+    const NSUInteger tpe_bytes = GROUPS * 2u * sizeof(uint32_t);
+    const NSUInteger hids_bytes = pair_rows * sizeof(int32_t);
+    const NSUInteger work_offset = align_up(tpe_bytes + hids_bytes, 8u);
+    const NSUInteger work_cap =
+        (pair_rows + 31u * GROUPS + 31u) / 32u;
+    return (map_layout) {
+        .tpe_bytes = tpe_bytes,
+        .hids_bytes = hids_bytes,
+        .work_offset = work_offset,
+        .total_bytes = work_offset + 8u + work_cap * 2u * sizeof(uint32_t),
+        .work_cap = work_cap,
+    };
+}
+
+static void fill_heads(float *values, NSUInteger count) {
+    for (NSUInteger i = 0; i < count; i++) {
+        const uint32_t bits =
+            (uint32_t)(i * 1103515245u + 12345u + (i >> 7u));
+        values[i] = ((float)((int32_t)((bits >> 16u) & 0x7ffu) - 1024)) /
+                    1024.0f;
+    }
+}
+
+static void fill_ids(int32_t *ids, uint32_t tokens) {
+    for (uint32_t token = 0; token < tokens; token++) {
+        for (uint32_t group = 0; group < GROUPS; group++) {
+            ids[(NSUInteger)token * GROUPS + group] = (int32_t)group;
+        }
+    }
+}
+
+static bool init_case(case_buffers *c, fixture *f, uint32_t tokens) {
+    c->tokens = tokens;
+    c->layout = make_map_layout(tokens);
+    NSUInteger head_values = 0u;
+    NSUInteger output_values = 0u;
+    if (!checked_mul((NSUInteger)tokens, GROUPS * K_DIM, &head_values) ||
+        !checked_mul(head_values, sizeof(float), &c->heads_bytes) ||
+        !checked_mul((NSUInteger)tokens, GROUPS * M_DIM, &output_values) ||
+        !checked_mul(output_values, sizeof(float), &c->output_bytes) ||
+        !checked_mul((NSUInteger)tokens, GROUPS * sizeof(int32_t),
+                     &c->ids_bytes)) {
+        fprintf(stderr,
+                "metal-q4-attn-out-a-direct-bench: N=%u size overflow\n",
+                tokens);
+        return false;
+    }
+
+    c->heads = guarded_buffer(f->device, c->heads_bytes, @"attention heads");
+    c->current_out = guarded_buffer(f->device, c->output_bytes,
+                                    @"map+routed output");
+    c->routed_out = guarded_buffer(f->device, c->output_bytes,
+                                   @"routed-only output");
+    c->direct_out = guarded_buffer(f->device, c->output_bytes,
+                                   @"direct output");
+    c->ids = guarded_buffer(f->device, c->ids_bytes, @"fixed route ids");
+    c->current_map = guarded_buffer(f->device, c->layout.total_bytes,
+                                    @"current route map");
+    c->prebuilt_map = guarded_buffer(f->device, c->layout.total_bytes,
+                                     @"prebuilt route map");
+    if (!c->heads || !c->current_out || !c->routed_out || !c->direct_out ||
+        !c->ids || !c->current_map || !c->prebuilt_map) {
+        return false;
+    }
+
+    fill_heads(payload(c->heads), head_values);
+    fill_ids(payload(c->ids), tokens);
+    c->heads_hash = hash_bytes(payload(c->heads), c->heads_bytes);
+    c->ids_hash = hash_bytes(payload(c->ids), c->ids_bytes);
+
+    const uint64_t row_bytes =
+        (uint64_t)(K_DIM / QK_K) * Q4_BLOCK_BYTES;
+    c->map = (map_args) {
+        .ne02 = GROUPS,
+        .ne10 = K_DIM,
+        .ne11 = GROUPS,
+        .nb11 = (uint64_t)K_DIM * sizeof(float),
+        .nb12 = (uint64_t)GROUPS * K_DIM * sizeof(float),
+        .ne21 = (int32_t)tokens,
+        .ne20 = GROUPS,
+        .nb21 = (uint64_t)GROUPS * sizeof(int32_t),
+    };
+    c->mm = (mm_args) {
+        .ne00 = K_DIM,
+        .ne02 = GROUPS,
+        .nb01 = row_bytes,
+        .nb02 = (uint64_t)M_DIM * row_bytes,
+        .nb03 = (uint64_t)GROUPS * M_DIM * row_bytes,
+        .ne11 = GROUPS,
+        .nb10 = sizeof(float),
+        .nb11 = (uint64_t)K_DIM * sizeof(float),
+        .nb12 = (uint64_t)GROUPS * K_DIM * sizeof(float),
+        .nb13 = (uint64_t)tokens * GROUPS * K_DIM * sizeof(float),
+        .ne20 = GROUPS,
+        .ne21 = (int32_t)tokens,
+        .ne0 = M_DIM,
+        .ne1 = GROUPS,
+        .r2 = 1,
+        .r3 = 1,
+        .tp_rank = 0,
+        .tp_world = 1,
+        .tp_expert_base = 0,
+    };
+    return true;
+}
+
+static void encode_map(id<MTLComputeCommandEncoder> encoder,
+                       fixture *f, case_buffers *c,
+                       id<MTLBuffer> map_buffer) {
+    [encoder setComputePipelineState:f->map_pipeline];
+    [encoder setBytes:&c->map length:sizeof(c->map) atIndex:0];
+    [encoder setBuffer:c->ids offset:GUARD_BYTES atIndex:1];
+    [encoder setBuffer:map_buffer offset:GUARD_BYTES atIndex:2];
+    [encoder setBuffer:map_buffer
+                 offset:GUARD_BYTES + c->layout.tpe_bytes
+                atIndex:3];
+    [encoder setBuffer:map_buffer
+                 offset:GUARD_BYTES + c->layout.work_offset
+                atIndex:4];
+    const NSUInteger staging = GROUPS * GROUPS * sizeof(uint16_t);
+    const NSUInteger scatter = GROUPS * 2u * sizeof(uint32_t);
+    [encoder setThreadgroupMemoryLength:
+        staging > scatter ? staging : scatter atIndex:0];
+    [encoder dispatchThreadgroups:MTLSizeMake(1u, 1u, 1u)
+          threadsPerThreadgroup:MTLSizeMake(GROUPS, 1u, 1u)];
+}
+
+static void encode_routed(id<MTLComputeCommandEncoder> encoder,
+                          fixture *f, case_buffers *c,
+                          id<MTLBuffer> map_buffer,
+                          id<MTLBuffer> output) {
+    [encoder setComputePipelineState:f->routed_pipeline];
+    [encoder setBytes:&c->mm length:sizeof(c->mm) atIndex:0];
+    [encoder setBuffer:f->weights offset:GUARD_BYTES atIndex:1];
+    [encoder setBuffer:c->heads offset:GUARD_BYTES atIndex:2];
+    [encoder setBuffer:map_buffer offset:GUARD_BYTES atIndex:3];
+    [encoder setBuffer:map_buffer
+                 offset:GUARD_BYTES + c->layout.tpe_bytes
+                atIndex:4];
+    [encoder setBuffer:output offset:GUARD_BYTES atIndex:5];
+    [encoder setBuffer:map_buffer
+                 offset:GUARD_BYTES + c->layout.work_offset
+                atIndex:6];
+    [encoder setThreadgroupMemoryLength:THREADGROUP_MEMORY_BYTES atIndex:0];
+    [encoder dispatchThreadgroups:
+        MTLSizeMake(c->layout.work_cap, (M_DIM + 63u) / 64u, 1u)
+          threadsPerThreadgroup:MTLSizeMake(THREADS_PER_GROUP, 1u, 1u)];
+}
+
+static void encode_direct(id<MTLComputeCommandEncoder> encoder,
+                          fixture *f, case_buffers *c) {
+    [encoder setComputePipelineState:f->direct_pipeline];
+    [encoder setBytes:&c->mm length:sizeof(c->mm) atIndex:0];
+    [encoder setBuffer:f->weights offset:GUARD_BYTES atIndex:1];
+    [encoder setBuffer:c->heads offset:GUARD_BYTES atIndex:2];
+    [encoder setBuffer:c->direct_out offset:GUARD_BYTES atIndex:3];
+    [encoder setThreadgroupMemoryLength:THREADGROUP_MEMORY_BYTES atIndex:0];
+    [encoder dispatchThreadgroups:
+        MTLSizeMake((c->tokens + 31u) / 32u,
+                    (M_DIM + 63u) / 64u, GROUPS)
+          threadsPerThreadgroup:MTLSizeMake(THREADS_PER_GROUP, 1u, 1u)];
+}
+
+static bool finish_command_buffer(id<MTLCommandBuffer> cb,
+                                  const char *label) {
+    [cb commit];
+    [cb waitUntilCompleted];
+    if (cb.status == MTLCommandBufferStatusCompleted) return true;
+    fprintf(stderr,
+            "metal-q4-attn-out-a-direct-bench: %s failed: %s\n",
+            label, cb.error ? [[cb.error localizedDescription] UTF8String]
+                            : "unknown Metal error");
+    return false;
+}
+
+static bool build_prebuilt_map(fixture *f, case_buffers *c) {
+    id<MTLCommandBuffer> cb = [f->queue commandBuffer];
+    id<MTLComputeCommandEncoder> encoder = [cb computeCommandEncoder];
+    encode_map(encoder, f, c, c->prebuilt_map);
+    [encoder endEncoding];
+    if (!finish_command_buffer(cb, "prebuilt map")) return false;
+    c->prebuilt_map_hash =
+        hash_bytes(payload(c->prebuilt_map), c->layout.total_bytes);
+    return true;
+}
+
+static bool run_arm(fixture *f, case_buffers *c, bench_arm arm,
+                    double *gpu_seconds) {
+    id<MTLCommandBuffer> cb = [f->queue commandBuffer];
+    id<MTLComputeCommandEncoder> encoder = [cb computeCommandEncoder];
+    switch (arm) {
+    case ARM_CURRENT:
+        encode_map(encoder, f, c, c->current_map);
+        encode_routed(encoder, f, c, c->current_map, c->current_out);
+        break;
+    case ARM_ROUTED_ONLY:
+        encode_routed(encoder, f, c, c->prebuilt_map, c->routed_out);
+        break;
+    case ARM_DIRECT:
+        encode_direct(encoder, f, c);
+        break;
+    }
+    [encoder endEncoding];
+    const char *label = arm == ARM_CURRENT ? "map+routed" :
+                        arm == ARM_ROUTED_ONLY ? "routed-only" : "direct";
+    if (!finish_command_buffer(cb, label)) return false;
+    const double elapsed = cb.GPUEndTime - cb.GPUStartTime;
+    if (!(elapsed > 0.0) || !isfinite(elapsed)) {
+        fprintf(stderr,
+                "metal-q4-attn-out-a-direct-bench: GPU timestamps "
+                "unavailable for %s\n", label);
+        return false;
+    }
+    *gpu_seconds = elapsed;
+    return true;
+}
+
+static void poison_output(id<MTLBuffer> output, NSUInteger bytes,
+                          uint32_t poison) {
+    uint32_t *words = payload(output);
+    for (NSUInteger i = 0; i < bytes / sizeof(*words); i++) {
+        words[i] = poison;
+    }
+}
+
+static bool check_bitwise_outputs(case_buffers *c) {
+    const uint32_t *current = payload(c->current_out);
+    const uint32_t *routed = payload(c->routed_out);
+    const uint32_t *direct = payload(c->direct_out);
+    const NSUInteger count = c->output_bytes / sizeof(uint32_t);
+    for (NSUInteger i = 0; i < count; i++) {
+        if (current[i] != routed[i] || current[i] != direct[i]) {
+            fprintf(stderr,
+                    "metal-q4-attn-out-a-direct-bench: bitwise mismatch "
+                    "N=%u element=%llu current=%08x routed=%08x direct=%08x\n",
+                    c->tokens, (unsigned long long)i,
+                    current[i], routed[i], direct[i]);
+            return false;
+        }
+    }
+    return true;
+}
+
+static bool check_integrity(fixture *f, case_buffers *c) {
+    bool ok = true;
+    if (hash_bytes(payload(f->weights), f->weights_bytes) != f->weights_hash) {
+        fprintf(stderr,
+                "metal-q4-attn-out-a-direct-bench: weights changed N=%u\n",
+                c->tokens);
+        ok = false;
+    }
+    if (hash_bytes(payload(c->heads), c->heads_bytes) != c->heads_hash) {
+        fprintf(stderr,
+                "metal-q4-attn-out-a-direct-bench: heads changed N=%u\n",
+                c->tokens);
+        ok = false;
+    }
+    if (hash_bytes(payload(c->ids), c->ids_bytes) != c->ids_hash) {
+        fprintf(stderr,
+                "metal-q4-attn-out-a-direct-bench: ids changed N=%u\n",
+                c->tokens);
+        ok = false;
+    }
+    if (hash_bytes(payload(c->prebuilt_map), c->layout.total_bytes) !=
+        c->prebuilt_map_hash) {
+        fprintf(stderr,
+                "metal-q4-attn-out-a-direct-bench: routed-only map changed "
+                "N=%u\n", c->tokens);
+        ok = false;
+    }
+    if (hash_bytes(payload(c->current_map), c->layout.total_bytes) !=
+        c->prebuilt_map_hash) {
+        fprintf(stderr,
+                "metal-q4-attn-out-a-direct-bench: rebuilt map differs "
+                "N=%u\n", c->tokens);
+        ok = false;
+    }
+    ok = check_canary(f->weights, f->weights_bytes,
+                      "weights", c->tokens) && ok;
+    ok = check_canary(c->heads, c->heads_bytes,
+                      "heads", c->tokens) && ok;
+    ok = check_canary(c->current_out, c->output_bytes,
+                      "map+routed output", c->tokens) && ok;
+    ok = check_canary(c->routed_out, c->output_bytes,
+                      "routed-only output", c->tokens) && ok;
+    ok = check_canary(c->direct_out, c->output_bytes,
+                      "direct output", c->tokens) && ok;
+    ok = check_canary(c->ids, c->ids_bytes,
+                      "ids", c->tokens) && ok;
+    ok = check_canary(c->current_map, c->layout.total_bytes,
+                      "current map", c->tokens) && ok;
+    ok = check_canary(c->prebuilt_map, c->layout.total_bytes,
+                      "prebuilt map", c->tokens) && ok;
+    return ok;
+}
+
+static bool run_oracle(fixture *f, case_buffers *c) {
+    poison_output(c->current_out, c->output_bytes, k_poison_current);
+    poison_output(c->routed_out, c->output_bytes, k_poison_routed);
+    poison_output(c->direct_out, c->output_bytes, k_poison_direct);
+    double ignored = 0.0;
+    if (!run_arm(f, c, ARM_CURRENT, &ignored) ||
+        !run_arm(f, c, ARM_ROUTED_ONLY, &ignored) ||
+        !run_arm(f, c, ARM_DIRECT, &ignored)) {
+        return false;
+    }
+    return check_bitwise_outputs(c) && check_integrity(f, c);
+}
+
+static bool warm_up(fixture *f, case_buffers *c, uint32_t warmup) {
+    static const bench_arm orders[][3] = {
+        {ARM_CURRENT, ARM_ROUTED_ONLY, ARM_DIRECT},
+        {ARM_DIRECT, ARM_ROUTED_ONLY, ARM_CURRENT},
+        {ARM_ROUTED_ONLY, ARM_CURRENT, ARM_DIRECT},
+        {ARM_DIRECT, ARM_CURRENT, ARM_ROUTED_ONLY},
+        {ARM_CURRENT, ARM_DIRECT, ARM_ROUTED_ONLY},
+        {ARM_ROUTED_ONLY, ARM_DIRECT, ARM_CURRENT},
+    };
+    double ignored = 0.0;
+    for (uint32_t cycle = 0; cycle < warmup; cycle++) {
+        const bench_arm *order = orders[cycle %
+            (sizeof(orders) / sizeof(orders[0]))];
+        for (uint32_t position = 0; position < 3u; position++) {
+            if (!run_arm(f, c, order[position], &ignored)) return false;
+        }
+    }
+    return true;
+}
+
+static bool run_balanced_pair(fixture *f, case_buffers *c,
+                              bench_arm first, bench_arm second,
+                              uint32_t samples, pair_samples *result) {
+    uint32_t first_count = 0u;
+    uint32_t second_count = 0u;
+    for (uint32_t cycle = 0; cycle < samples / 2u; cycle++) {
+        const bench_arm abba[] = {first, second, second, first};
+        const bench_arm baab[] = {second, first, first, second};
+        const bench_arm *order = (cycle & 1u) ? baab : abba;
+        for (uint32_t position = 0; position < 4u; position++) {
+            double elapsed = 0.0;
+            if (!run_arm(f, c, order[position], &elapsed)) return false;
+            if (order[position] == first) {
+                result->first[first_count++] = elapsed;
+            } else {
+                result->second[second_count++] = elapsed;
+            }
+        }
+    }
+    return first_count == samples && second_count == samples;
+}
+
+static int compare_double(const void *lhs, const void *rhs) {
+    const double a = *(const double *)lhs;
+    const double b = *(const double *)rhs;
+    return (a > b) - (a < b);
+}
+
+static double median(const double *values, uint32_t count) {
+    double sorted[MAX_SAMPLES];
+    memcpy(sorted, values, count * sizeof(*sorted));
+    qsort(sorted, count, sizeof(*sorted), compare_double);
+    if (count & 1u) return sorted[count / 2u];
+    return 0.5 * (sorted[count / 2u - 1u] + sorted[count / 2u]);
+}
+
+static double paired_geomean_ratio(const double *baseline,
+                                    const double *candidate,
+                                    uint32_t count) {
+    double log_sum = 0.0;
+    for (uint32_t i = 0; i < count; i++) {
+        log_sum += log(baseline[i] / candidate[i]);
+    }
+    return exp(log_sum / count);
+}
+
+static bool run_benchmark_case(fixture *f, const bench_config *config,
+                               uint32_t tokens) {
+    @autoreleasepool {
+        case_buffers c = {0};
+        if (!init_case(&c, f, tokens) || !build_prebuilt_map(f, &c) ||
+            !run_oracle(f, &c) || !warm_up(f, &c, config->warmup)) {
+            return false;
+        }
+
+        pair_samples current_direct = {0};
+        pair_samples current_routed = {0};
+        pair_samples routed_direct = {0};
+        if (!run_balanced_pair(f, &c, ARM_CURRENT, ARM_DIRECT,
+                               config->samples, &current_direct) ||
+            !run_balanced_pair(f, &c, ARM_CURRENT, ARM_ROUTED_ONLY,
+                               config->samples, &current_routed) ||
+            !run_balanced_pair(f, &c, ARM_ROUTED_ONLY, ARM_DIRECT,
+                               config->samples, &routed_direct) ||
+            !check_bitwise_outputs(&c) || !check_integrity(f, &c)) {
+            return false;
+        }
+
+        const double current_ms =
+            median(current_direct.first, config->samples) * 1.0e3;
+        const double direct_ms =
+            median(current_direct.second, config->samples) * 1.0e3;
+        const double routed_ms =
+            median(current_routed.second, config->samples) * 1.0e3;
+        const double direct_ratio = paired_geomean_ratio(
+            current_direct.first, current_direct.second, config->samples);
+        const double map_ratio = paired_geomean_ratio(
+            current_routed.first, current_routed.second, config->samples);
+        double map_delta[MAX_SAMPLES];
+        for (uint32_t i = 0; i < config->samples; i++) {
+            map_delta[i] = (current_routed.first[i] -
+                            current_routed.second[i]) * 1.0e6;
+        }
+        const double map_delta_us = median(map_delta, config->samples);
+        const double routed_to_direct = paired_geomean_ratio(
+            routed_direct.first, routed_direct.second, config->samples);
+
+        printf("  N=%-4u current(map+routed) %8.3f ms  "
+               "routed-only %8.3f ms  direct %8.3f ms\n",
+               tokens, current_ms, routed_ms, direct_ms);
+        printf("         paired direct gain %6.3fx (%+6.2f%%); "
+               "map+dispatch %+8.3f us (%+6.2f%%); "
+               "paired direct vs routed %6.3fx (%+6.2f%%); PASS\n",
+               direct_ratio, (direct_ratio - 1.0) * 100.0,
+               map_delta_us, (map_ratio - 1.0) * 100.0,
+               routed_to_direct, (routed_to_direct - 1.0) * 100.0);
+        fflush(stdout);
+        return true;
+    }
+}
+
+int main(int argc, char **argv) {
+    @autoreleasepool {
+        const bench_config config = parse_options(argc, argv);
+        fixture f = {0};
+        if (!init_fixture(&f)) return 1;
+
+        printf("Metal Q4_K attention output-A direct kernel benchmark\n");
+        printf("  device: %s\n", [f.device.name UTF8String]);
+        printf("  shape: 4096 -> 1024 x 8 groups; resident anonymous Q4_K "
+               "weights (%.1f MiB)\n",
+               (double)f.weights_bytes / (1024.0 * 1024.0));
+        printf("  design: current map+routed, routed-only prebuilt map, "
+               "and fixed-route direct\n");
+        printf("  schedule: balanced ABBA/BAAB pair blocks, "
+               "%u samples/arm/pair, "
+               "%u warmup dispatches/arm, GPU timestamps only\n",
+               config.samples, config.warmup);
+
+        for (uint32_t i = 0; i < config.token_count; i++) {
+            if (!run_benchmark_case(&f, &config, config.tokens[i])) return 1;
+        }
+        printf("  correctness: all three outputs bit-identical; weights, "
+               "heads, ids, and prebuilt map hashes unchanged; all "
+               "prefix/suffix canaries intact\n");
+        printf("  scope: no GGUF, SSD I/O, model upload, GPU readback, or "
+               "CPU wall timing in measured command buffers\n");
+        return 0;
+    }
+}
diff --git a/speed-bench/metal_q4_dense_pair_bench.m b/speed-bench/metal_q4_dense_pair_bench.m
new file mode 100644
index 0000000000..3eeabfbb5b
--- /dev/null
+++ b/speed-bench/metal_q4_dense_pair_bench.m
@@ -0,0 +1,708 @@
+#import <Foundation/Foundation.h>
+#import <Metal/Metal.h>
+
+#include <errno.h>
+#include <limits.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+enum {
+    IN_DIM = 4096,
+    OUT0_DIM = 1024,
+    OUT1_DIM = 512,
+    QK_K = 256,
+    Q4_BLOCK_BYTES = 144,
+    NSG = 2,
+    NR0 = 2,
+    THREADS_PER_SIMDGROUP = 32,
+    GUARD_BYTES = 256,
+    DEFAULT_SETS = 64,
+    DEFAULT_DISPATCHES = 256,
+    DEFAULT_WARMUP_DISPATCHES = 64,
+    DEFAULT_SAMPLES = 16,
+};
+
+static const uint32_t k_guard = 0x7fc12345u;
+
+typedef struct {
+    uint16_t d;
+    uint16_t dmin;
+    uint8_t scales[12];
+    uint8_t qs[QK_K / 2];
+} block_q4_K_host;
+
+typedef struct {
+    int32_t ne00;
+    int32_t ne01;
+    int32_t ne02;
+    uint64_t nb00;
+    uint64_t nb01;
+    uint64_t nb02;
+    uint64_t nb03;
+    int32_t ne10;
+    int32_t ne11;
+    int32_t ne12;
+    uint64_t nb10;
+    uint64_t nb11;
+    uint64_t nb12;
+    uint64_t nb13;
+    int32_t ne0;
+    int32_t ne1;
+    int32_t nr0;
+    int16_t r2;
+    int16_t r3;
+} mul_mv_args;
+
+_Static_assert(sizeof(block_q4_K_host) == Q4_BLOCK_BYTES,
+               "Q4_K host fixture must match GGUF/Metal layout");
+_Static_assert(sizeof(mul_mv_args) == 112,
+               "Metal mul_mv argument ABI changed");
+
+typedef struct {
+    uint32_t sets;
+    uint32_t dispatches;
+    uint32_t warmup_dispatches;
+    uint32_t samples;
+} bench_config;
+
+typedef enum {
+    ARM_SEPARATE,
+    ARM_PAIR,
+} bench_arm;
+
+typedef struct {
+    __strong id<MTLDevice> device;
+    __strong id<MTLCommandQueue> queue;
+    __strong id<MTLComputePipelineState> standalone_pipeline;
+    __strong id<MTLComputePipelineState> pair_pipeline;
+    __strong id<MTLBuffer> w0;
+    __strong id<MTLBuffer> w1;
+    __strong id<MTLBuffer> x;
+    __strong id<MTLBuffer> separate0;
+    __strong id<MTLBuffer> separate1;
+    __strong id<MTLBuffer> pair0;
+    __strong id<MTLBuffer> pair1;
+    mul_mv_args args0;
+    mul_mv_args args1;
+    NSUInteger w0_set_bytes;
+    NSUInteger w1_set_bytes;
+    NSUInteger x_offset;
+    NSUInteger out0_stride;
+    NSUInteger out1_stride;
+    uint32_t sets;
+} fixture;
+
+static void usage(FILE *fp, const char *argv0) {
+    fprintf(fp,
+            "usage: %s [options]\n"
+            "\n"
+            "Kernel-only A/B for production decode Q4_K projections:\n"
+            "  4096 -> 1024 and 4096 -> 512, n_tok=1\n"
+            "\n"
+            "  --sets N          rotating resident weight sets (default: 64)\n"
+            "  --dispatches N     logical projection pairs per sample (default: 256)\n"
+            "  --warmup N         logical projection pairs per warmup arm (default: 64)\n"
+            "  --samples N        samples per arm; must be even (default: 16)\n"
+            "  -h, --help         show this help\n"
+            "\n"
+            "Set DS4_SOURCE_ROOT when running outside the repository root.\n",
+            argv0);
+}
+
+static uint32_t parse_u32(const char *text, const char *option,
+                          uint32_t minimum) {
+    char *end = NULL;
+    errno = 0;
+    unsigned long long value = strtoull(text, &end, 10);
+    if (errno != 0 || text[0] == '\0' || !end || *end != '\0' ||
+        value < minimum || value > UINT32_MAX) {
+        fprintf(stderr, "metal-q4-dense-pair-bench: invalid %s: %s\n",
+                option, text);
+        exit(2);
+    }
+    return (uint32_t)value;
+}
+
+static const char *need_arg(int *i, int argc, char **argv) {
+    if (*i + 1 >= argc) {
+        fprintf(stderr, "metal-q4-dense-pair-bench: %s needs a value\n",
+                argv[*i]);
+        exit(2);
+    }
+    return argv[++*i];
+}
+
+static bench_config parse_options(int argc, char **argv) {
+    bench_config cfg = {
+        .sets = DEFAULT_SETS,
+        .dispatches = DEFAULT_DISPATCHES,
+        .warmup_dispatches = DEFAULT_WARMUP_DISPATCHES,
+        .samples = DEFAULT_SAMPLES,
+    };
+    for (int i = 1; i < argc; i++) {
+        const char *arg = argv[i];
+        if (!strcmp(arg, "-h") || !strcmp(arg, "--help")) {
+            usage(stdout, argv[0]);
+            exit(0);
+        } else if (!strcmp(arg, "--sets")) {
+            cfg.sets = parse_u32(need_arg(&i, argc, argv), arg, 1);
+        } else if (!strcmp(arg, "--dispatches")) {
+            cfg.dispatches =
+                parse_u32(need_arg(&i, argc, argv), arg, 1);
+        } else if (!strcmp(arg, "--warmup")) {
+            cfg.warmup_dispatches =
+                parse_u32(need_arg(&i, argc, argv), arg, 0);
+        } else if (!strcmp(arg, "--samples")) {
+            cfg.samples = parse_u32(need_arg(&i, argc, argv), arg, 2);
+        } else {
+            fprintf(stderr, "metal-q4-dense-pair-bench: unknown option: %s\n",
+                    arg);
+            usage(stderr, argv[0]);
+            exit(2);
+        }
+    }
+    if ((cfg.samples & 1u) != 0u) {
+        fprintf(stderr,
+                "metal-q4-dense-pair-bench: --samples must be even for "
+                "ABBA/BAAB balance\n");
+        exit(2);
+    }
+    return cfg;
+}
+
+static NSString *metal_prelude(void) {
+    return @"#include <metal_stdlib>\n"
+            "using namespace metal;\n"
+            "#define MAX(x, y) ((x) > (y) ? (x) : (y))\n"
+            "#define MIN(x, y) ((x) < (y) ? (x) : (y))\n"
+            "#define SWAP(x, y) { auto tmp = (x); (x) = (y); (y) = tmp; }\n"
+            "#define QK8_0 32\n"
+            "#ifndef QK_K\n#define QK_K 256\n#endif\n"
+            "#define N_SIMDWIDTH 32\n"
+            "#define N_R0_Q8_0 2\n"
+            "#define N_SG_Q8_0 4\n"
+            "#define FC_MUL_MV 600\n"
+            "#define FC_MUL_MM 700\n"
+            "#define FC_BIN 1300\n"
+            "#define FOR_UNROLL(x) _Pragma(\"clang loop unroll(full)\") for (x)\n"
+            "#define M_PI_F 3.14159265358979323846f\n"
+            "enum ds4_sort_order { DS4_SORT_ORDER_ASC, DS4_SORT_ORDER_DESC };\n"
+            "struct block_q8_0 { half d; int8_t qs[QK8_0]; };\n"
+            "struct block_q8_K { float d; int8_t qs[QK_K]; "
+            "int16_t bsums[QK_K / 16]; };\n";
+}
+
+/* Keep the same concatenation order as ds4_metal.m. The benchmark specializes
+ * and dispatches the checked-in production kernels; it carries no kernel copy. */
+static NSString *load_metal_source(void) {
+    static const char *paths[] = {
+        "metal/activations.metal",
+        "metal/flash_attn.metal",
+        "metal/dense.metal",
+        "metal/moe.metal",
+        "metal/dsv4_hc.metal",
+        "metal/unary.metal",
+        "metal/dsv4_kv.metal",
+        "metal/dsv4_rope.metal",
+        "metal/dsv4_misc.metal",
+        "metal/argsort.metal",
+        "metal/cpy.metal",
+        "metal/concat.metal",
+        "metal/get_rows.metal",
+        "metal/sum_rows.metal",
+        "metal/softmax.metal",
+        "metal/repeat.metal",
+        "metal/glu.metal",
+        "metal/norm.metal",
+        "metal/bin.metal",
+        "metal/set_rows.metal",
+    };
+    const char *root_env = getenv("DS4_SOURCE_ROOT");
+    NSString *root = root_env && root_env[0]
+        ? [NSString stringWithUTF8String:root_env]
+        : @".";
+    NSMutableString *source = [NSMutableString stringWithString:metal_prelude()];
+    for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); i++) {
+        NSString *relative = [NSString stringWithUTF8String:paths[i]];
+        NSString *path = [root stringByAppendingPathComponent:relative];
+        NSError *error = nil;
+        NSString *part = [NSString stringWithContentsOfFile:path
+                                                   encoding:NSUTF8StringEncoding
+                                                      error:&error];
+        if (!part) {
+            fprintf(stderr, "metal-q4-dense-pair-bench: cannot read %s: %s\n",
+                    [path fileSystemRepresentation],
+                    [[error localizedDescription] UTF8String]);
+            return nil;
+        }
+        [source appendFormat:@"\n// appended %@\n%@\n", relative, part];
+    }
+    return source;
+}
+
+static id<MTLComputePipelineState> make_pipeline(
+        id<MTLDevice> device, id<MTLLibrary> library, NSString *name) {
+    int16_t nsg = NSG;
+    int16_t nxpsg = 8;
+    MTLFunctionConstantValues *values = [MTLFunctionConstantValues new];
+    [values setConstantValue:&nsg type:MTLDataTypeShort atIndex:600];
+    [values setConstantValue:&nxpsg type:MTLDataTypeShort atIndex:601];
+    NSError *error = nil;
+    id<MTLFunction> function = [library newFunctionWithName:name
+                                            constantValues:values
+                                                     error:&error];
+    if (!function) {
+        fprintf(stderr, "metal-q4-dense-pair-bench: function %s: %s\n",
+                [name UTF8String], [[error localizedDescription] UTF8String]);
+        return nil;
+    }
+    error = nil;
+    id<MTLComputePipelineState> pipeline =
+        [device newComputePipelineStateWithFunction:function error:&error];
+    if (!pipeline) {
+        fprintf(stderr, "metal-q4-dense-pair-bench: pipeline %s: %s\n",
+                [name UTF8String], [[error localizedDescription] UTF8String]);
+    }
+    return pipeline;
+}
+
+static uint32_t lcg_next(uint32_t *state) {
+    *state = *state * 1664525u + 1013904223u;
+    return *state;
+}
+
+static void fill_q4(void *storage, NSUInteger bytes, uint32_t seed) {
+    block_q4_K_host *blocks = storage;
+    const NSUInteger count = bytes / sizeof(*blocks);
+    uint32_t state = seed;
+    for (NSUInteger b = 0; b < count; b++) {
+        blocks[b].d = (uint16_t)(0x2400u | (lcg_next(&state) & 0x03ffu));
+        blocks[b].dmin =
+            (uint16_t)(0x1c00u | (lcg_next(&state) & 0x03ffu));
+        for (size_t i = 0; i < sizeof(blocks[b].scales); i++) {
+            blocks[b].scales[i] = (uint8_t)(lcg_next(&state) >> 24u);
+        }
+        for (size_t i = 0; i < sizeof(blocks[b].qs); i++) {
+            blocks[b].qs[i] = (uint8_t)(lcg_next(&state) >> 24u);
+        }
+    }
+}
+
+static void fill_activation(float *x) {
+    uint32_t state = 0x243f6a88u;
+    for (uint32_t i = 0; i < IN_DIM; i++) {
+        int32_t centered = (int32_t)(lcg_next(&state) & 0xffffu) - 32768;
+        x[i] = (float)centered / 32768.0f;
+    }
+}
+
+static NSUInteger align_up(NSUInteger value, NSUInteger alignment) {
+    return (value + alignment - 1u) / alignment * alignment;
+}
+
+static bool checked_product(NSUInteger a, NSUInteger b, NSUInteger *out) {
+    if (a != 0 && b > NSUIntegerMax / a) return false;
+    *out = a * b;
+    return true;
+}
+
+static id<MTLBuffer> alloc_buffer(id<MTLDevice> device, NSUInteger bytes,
+                                  NSString *label) {
+    if (bytes == 0 || bytes > device.maxBufferLength) {
+        fprintf(stderr,
+                "metal-q4-dense-pair-bench: %s size %.1f MiB exceeds device limit\n",
+                [label UTF8String], (double)bytes / (1024.0 * 1024.0));
+        return nil;
+    }
+    id<MTLBuffer> buffer =
+        [device newBufferWithLength:bytes options:MTLResourceStorageModeShared];
+    buffer.label = label;
+    if (!buffer) {
+        fprintf(stderr, "metal-q4-dense-pair-bench: allocation failed for %s\n",
+                [label UTF8String]);
+    }
+    return buffer;
+}
+
+static mul_mv_args make_args(uint32_t out_dim) {
+    const uint64_t row_bytes = (IN_DIM / QK_K) * Q4_BLOCK_BYTES;
+    return (mul_mv_args){
+        .ne00 = IN_DIM,
+        .ne01 = (int32_t)out_dim,
+        .ne02 = 1,
+        .nb00 = 1,
+        .nb01 = row_bytes,
+        .nb02 = row_bytes * out_dim,
+        .nb03 = row_bytes * out_dim,
+        .ne10 = IN_DIM,
+        .ne11 = 1,
+        .ne12 = 1,
+        .nb10 = sizeof(float),
+        .nb11 = IN_DIM * sizeof(float),
+        .nb12 = IN_DIM * sizeof(float),
+        .nb13 = IN_DIM * sizeof(float),
+        .ne0 = (int32_t)out_dim,
+        .ne1 = 1,
+        .nr0 = NR0,
+        .r2 = 1,
+        .r3 = 1,
+    };
+}
+
+static void fill_guard_buffer(id<MTLBuffer> buffer) {
+    uint32_t *words = buffer.contents;
+    for (NSUInteger i = 0; i < buffer.length / sizeof(*words); i++) {
+        words[i] = k_guard;
+    }
+}
+
+static bool init_fixture(fixture *f, const bench_config *cfg) {
+    f->device = MTLCreateSystemDefaultDevice();
+    if (!f->device) {
+        fprintf(stderr, "metal-q4-dense-pair-bench: no Metal device\n");
+        return false;
+    }
+    f->queue = [f->device newCommandQueue];
+    NSString *source = load_metal_source();
+    if (!f->queue || !source) return false;
+
+    NSError *error = nil;
+    MTLCompileOptions *options = [MTLCompileOptions new];
+    id<MTLLibrary> library =
+        [f->device newLibraryWithSource:source options:options error:&error];
+    if (!library) {
+        fprintf(stderr, "metal-q4-dense-pair-bench: Metal compile failed: %s\n",
+                [[error localizedDescription] UTF8String]);
+        return false;
+    }
+    f->standalone_pipeline = make_pipeline(
+        f->device, library, @"kernel_mul_mv_q4_K_dense_f32");
+    f->pair_pipeline = make_pipeline(
+        f->device, library, @"kernel_mul_mv_q4_K_dense_pair_f32");
+    if (!f->standalone_pipeline || !f->pair_pipeline) return false;
+
+    f->sets = cfg->sets;
+    f->w0_set_bytes = (IN_DIM / QK_K) * Q4_BLOCK_BYTES * OUT0_DIM;
+    f->w1_set_bytes = (IN_DIM / QK_K) * Q4_BLOCK_BYTES * OUT1_DIM;
+    f->x_offset = GUARD_BYTES;
+    f->out0_stride = align_up(
+        GUARD_BYTES + OUT0_DIM * sizeof(float) + GUARD_BYTES, 256u);
+    f->out1_stride = align_up(
+        GUARD_BYTES + OUT1_DIM * sizeof(float) + GUARD_BYTES, 256u);
+    f->args0 = make_args(OUT0_DIM);
+    f->args1 = make_args(OUT1_DIM);
+
+    NSUInteger w0_bytes = 0, w1_bytes = 0, out0_bytes = 0, out1_bytes = 0;
+    if (!checked_product(f->w0_set_bytes, cfg->sets, &w0_bytes) ||
+        !checked_product(f->w1_set_bytes, cfg->sets, &w1_bytes) ||
+        !checked_product(f->out0_stride, cfg->sets, &out0_bytes) ||
+        !checked_product(f->out1_stride, cfg->sets, &out1_bytes)) {
+        fprintf(stderr, "metal-q4-dense-pair-bench: requested sizes overflow\n");
+        return false;
+    }
+
+    f->w0 = alloc_buffer(f->device, w0_bytes, @"q4-w0-resident");
+    f->w1 = alloc_buffer(f->device, w1_bytes, @"q4-w1-resident");
+    f->x = alloc_buffer(f->device,
+        GUARD_BYTES + IN_DIM * sizeof(float) + GUARD_BYTES,
+        @"decode-activation");
+    f->separate0 = alloc_buffer(f->device, out0_bytes, @"separate-out0");
+    f->separate1 = alloc_buffer(f->device, out1_bytes, @"separate-out1");
+    f->pair0 = alloc_buffer(f->device, out0_bytes, @"pair-out0");
+    f->pair1 = alloc_buffer(f->device, out1_bytes, @"pair-out1");
+    if (!f->w0 || !f->w1 || !f->x || !f->separate0 || !f->separate1 ||
+        !f->pair0 || !f->pair1) {
+        return false;
+    }
+
+    fill_q4(f->w0.contents, f->w0.length, 0x41c64e6du);
+    fill_q4(f->w1.contents, f->w1.length, 0x9e3779b9u);
+    fill_guard_buffer(f->x);
+    fill_activation((float *)((uint8_t *)f->x.contents + f->x_offset));
+    fill_guard_buffer(f->separate0);
+    fill_guard_buffer(f->separate1);
+    fill_guard_buffer(f->pair0);
+    fill_guard_buffer(f->pair1);
+    return true;
+}
+
+static void encode_standalone(fixture *f, id<MTLComputeCommandEncoder> enc,
+                              uint32_t set) {
+    const NSUInteger out0 = (NSUInteger)set * f->out0_stride + GUARD_BYTES;
+    const NSUInteger out1 = (NSUInteger)set * f->out1_stride + GUARD_BYTES;
+    [enc setComputePipelineState:f->standalone_pipeline];
+    [enc setBytes:&f->args0 length:sizeof(f->args0) atIndex:0];
+    [enc setBuffer:f->w0 offset:(NSUInteger)set * f->w0_set_bytes atIndex:1];
+    [enc setBuffer:f->x offset:f->x_offset atIndex:2];
+    [enc setBuffer:f->separate0 offset:out0 atIndex:3];
+    [enc setThreadgroupMemoryLength:32 atIndex:0];
+    [enc dispatchThreadgroups:MTLSizeMake((OUT0_DIM + NSG * NR0 - 1) /
+                                          (NSG * NR0), 1, 1)
+         threadsPerThreadgroup:MTLSizeMake(THREADS_PER_SIMDGROUP, NSG, 1)];
+
+    [enc setBytes:&f->args1 length:sizeof(f->args1) atIndex:0];
+    [enc setBuffer:f->w1 offset:(NSUInteger)set * f->w1_set_bytes atIndex:1];
+    [enc setBuffer:f->separate1 offset:out1 atIndex:3];
+    [enc dispatchThreadgroups:MTLSizeMake((OUT1_DIM + NSG * NR0 - 1) /
+                                          (NSG * NR0), 1, 1)
+         threadsPerThreadgroup:MTLSizeMake(THREADS_PER_SIMDGROUP, NSG, 1)];
+}
+
+static void encode_pair(fixture *f, id<MTLComputeCommandEncoder> enc,
+                        uint32_t set) {
+    const NSUInteger out0 = (NSUInteger)set * f->out0_stride + GUARD_BYTES;
+    const NSUInteger out1 = (NSUInteger)set * f->out1_stride + GUARD_BYTES;
+    [enc setComputePipelineState:f->pair_pipeline];
+    [enc setBytes:&f->args0 length:sizeof(f->args0) atIndex:0];
+    [enc setBytes:&f->args1 length:sizeof(f->args1) atIndex:1];
+    [enc setBuffer:f->w0 offset:(NSUInteger)set * f->w0_set_bytes atIndex:2];
+    [enc setBuffer:f->w1 offset:(NSUInteger)set * f->w1_set_bytes atIndex:3];
+    [enc setBuffer:f->x offset:f->x_offset atIndex:4];
+    [enc setBuffer:f->pair0 offset:out0 atIndex:5];
+    [enc setBuffer:f->pair1 offset:out1 atIndex:6];
+    [enc setThreadgroupMemoryLength:32 atIndex:0];
+    [enc dispatchThreadgroups:MTLSizeMake((OUT0_DIM + NSG * NR0 - 1) /
+                                          (NSG * NR0), 1, 1)
+         threadsPerThreadgroup:MTLSizeMake(THREADS_PER_SIMDGROUP, NSG, 1)];
+}
+
+static bool finish_command_buffer(id<MTLCommandBuffer> cb, const char *label) {
+    [cb commit];
+    [cb waitUntilCompleted];
+    if (cb.status != MTLCommandBufferStatusCompleted) {
+        fprintf(stderr, "metal-q4-dense-pair-bench: %s failed: %s\n", label,
+                [[cb.error localizedDescription] UTF8String]);
+        return false;
+    }
+    return true;
+}
+
+static bool check_guard_records(id<MTLBuffer> buffer, NSUInteger stride,
+                                NSUInteger output_bytes, uint32_t sets,
+                                const char *label) {
+    const uint32_t *words = buffer.contents;
+    for (uint32_t set = 0; set < sets; set++) {
+        const NSUInteger record = (NSUInteger)set * stride;
+        const NSUInteger output_begin = record + GUARD_BYTES;
+        const NSUInteger output_end = output_begin + output_bytes;
+        for (NSUInteger off = record; off < output_begin; off += sizeof(*words)) {
+            if (words[off / sizeof(*words)] != k_guard) goto bad;
+        }
+        for (NSUInteger off = output_end; off < record + stride;
+             off += sizeof(*words)) {
+            if (words[off / sizeof(*words)] != k_guard) goto bad;
+        }
+    }
+    return true;
+
+bad:
+    fprintf(stderr, "metal-q4-dense-pair-bench: %s canary changed\n", label);
+    return false;
+}
+
+static bool check_all_guards(fixture *f) {
+    const uint32_t *x = f->x.contents;
+    for (NSUInteger off = 0; off < f->x_offset; off += sizeof(*x)) {
+        if (x[off / sizeof(*x)] != k_guard) goto x_bad;
+    }
+    for (NSUInteger off = f->x_offset + IN_DIM * sizeof(float);
+         off < f->x.length; off += sizeof(*x)) {
+        if (x[off / sizeof(*x)] != k_guard) goto x_bad;
+    }
+    return check_guard_records(f->separate0, f->out0_stride,
+                               OUT0_DIM * sizeof(float), f->sets,
+                               "separate out0") &&
+           check_guard_records(f->separate1, f->out1_stride,
+                               OUT1_DIM * sizeof(float), f->sets,
+                               "separate out1") &&
+           check_guard_records(f->pair0, f->out0_stride,
+                               OUT0_DIM * sizeof(float), f->sets,
+                               "pair out0") &&
+           check_guard_records(f->pair1, f->out1_stride,
+                               OUT1_DIM * sizeof(float), f->sets,
+                               "pair out1");
+
+x_bad:
+    fprintf(stderr, "metal-q4-dense-pair-bench: activation canary changed\n");
+    return false;
+}
+
+static bool check_outputs(fixture *f) {
+    for (uint32_t set = 0; set < f->sets; set++) {
+        const uint8_t *separate0 = (uint8_t *)f->separate0.contents +
+            (NSUInteger)set * f->out0_stride + GUARD_BYTES;
+        const uint8_t *pair0 = (uint8_t *)f->pair0.contents +
+            (NSUInteger)set * f->out0_stride + GUARD_BYTES;
+        const uint8_t *separate1 = (uint8_t *)f->separate1.contents +
+            (NSUInteger)set * f->out1_stride + GUARD_BYTES;
+        const uint8_t *pair1 = (uint8_t *)f->pair1.contents +
+            (NSUInteger)set * f->out1_stride + GUARD_BYTES;
+        if (memcmp(separate0, pair0, OUT0_DIM * sizeof(float)) != 0 ||
+            memcmp(separate1, pair1, OUT1_DIM * sizeof(float)) != 0) {
+            fprintf(stderr,
+                    "metal-q4-dense-pair-bench: bitwise mismatch at set %u\n",
+                    set);
+            return false;
+        }
+    }
+    return true;
+}
+
+static bool run_oracle(fixture *f) {
+    id<MTLCommandBuffer> cb = [f->queue commandBuffer];
+    id<MTLComputeCommandEncoder> enc = [cb computeCommandEncoder];
+    for (uint32_t set = 0; set < f->sets; set++) {
+        encode_standalone(f, enc, set);
+    }
+    for (uint32_t set = 0; set < f->sets; set++) {
+        encode_pair(f, enc, set);
+    }
+    [enc endEncoding];
+    if (!finish_command_buffer(cb, "correctness oracle")) return false;
+    if (!check_outputs(f) || !check_all_guards(f)) return false;
+    fprintf(stderr,
+            "Metal Q4_K dense pair oracle: PASS (%u resident sets, bit-exact, canaries intact)\n",
+            f->sets);
+    return true;
+}
+
+static bool run_workload(fixture *f, bench_arm arm, uint32_t dispatches,
+                         uint32_t start_set, double *gpu_seconds) {
+    id<MTLCommandBuffer> cb = [f->queue commandBuffer];
+    id<MTLComputeCommandEncoder> enc = [cb computeCommandEncoder];
+    for (uint32_t i = 0; i < dispatches; i++) {
+        const uint32_t set = (start_set + i) % f->sets;
+        if (arm == ARM_SEPARATE) encode_standalone(f, enc, set);
+        else encode_pair(f, enc, set);
+    }
+    [enc endEncoding];
+    if (!finish_command_buffer(cb, arm == ARM_SEPARATE ?
+                               "separate workload" : "pair workload")) {
+        return false;
+    }
+    const double elapsed = cb.GPUEndTime - cb.GPUStartTime;
+    if (!(elapsed > 0.0)) {
+        fprintf(stderr,
+                "metal-q4-dense-pair-bench: GPU timestamps unavailable\n");
+        return false;
+    }
+    if (gpu_seconds) *gpu_seconds = elapsed;
+    return true;
+}
+
+static int compare_double(const void *a, const void *b) {
+    const double x = *(const double *)a;
+    const double y = *(const double *)b;
+    return (x > y) - (x < y);
+}
+
+static double percentile(const double *sorted, uint32_t n, double p) {
+    const double position = p * (double)(n - 1u);
+    const uint32_t lo = (uint32_t)position;
+    const uint32_t hi = lo + 1u < n ? lo + 1u : lo;
+    const double fraction = position - (double)lo;
+    return sorted[lo] + (sorted[hi] - sorted[lo]) * fraction;
+}
+
+static bool run_benchmark(fixture *f, const bench_config *cfg) {
+    double ignored = 0.0;
+    if (cfg->warmup_dispatches != 0u) {
+        static const bench_arm warm_order[] = {
+            ARM_SEPARATE, ARM_PAIR, ARM_PAIR, ARM_SEPARATE,
+        };
+        for (size_t i = 0; i < sizeof(warm_order) / sizeof(warm_order[0]); i++) {
+            if (!run_workload(f, warm_order[i], cfg->warmup_dispatches,
+                              (uint32_t)(i * cfg->warmup_dispatches) % f->sets,
+                              &ignored)) {
+                return false;
+            }
+        }
+    }
+
+    double *separate = calloc(cfg->samples, sizeof(*separate));
+    double *pair = calloc(cfg->samples, sizeof(*pair));
+    if (!separate || !pair) {
+        fprintf(stderr, "metal-q4-dense-pair-bench: sample allocation failed\n");
+        free(separate);
+        free(pair);
+        return false;
+    }
+
+    uint32_t separate_count = 0;
+    uint32_t pair_count = 0;
+    bool ok = true;
+    for (uint32_t cycle = 0; ok && cycle < cfg->samples / 2u; cycle++) {
+        const bench_arm abba[] = {
+            ARM_SEPARATE, ARM_PAIR, ARM_PAIR, ARM_SEPARATE,
+        };
+        const bench_arm baab[] = {
+            ARM_PAIR, ARM_SEPARATE, ARM_SEPARATE, ARM_PAIR,
+        };
+        const bench_arm *order = (cycle & 1u) ? baab : abba;
+        for (uint32_t j = 0; ok && j < 4u; j++) {
+            const bench_arm arm = order[j];
+            uint32_t arm_index = arm == ARM_SEPARATE ? separate_count : pair_count;
+            uint32_t start = (uint32_t)((uint64_t)arm_index * cfg->dispatches %
+                                        f->sets);
+            double elapsed = 0.0;
+            ok = run_workload(f, arm, cfg->dispatches, start, &elapsed);
+            if (ok && arm == ARM_SEPARATE) separate[separate_count++] = elapsed;
+            if (ok && arm == ARM_PAIR) pair[pair_count++] = elapsed;
+        }
+    }
+    if (ok && (separate_count != cfg->samples || pair_count != cfg->samples)) {
+        ok = false;
+    }
+    if (ok) ok = check_outputs(f) && check_all_guards(f);
+
+    if (ok) {
+        qsort(separate, cfg->samples, sizeof(*separate), compare_double);
+        qsort(pair, cfg->samples, sizeof(*pair), compare_double);
+        const double separate_median = percentile(separate, cfg->samples, 0.50) *
+            1.0e6 / cfg->dispatches;
+        const double separate_p95 = percentile(separate, cfg->samples, 0.95) *
+            1.0e6 / cfg->dispatches;
+        const double pair_median = percentile(pair, cfg->samples, 0.50) *
+            1.0e6 / cfg->dispatches;
+        const double pair_p95 = percentile(pair, cfg->samples, 0.95) *
+            1.0e6 / cfg->dispatches;
+        const double saved = separate_median - pair_median;
+        const double working_set =
+            (double)(f->w0.length + f->w1.length) / (1024.0 * 1024.0);
+
+        printf("Metal Q4_K decode kernel-only A/B\n");
+        printf("  shape: n_tok=1, 4096->1024 + 4096->512\n");
+        printf("  resident anonymous weights: %.1f MiB across %u rotating sets\n",
+               working_set, f->sets);
+        printf("  design: alternating ABBA/BAAB, %u logical calls/sample, "
+               "%u samples/arm, GPU timestamps\n",
+               cfg->dispatches, cfg->samples);
+        printf("  separate (2 dispatches): median %.3f us, p95 %.3f us\n",
+               separate_median, separate_p95);
+        printf("  pair     (1 dispatch):   median %.3f us, p95 %.3f us\n",
+               pair_median, pair_p95);
+        printf("  saved: %.3f us/logical call, speedup %.3fx, reduction %.2f%%\n",
+               saved, separate_median / pair_median,
+               saved * 100.0 / separate_median);
+        printf("  correctness: bit-exact outputs; activation/output canaries intact\n");
+        printf("  scope: no GGUF, mmap, model runtime, SSD I/O, or CPU wall timing\n");
+    }
+
+    free(separate);
+    free(pair);
+    return ok;
+}
+
+int main(int argc, char **argv) {
+    @autoreleasepool {
+        const bench_config cfg = parse_options(argc, argv);
+        fixture f = {0};
+        if (!init_fixture(&f, &cfg)) return 1;
+        if (!run_oracle(&f)) return 1;
+        if (!run_benchmark(&f, &cfg)) return 1;
+        return 0;
+    }
+}
diff --git a/speed-bench/metal_q4_mm_tail_cull_bench.m b/speed-bench/metal_q4_mm_tail_cull_bench.m
new file mode 100644
index 0000000000..361c0afe3f
--- /dev/null
+++ b/speed-bench/metal_q4_mm_tail_cull_bench.m
@@ -0,0 +1,780 @@
+#import <Foundation/Foundation.h>
+#import <Metal/Metal.h>
+
+#include <errno.h>
+#include <limits.h>
+#include <math.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+enum {
+    QK_K = 256,
+    Q4_BLOCK_BYTES = 144,
+    THREADS_PER_GROUP = 128,
+    THREADGROUP_MEMORY_BYTES = 8192,
+    GUARD_BYTES = 256,
+    MAX_TOKENS = 65,
+    DEFAULT_IN_DIM = 4096,
+    DEFAULT_OUT_DIM = 1024,
+    DEFAULT_SAMPLES = 12,
+    DEFAULT_WARMUP_DISPATCHES = 16,
+};
+
+static const uint32_t k_guard = 0x7fc12345u;
+static const uint32_t k_baseline_poison = 0x7fc0b001u;
+static const uint32_t k_candidate_poison = 0x7fc0c001u;
+static const uint32_t k_token_cases[] = {
+    9u, 16u, 17u, 31u, 33u, 47u, 63u, 65u,
+};
+
+typedef struct {
+    uint16_t d;
+    uint16_t dmin;
+    uint8_t scales[12];
+    uint8_t qs[QK_K / 2];
+} block_q4_K_host;
+
+typedef struct {
+    int32_t ne00;
+    int32_t ne02;
+    uint64_t nb01;
+    uint64_t nb02;
+    uint64_t nb03;
+    int32_t ne12;
+    uint64_t nb10;
+    uint64_t nb11;
+    uint64_t nb12;
+    uint64_t nb13;
+    int32_t ne0;
+    int32_t ne1;
+    int16_t r2;
+    int16_t r3;
+} mul_mm_args;
+
+_Static_assert(sizeof(block_q4_K_host) == Q4_BLOCK_BYTES,
+               "Q4_K host fixture must match GGUF/Metal layout");
+_Static_assert(sizeof(mul_mm_args) == 88,
+               "Metal mul_mm argument ABI changed");
+
+typedef struct {
+    uint32_t in_dim;
+    uint32_t out_dim;
+    uint32_t sets;
+    uint32_t dispatches;
+    uint32_t warmup_dispatches;
+    uint32_t samples;
+    bool sets_explicit;
+    bool dispatches_explicit;
+} bench_config;
+
+typedef enum {
+    ARM_BASELINE,
+    ARM_CANDIDATE,
+} bench_arm;
+
+typedef struct {
+    __strong id<MTLDevice> device;
+    __strong id<MTLCommandQueue> queue;
+    __strong id<MTLComputePipelineState> baseline_pipeline;
+    __strong id<MTLComputePipelineState> candidate_pipeline;
+    __strong id<MTLBuffer> weights;
+    __strong id<MTLBuffer> x;
+    __strong id<MTLBuffer> baseline_out;
+    __strong id<MTLBuffer> candidate_out;
+    uint8_t *x_snapshot;
+    NSUInteger weight_set_bytes;
+    NSUInteger output_stride;
+    NSUInteger x_offset;
+    uint32_t in_dim;
+    uint32_t out_dim;
+    uint32_t sets;
+} fixture;
+
+static void usage(FILE *fp, const char *argv0) {
+    fprintf(fp,
+            "usage: %s [options]\n"
+            "\n"
+            "Kernel-only A/B of the production Metal Q4_K generic-MM tail "
+            "SIMDgroup cull.\n"
+            "\n"
+            "  --in-dim N       K dimension (default: %u)\n"
+            "  --out-dim N      M dimension (default: %u)\n"
+            "  --sets N         rotating resident weight sets (default: auto)\n"
+            "  --dispatches N   dispatches per sample (default: auto)\n"
+            "  --warmup N       dispatches per warmup arm (default: %u)\n"
+            "  --samples N      samples per arm, multiple of 4 (default: %u)\n"
+            "  -h, --help       show this help\n"
+            "\n"
+            "The default shape is the Flash Q-A projection 4096->1024. "
+            "Use --in-dim 1024 --out-dim 32768 for attn_q_b.\n"
+            "Set DS4_SOURCE_ROOT when running outside the repository root.\n",
+            argv0, DEFAULT_IN_DIM, DEFAULT_OUT_DIM,
+            DEFAULT_WARMUP_DISPATCHES, DEFAULT_SAMPLES);
+}
+
+static uint32_t parse_u32(const char *text, const char *option,
+                          uint32_t minimum) {
+    char *end = NULL;
+    errno = 0;
+    const unsigned long long value = strtoull(text, &end, 10);
+    if (errno != 0 || !text[0] || !end || *end ||
+        value < minimum || value > UINT32_MAX) {
+        fprintf(stderr, "metal-q4-mm-tail-cull-bench: invalid %s: %s\n",
+                option, text);
+        exit(2);
+    }
+    return (uint32_t)value;
+}
+
+static const char *need_arg(int *index, int argc, char **argv) {
+    if (*index + 1 >= argc) {
+        fprintf(stderr, "metal-q4-mm-tail-cull-bench: %s needs a value\n",
+                argv[*index]);
+        exit(2);
+    }
+    return argv[++*index];
+}
+
+static bench_config parse_options(int argc, char **argv) {
+    bench_config config = {
+        .in_dim = DEFAULT_IN_DIM,
+        .out_dim = DEFAULT_OUT_DIM,
+        .sets = 0u,
+        .dispatches = 0u,
+        .warmup_dispatches = DEFAULT_WARMUP_DISPATCHES,
+        .samples = DEFAULT_SAMPLES,
+    };
+    for (int i = 1; i < argc; i++) {
+        if (!strcmp(argv[i], "-h") || !strcmp(argv[i], "--help")) {
+            usage(stdout, argv[0]);
+            exit(0);
+        } else if (!strcmp(argv[i], "--in-dim")) {
+            const char *option = argv[i];
+            config.in_dim =
+                parse_u32(need_arg(&i, argc, argv), option, QK_K);
+        } else if (!strcmp(argv[i], "--out-dim")) {
+            const char *option = argv[i];
+            config.out_dim =
+                parse_u32(need_arg(&i, argc, argv), option, 64u);
+        } else if (!strcmp(argv[i], "--sets")) {
+            const char *option = argv[i];
+            config.sets = parse_u32(need_arg(&i, argc, argv), option, 1u);
+            config.sets_explicit = true;
+        } else if (!strcmp(argv[i], "--dispatches")) {
+            const char *option = argv[i];
+            config.dispatches =
+                parse_u32(need_arg(&i, argc, argv), option, 1u);
+            config.dispatches_explicit = true;
+        } else if (!strcmp(argv[i], "--warmup")) {
+            const char *option = argv[i];
+            config.warmup_dispatches =
+                parse_u32(need_arg(&i, argc, argv), option, 0u);
+        } else if (!strcmp(argv[i], "--samples")) {
+            const char *option = argv[i];
+            config.samples =
+                parse_u32(need_arg(&i, argc, argv), option, 4u);
+        } else {
+            fprintf(stderr,
+                    "metal-q4-mm-tail-cull-bench: unknown option: %s\n",
+                    argv[i]);
+            usage(stderr, argv[0]);
+            exit(2);
+        }
+    }
+    if ((config.in_dim % QK_K) != 0u ||
+        (config.out_dim % 64u) != 0u) {
+        fprintf(stderr,
+                "metal-q4-mm-tail-cull-bench: in-dim must be divisible by "
+                "%u and out-dim by 64\n", QK_K);
+        exit(2);
+    }
+    if ((config.samples % 4u) != 0u) {
+        fprintf(stderr,
+                "metal-q4-mm-tail-cull-bench: --samples must be divisible "
+                "by 4 for equal ABBA/BAAB cycles\n");
+        exit(2);
+    }
+    return config;
+}
+
+static bool checked_mul(NSUInteger a, NSUInteger b, NSUInteger *out) {
+    if (a != 0u && b > NSUIntegerMax / a) return false;
+    *out = a * b;
+    return true;
+}
+
+static NSUInteger align_up(NSUInteger value, NSUInteger alignment) {
+    return (value + alignment - 1u) / alignment * alignment;
+}
+
+static NSString *metal_prelude(void) {
+    return @"#include <metal_stdlib>\n"
+            "using namespace metal;\n"
+            "#define MAX(x, y) ((x) > (y) ? (x) : (y))\n"
+            "#define MIN(x, y) ((x) < (y) ? (x) : (y))\n"
+            "#define SWAP(x, y) { auto tmp = (x); (x) = (y); (y) = tmp; }\n"
+            "#define QK8_0 32\n"
+            "#ifndef QK_K\n#define QK_K 256\n#endif\n"
+            "#define N_SIMDWIDTH 32\n"
+            "#define N_R0_Q8_0 2\n"
+            "#define N_SG_Q8_0 4\n"
+            "#define FC_MUL_MV 600\n"
+            "#define FC_MUL_MM 700\n"
+            "#define FC_BIN 1300\n"
+            "#define FOR_UNROLL(x) _Pragma(\"clang loop unroll(full)\") for (x)\n"
+            "#define M_PI_F 3.14159265358979323846f\n"
+            "enum ds4_sort_order { DS4_SORT_ORDER_ASC, DS4_SORT_ORDER_DESC };\n"
+            "struct block_q8_0 { half d; int8_t qs[QK8_0]; };\n"
+            "struct block_q8_K { float d; int8_t qs[QK_K]; "
+            "int16_t bsums[QK_K / 16]; };\n";
+}
+
+static NSString *load_metal_source(void) {
+    static const char *paths[] = {
+        "metal/activations.metal",
+        "metal/flash_attn.metal",
+        "metal/dense.metal",
+        "metal/moe.metal",
+        "metal/dsv4_hc.metal",
+        "metal/unary.metal",
+        "metal/dsv4_kv.metal",
+        "metal/dsv4_rope.metal",
+        "metal/dsv4_misc.metal",
+        "metal/argsort.metal",
+        "metal/cpy.metal",
+        "metal/concat.metal",
+        "metal/get_rows.metal",
+        "metal/sum_rows.metal",
+        "metal/softmax.metal",
+        "metal/repeat.metal",
+        "metal/glu.metal",
+        "metal/norm.metal",
+        "metal/bin.metal",
+        "metal/set_rows.metal",
+    };
+    const char *root_env = getenv("DS4_SOURCE_ROOT");
+    NSString *root = root_env && root_env[0]
+        ? [NSString stringWithUTF8String:root_env]
+        : @".";
+    NSMutableString *source = [NSMutableString stringWithString:metal_prelude()];
+    for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); i++) {
+        NSString *relative = [NSString stringWithUTF8String:paths[i]];
+        NSString *path = [root stringByAppendingPathComponent:relative];
+        NSError *error = nil;
+        NSString *part = [NSString stringWithContentsOfFile:path
+                                                   encoding:NSUTF8StringEncoding
+                                                      error:&error];
+        if (!part) {
+            fprintf(stderr,
+                    "metal-q4-mm-tail-cull-bench: cannot read %s: %s\n",
+                    [path fileSystemRepresentation],
+                    [[error localizedDescription] UTF8String]);
+            return nil;
+        }
+        [source appendFormat:@"\n// appended %@\n%@\n", relative, part];
+    }
+    return source;
+}
+
+static id<MTLComputePipelineState> make_mm_pipeline(
+        id<MTLDevice> device, id<MTLLibrary> library, NSString *name) {
+    bool bc_inp = false;
+    bool bc_out = true;
+    MTLFunctionConstantValues *values = [MTLFunctionConstantValues new];
+    [values setConstantValue:&bc_inp type:MTLDataTypeBool atIndex:700];
+    [values setConstantValue:&bc_out type:MTLDataTypeBool atIndex:701];
+    NSError *error = nil;
+    id<MTLFunction> function = [library newFunctionWithName:name
+                                            constantValues:values
+                                                     error:&error];
+    if (!function) {
+        fprintf(stderr, "metal-q4-mm-tail-cull-bench: function %s: %s\n",
+                [name UTF8String], [[error localizedDescription] UTF8String]);
+        return nil;
+    }
+    id<MTLComputePipelineState> pipeline =
+        [device newComputePipelineStateWithFunction:function error:&error];
+    if (!pipeline) {
+        fprintf(stderr, "metal-q4-mm-tail-cull-bench: pipeline %s: %s\n",
+                [name UTF8String], [[error localizedDescription] UTF8String]);
+    }
+    return pipeline;
+}
+
+static uint32_t lcg_next(uint32_t *state) {
+    *state = *state * 1664525u + 1013904223u;
+    return *state;
+}
+
+static void fill_q4(void *storage, NSUInteger bytes) {
+    block_q4_K_host *blocks = storage;
+    const NSUInteger count = bytes / sizeof(*blocks);
+    uint32_t state = 0x41c64e6du;
+    for (NSUInteger b = 0; b < count; b++) {
+        blocks[b].d = (uint16_t)(0x2400u | (lcg_next(&state) & 0x03ffu));
+        blocks[b].dmin =
+            (uint16_t)(0x1c00u | (lcg_next(&state) & 0x03ffu));
+        for (size_t i = 0; i < sizeof(blocks[b].scales); i++) {
+            blocks[b].scales[i] = (uint8_t)lcg_next(&state);
+        }
+        for (size_t i = 0; i < sizeof(blocks[b].qs); i++) {
+            blocks[b].qs[i] = (uint8_t)lcg_next(&state);
+        }
+    }
+}
+
+static void fill_guard(id<MTLBuffer> buffer) {
+    uint32_t *words = buffer.contents;
+    for (NSUInteger i = 0; i < buffer.length / sizeof(*words); i++) {
+        words[i] = k_guard;
+    }
+}
+
+static void fill_output_payload(id<MTLBuffer> buffer, NSUInteger stride,
+                                uint32_t sets, NSUInteger payload_bytes,
+                                uint32_t poison) {
+    for (uint32_t set = 0; set < sets; set++) {
+        uint32_t *payload = (uint32_t *)((uint8_t *)buffer.contents +
+            (NSUInteger)set * stride + GUARD_BYTES);
+        for (NSUInteger offset = 0; offset < payload_bytes;
+             offset += sizeof(*payload)) {
+            payload[offset / sizeof(*payload)] = poison;
+        }
+    }
+}
+
+static void fill_activation(float *values, NSUInteger count) {
+    for (NSUInteger i = 0; i < count; i++) {
+        const int32_t centered = (int32_t)((i * 37u + i / 11u) % 257u) - 128;
+        values[i] = (float)centered / 96.0f;
+    }
+}
+
+static id<MTLBuffer> alloc_buffer(id<MTLDevice> device, NSUInteger bytes,
+                                  NSString *label) {
+    id<MTLBuffer> buffer =
+        [device newBufferWithLength:bytes options:MTLResourceStorageModeShared];
+    if (!buffer) {
+        fprintf(stderr,
+                "metal-q4-mm-tail-cull-bench: allocation failed for %s "
+                "(%llu bytes)\n",
+                [label UTF8String], (unsigned long long)bytes);
+        return nil;
+    }
+    buffer.label = label;
+    return buffer;
+}
+
+static bool init_fixture(fixture *f, bench_config *config) {
+    f->device = MTLCreateSystemDefaultDevice();
+    if (!f->device) {
+        fprintf(stderr, "metal-q4-mm-tail-cull-bench: no Metal device\n");
+        return false;
+    }
+    f->queue = [f->device newCommandQueue];
+    NSString *source = load_metal_source();
+    if (!f->queue || !source) return false;
+
+    NSError *error = nil;
+    id<MTLLibrary> library =
+        [f->device newLibraryWithSource:source options:nil error:&error];
+    if (!library) {
+        fprintf(stderr,
+                "metal-q4-mm-tail-cull-bench: Metal compile failed: %s\n",
+                [[error localizedDescription] UTF8String]);
+        return false;
+    }
+    f->baseline_pipeline = make_mm_pipeline(
+        f->device, library, @"kernel_mul_mm_q4_K_f32");
+    f->candidate_pipeline = make_mm_pipeline(
+        f->device, library, @"kernel_mul_mm_q4_K_f32_tail_cull");
+    if (!f->baseline_pipeline || !f->candidate_pipeline) return false;
+    if (f->baseline_pipeline.threadExecutionWidth != 32u ||
+        f->candidate_pipeline.threadExecutionWidth != 32u ||
+        f->baseline_pipeline.maxTotalThreadsPerThreadgroup < THREADS_PER_GROUP ||
+        f->candidate_pipeline.maxTotalThreadsPerThreadgroup < THREADS_PER_GROUP) {
+        fprintf(stderr,
+                "metal-q4-mm-tail-cull-bench: unexpected pipeline geometry\n");
+        return false;
+    }
+
+    f->in_dim = config->in_dim;
+    f->out_dim = config->out_dim;
+    const NSUInteger row_bytes =
+        (NSUInteger)(f->in_dim / QK_K) * Q4_BLOCK_BYTES;
+    if (!checked_mul(row_bytes, f->out_dim, &f->weight_set_bytes)) {
+        fprintf(stderr, "metal-q4-mm-tail-cull-bench: weight size overflow\n");
+        return false;
+    }
+    if (!config->sets_explicit) {
+        const NSUInteger target = 144u * 1024u * 1024u;
+        NSUInteger sets = (target + f->weight_set_bytes - 1u) /
+                          f->weight_set_bytes;
+        if (sets < 1u) sets = 1u;
+        if (sets > 64u) sets = 64u;
+        config->sets = (uint32_t)sets;
+    }
+    if (!config->dispatches_explicit) {
+        const NSUInteger target = 512u * 1024u * 1024u;
+        NSUInteger dispatches =
+            (target + f->weight_set_bytes - 1u) / f->weight_set_bytes;
+        if (dispatches < 8u) dispatches = 8u;
+        if (dispatches > 256u) dispatches = 256u;
+        config->dispatches = (uint32_t)dispatches;
+    }
+    f->sets = config->sets;
+
+    NSUInteger max_output_bytes = 0u;
+    if (!checked_mul((NSUInteger)MAX_TOKENS, f->out_dim,
+                     &max_output_bytes) ||
+        !checked_mul(max_output_bytes, sizeof(float), &max_output_bytes)) {
+        fprintf(stderr, "metal-q4-mm-tail-cull-bench: output size overflow\n");
+        return false;
+    }
+    f->output_stride =
+        align_up(GUARD_BYTES + max_output_bytes + GUARD_BYTES, 256u);
+    f->x_offset = GUARD_BYTES;
+
+    NSUInteger weight_bytes = 0u;
+    NSUInteger output_bytes = 0u;
+    NSUInteger x_payload_bytes = 0u;
+    if (!checked_mul(f->weight_set_bytes, f->sets, &weight_bytes) ||
+        !checked_mul(f->output_stride, f->sets, &output_bytes) ||
+        !checked_mul((NSUInteger)MAX_TOKENS, f->in_dim, &x_payload_bytes) ||
+        !checked_mul(x_payload_bytes, sizeof(float), &x_payload_bytes)) {
+        fprintf(stderr, "metal-q4-mm-tail-cull-bench: fixture size overflow\n");
+        return false;
+    }
+    f->weights = alloc_buffer(f->device, weight_bytes, @"q4-resident-weights");
+    f->x = alloc_buffer(f->device,
+                        GUARD_BYTES + x_payload_bytes + GUARD_BYTES,
+                        @"prefill-activation");
+    f->baseline_out =
+        alloc_buffer(f->device, output_bytes, @"q4-baseline-output");
+    f->candidate_out =
+        alloc_buffer(f->device, output_bytes, @"q4-candidate-output");
+    if (!f->weights || !f->x || !f->baseline_out || !f->candidate_out) {
+        return false;
+    }
+
+    fill_q4(f->weights.contents, f->weights.length);
+    fill_guard(f->x);
+    fill_activation((float *)((uint8_t *)f->x.contents + f->x_offset),
+                    (NSUInteger)MAX_TOKENS * f->in_dim);
+    fill_guard(f->baseline_out);
+    fill_guard(f->candidate_out);
+    f->x_snapshot = malloc(f->x.length);
+    if (!f->x_snapshot) {
+        fprintf(stderr,
+                "metal-q4-mm-tail-cull-bench: input snapshot allocation failed\n");
+        return false;
+    }
+    memcpy(f->x_snapshot, f->x.contents, f->x.length);
+    return true;
+}
+
+static mul_mm_args make_args(const fixture *f, uint32_t n_tokens) {
+    const uint64_t row_bytes =
+        (uint64_t)(f->in_dim / QK_K) * Q4_BLOCK_BYTES;
+    return (mul_mm_args) {
+        .ne00 = (int32_t)f->in_dim,
+        .ne02 = 1,
+        .nb01 = row_bytes,
+        .nb02 = row_bytes * f->out_dim,
+        .nb03 = row_bytes * f->out_dim,
+        .ne12 = 1,
+        .nb10 = sizeof(float),
+        .nb11 = (uint64_t)f->in_dim * sizeof(float),
+        .nb12 = (uint64_t)f->in_dim * n_tokens * sizeof(float),
+        .nb13 = (uint64_t)f->in_dim * n_tokens * sizeof(float),
+        .ne0 = (int32_t)f->out_dim,
+        .ne1 = (int32_t)n_tokens,
+        .r2 = 1,
+        .r3 = 1,
+    };
+}
+
+static void encode_dispatch(fixture *f, id<MTLComputeCommandEncoder> encoder,
+                            bench_arm arm, uint32_t n_tokens,
+                            uint32_t set) {
+    const mul_mm_args args = make_args(f, n_tokens);
+    id<MTLBuffer> output = arm == ARM_BASELINE
+        ? f->baseline_out : f->candidate_out;
+    [encoder setComputePipelineState:arm == ARM_BASELINE
+        ? f->baseline_pipeline : f->candidate_pipeline];
+    [encoder setBytes:&args length:sizeof(args) atIndex:0];
+    [encoder setBuffer:f->weights
+                 offset:(NSUInteger)set * f->weight_set_bytes
+                atIndex:1];
+    [encoder setBuffer:f->x offset:f->x_offset atIndex:2];
+    [encoder setBuffer:output
+                 offset:(NSUInteger)set * f->output_stride + GUARD_BYTES
+                atIndex:3];
+    [encoder setThreadgroupMemoryLength:THREADGROUP_MEMORY_BYTES atIndex:0];
+    [encoder dispatchThreadgroups:MTLSizeMake((n_tokens + 31u) / 32u,
+                                               f->out_dim / 64u, 1u)
+          threadsPerThreadgroup:MTLSizeMake(THREADS_PER_GROUP, 1u, 1u)];
+}
+
+static bool finish_command_buffer(id<MTLCommandBuffer> cb, const char *label) {
+    [cb commit];
+    [cb waitUntilCompleted];
+    if (cb.status == MTLCommandBufferStatusCompleted) return true;
+    fprintf(stderr, "metal-q4-mm-tail-cull-bench: %s failed: %s\n",
+            label, cb.error ? [[cb.error localizedDescription] UTF8String]
+                            : "unknown Metal error");
+    return false;
+}
+
+static bool check_guard_words(const uint8_t *bytes, NSUInteger begin,
+                              NSUInteger end, const char *label,
+                              uint32_t set, uint32_t n_tokens) {
+    for (NSUInteger offset = begin; offset < end; offset += sizeof(uint32_t)) {
+        uint32_t actual = 0u;
+        memcpy(&actual, bytes + offset, sizeof(actual));
+        if (actual != k_guard) {
+            fprintf(stderr,
+                    "metal-q4-mm-tail-cull-bench: %s canary changed "
+                    "N=%u set=%u offset=%llu\n",
+                    label, n_tokens, set, (unsigned long long)offset);
+            return false;
+        }
+    }
+    return true;
+}
+
+static bool check_outputs(fixture *f, uint32_t n_tokens) {
+    const NSUInteger payload_bytes =
+        (NSUInteger)n_tokens * f->out_dim * sizeof(float);
+    const uint8_t *baseline = f->baseline_out.contents;
+    const uint8_t *candidate = f->candidate_out.contents;
+    for (uint32_t set = 0; set < f->sets; set++) {
+        const NSUInteger record = (NSUInteger)set * f->output_stride;
+        const NSUInteger payload = record + GUARD_BYTES;
+        const NSUInteger payload_end = payload + payload_bytes;
+        if (memcmp(baseline + payload, candidate + payload, payload_bytes) != 0) {
+            fprintf(stderr,
+                    "metal-q4-mm-tail-cull-bench: bitwise mismatch "
+                    "N=%u set=%u\n", n_tokens, set);
+            return false;
+        }
+        if (!check_guard_words(baseline, record, payload,
+                               "baseline prefix", set, n_tokens) ||
+            !check_guard_words(candidate, record, payload,
+                               "candidate prefix", set, n_tokens) ||
+            !check_guard_words(baseline, payload_end,
+                               record + f->output_stride,
+                               "baseline suffix", set, n_tokens) ||
+            !check_guard_words(candidate, payload_end,
+                               record + f->output_stride,
+                               "candidate suffix", set, n_tokens)) {
+            return false;
+        }
+    }
+    if (memcmp(f->x.contents, f->x_snapshot, f->x.length) != 0) {
+        fprintf(stderr,
+                "metal-q4-mm-tail-cull-bench: input was modified at N=%u\n",
+                n_tokens);
+        return false;
+    }
+    return true;
+}
+
+static bool run_oracle_case(fixture *f, uint32_t n_tokens) {
+    fill_guard(f->baseline_out);
+    fill_guard(f->candidate_out);
+    const NSUInteger payload_bytes =
+        (NSUInteger)n_tokens * f->out_dim * sizeof(float);
+    fill_output_payload(f->baseline_out, f->output_stride, f->sets,
+                        payload_bytes, k_baseline_poison);
+    fill_output_payload(f->candidate_out, f->output_stride, f->sets,
+                        payload_bytes, k_candidate_poison);
+    id<MTLCommandBuffer> cb = [f->queue commandBuffer];
+    id<MTLComputeCommandEncoder> encoder = [cb computeCommandEncoder];
+    for (uint32_t set = 0; set < f->sets; set++) {
+        encode_dispatch(f, encoder, ARM_BASELINE, n_tokens, set);
+    }
+    for (uint32_t set = 0; set < f->sets; set++) {
+        encode_dispatch(f, encoder, ARM_CANDIDATE, n_tokens, set);
+    }
+    [encoder endEncoding];
+    if (!finish_command_buffer(cb, "oracle")) return false;
+    return check_outputs(f, n_tokens);
+}
+
+static bool run_workload(fixture *f, bench_arm arm, uint32_t n_tokens,
+                         uint32_t dispatches, uint32_t start_set,
+                         double *gpu_seconds) {
+    id<MTLCommandBuffer> cb = [f->queue commandBuffer];
+    id<MTLComputeCommandEncoder> encoder = [cb computeCommandEncoder];
+    for (uint32_t i = 0; i < dispatches; i++) {
+        encode_dispatch(f, encoder, arm, n_tokens,
+                        (start_set + i) % f->sets);
+    }
+    [encoder endEncoding];
+    if (!finish_command_buffer(cb, arm == ARM_BASELINE
+                               ? "baseline workload" : "candidate workload")) {
+        return false;
+    }
+    const double elapsed = cb.GPUEndTime - cb.GPUStartTime;
+    if (!(elapsed > 0.0)) {
+        fprintf(stderr,
+                "metal-q4-mm-tail-cull-bench: GPU timestamps unavailable\n");
+        return false;
+    }
+    *gpu_seconds = elapsed;
+    return true;
+}
+
+static int compare_double(const void *lhs, const void *rhs) {
+    const double a = *(const double *)lhs;
+    const double b = *(const double *)rhs;
+    return (a > b) - (a < b);
+}
+
+static double percentile(const double *sorted, uint32_t count, double p) {
+    const double position = p * (double)(count - 1u);
+    const uint32_t lo = (uint32_t)position;
+    const uint32_t hi = lo + 1u < count ? lo + 1u : lo;
+    const double fraction = position - (double)lo;
+    return sorted[lo] + (sorted[hi] - sorted[lo]) * fraction;
+}
+
+static bool run_benchmark_case(fixture *f, const bench_config *config,
+                               uint32_t n_tokens) {
+    double ignored = 0.0;
+    if (config->warmup_dispatches != 0u) {
+        static const bench_arm warm_order[] = {
+            ARM_BASELINE, ARM_CANDIDATE,
+            ARM_CANDIDATE, ARM_BASELINE,
+        };
+        for (size_t i = 0; i < sizeof(warm_order) / sizeof(warm_order[0]); i++) {
+            if (!run_workload(f, warm_order[i], n_tokens,
+                              config->warmup_dispatches,
+                              (uint32_t)(i * config->warmup_dispatches) % f->sets,
+                              &ignored)) {
+                return false;
+            }
+        }
+    }
+
+    double *baseline = calloc(config->samples, sizeof(*baseline));
+    double *candidate = calloc(config->samples, sizeof(*candidate));
+    double *baseline_sorted = calloc(config->samples, sizeof(*baseline_sorted));
+    double *candidate_sorted = calloc(config->samples, sizeof(*candidate_sorted));
+    if (!baseline || !candidate || !baseline_sorted || !candidate_sorted) {
+        fprintf(stderr,
+                "metal-q4-mm-tail-cull-bench: sample allocation failed\n");
+        free(baseline);
+        free(candidate);
+        free(baseline_sorted);
+        free(candidate_sorted);
+        return false;
+    }
+
+    uint32_t baseline_count = 0u;
+    uint32_t candidate_count = 0u;
+    bool ok = true;
+    for (uint32_t cycle = 0; ok && cycle < config->samples / 2u; cycle++) {
+        static const bench_arm abba[] = {
+            ARM_BASELINE, ARM_CANDIDATE,
+            ARM_CANDIDATE, ARM_BASELINE,
+        };
+        static const bench_arm baab[] = {
+            ARM_CANDIDATE, ARM_BASELINE,
+            ARM_BASELINE, ARM_CANDIDATE,
+        };
+        const bench_arm *order = (cycle & 1u) ? baab : abba;
+        for (uint32_t position = 0; ok && position < 4u; position++) {
+            const bench_arm arm = order[position];
+            const uint32_t arm_index = arm == ARM_BASELINE
+                ? baseline_count : candidate_count;
+            const uint32_t start_set =
+                (uint32_t)((uint64_t)arm_index * config->dispatches % f->sets);
+            double elapsed = 0.0;
+            ok = run_workload(f, arm, n_tokens, config->dispatches,
+                              start_set, &elapsed);
+            if (ok && arm == ARM_BASELINE) baseline[baseline_count++] = elapsed;
+            if (ok && arm == ARM_CANDIDATE) candidate[candidate_count++] = elapsed;
+        }
+    }
+    if (ok && (baseline_count != config->samples ||
+               candidate_count != config->samples)) {
+        ok = false;
+    }
+    if (ok) ok = check_outputs(f, n_tokens);
+
+    if (ok) {
+        memcpy(baseline_sorted, baseline,
+               config->samples * sizeof(*baseline_sorted));
+        memcpy(candidate_sorted, candidate,
+               config->samples * sizeof(*candidate_sorted));
+        qsort(baseline_sorted, config->samples,
+              sizeof(*baseline_sorted), compare_double);
+        qsort(candidate_sorted, config->samples,
+              sizeof(*candidate_sorted), compare_double);
+        const double scale = 1.0e6 / config->dispatches;
+        const double baseline_median =
+            percentile(baseline_sorted, config->samples, 0.50) * scale;
+        const double baseline_p95 =
+            percentile(baseline_sorted, config->samples, 0.95) * scale;
+        const double candidate_median =
+            percentile(candidate_sorted, config->samples, 0.50) * scale;
+        const double candidate_p95 =
+            percentile(candidate_sorted, config->samples, 0.95) * scale;
+        double log_speedup = 0.0;
+        for (uint32_t i = 0; i < config->samples; i++) {
+            log_speedup += log(baseline[i] / candidate[i]);
+        }
+        const double paired_speedup = exp(log_speedup / config->samples);
+        const double reduction =
+            (baseline_median - candidate_median) * 100.0 / baseline_median;
+        printf("  N=%-2u baseline %.3f us [p95 %.3f]  "
+               "candidate %.3f us [p95 %.3f]  "
+               "median %.3fx (%+.2f%%), paired-gmean %.3fx\n",
+               n_tokens, baseline_median, baseline_p95,
+               candidate_median, candidate_p95,
+               baseline_median / candidate_median, reduction,
+               paired_speedup);
+    }
+
+    free(baseline);
+    free(candidate);
+    free(baseline_sorted);
+    free(candidate_sorted);
+    return ok;
+}
+
+int main(int argc, char **argv) {
+    @autoreleasepool {
+        bench_config config = parse_options(argc, argv);
+        fixture f = {0};
+        if (!init_fixture(&f, &config)) return 1;
+
+        printf("Metal Q4_K generic-MM tail-cull kernel-only A/B\n");
+        printf("  shape: %u -> %u, resident anonymous Q4_K weights\n",
+               config.in_dim, config.out_dim);
+        printf("  working set: %.1f MiB across %u rotating sets\n",
+               (double)f.weights.length / (1024.0 * 1024.0), f.sets);
+        printf("  design: ABBA/BAAB, %u dispatches/sample, "
+               "%u samples/arm, GPU timestamps\n",
+               config.dispatches, config.samples);
+        for (size_t i = 0;
+             i < sizeof(k_token_cases) / sizeof(k_token_cases[0]); i++) {
+            if (!run_oracle_case(&f, k_token_cases[i])) return 1;
+            if (!run_benchmark_case(&f, &config, k_token_cases[i])) return 1;
+        }
+        fprintf(stderr,
+                "Metal Q4_K MM tail-cull oracle: PASS "
+                "(N=9,16,17,31,33,47,63,65; bit-exact; "
+                "distinct payload poison; canaries intact)\n");
+        printf("  correctness: bit-exact outputs; input/output canaries intact\n");
+        printf("  scope: no GGUF, mmap, model runtime, SSD I/O, uploads, "
+               "readback, or CPU wall timing in measured command buffers\n");
+        free(f.x_snapshot);
+        return 0;
+    }
+}
diff --git a/speed-bench/metal_q4_prefill_pair_bench.m b/speed-bench/metal_q4_prefill_pair_bench.m
new file mode 100644
index 0000000000..50e8be9b3d
--- /dev/null
+++ b/speed-bench/metal_q4_prefill_pair_bench.m
@@ -0,0 +1,405 @@
+#import <Foundation/Foundation.h>
+#import <Metal/Metal.h>
+
+#include <errno.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+enum {
+    IN_DIM = 4096,
+    OUT0_DIM = 1024,
+    OUT1_DIM = 512,
+    MAX_TOKENS = 128,
+    QK_K = 256,
+    Q4_BLOCK_BYTES = 144,
+    THREADS = 128,
+    SMEM = 6144,
+    GUARD_BYTES = 256,
+};
+
+static const uint32_t k_guard = 0x7fc12345u;
+static const uint32_t k_baseline_poison = 0x7fc0b001u;
+static const uint32_t k_pair_poison = 0x7fc0c001u;
+static const uint32_t k_tokens[] = {32u, 64u, 96u, 128u};
+
+typedef struct {
+    uint16_t d, dmin;
+    uint8_t scales[12];
+    uint8_t qs[QK_K / 2];
+} block_q4_K_host;
+
+typedef struct {
+    int32_t ne00, ne02;
+    uint64_t nb01, nb02, nb03;
+    int32_t ne12;
+    uint64_t nb10, nb11, nb12, nb13;
+    int32_t ne0, ne1;
+    int16_t r2, r3;
+} mul_mm_args;
+
+_Static_assert(sizeof(block_q4_K_host) == Q4_BLOCK_BYTES, "Q4_K ABI");
+_Static_assert(sizeof(mul_mm_args) == 88, "mul_mm ABI");
+
+typedef struct {
+    uint32_t sets, dispatches, warmup, samples;
+} config;
+
+typedef enum { ARM_BASELINE, ARM_PAIR_F16 } arm;
+
+typedef struct {
+    __strong id<MTLDevice> device;
+    __strong id<MTLCommandQueue> queue;
+    __strong id<MTLComputePipelineState> mm_f32;
+    __strong id<MTLComputePipelineState> mm_f16;
+    __strong id<MTLComputePipelineState> copy;
+    __strong id<MTLBuffer> w0, w1, x, rhs;
+    __strong id<MTLBuffer> base0, base1, pair0, pair1;
+    NSUInteger w0_set_bytes, w1_set_bytes;
+    NSUInteger x_off, rhs_off, out0_stride, out1_stride;
+    uint32_t sets;
+    uint8_t *x_snapshot;
+} fixture;
+
+static void usage(const char *argv0) {
+    printf("usage: %s [--sets N] [--dispatches N] [--warmup N] [--samples N]\n",
+           argv0);
+}
+
+static uint32_t parse_u32(const char *s, const char *name, uint32_t min) {
+    char *end = NULL;
+    errno = 0;
+    unsigned long long v = strtoull(s, &end, 10);
+    if (errno || !s[0] || !end || *end || v < min || v > UINT32_MAX) {
+        fprintf(stderr, "metal-q4-prefill-pair-bench: invalid %s: %s\n", name, s);
+        exit(2);
+    }
+    return (uint32_t)v;
+}
+
+static const char *arg_value(int *i, int argc, char **argv) {
+    if (*i + 1 >= argc) {
+        fprintf(stderr, "metal-q4-prefill-pair-bench: %s needs a value\n", argv[*i]);
+        exit(2);
+    }
+    return argv[++*i];
+}
+
+static config parse_options(int argc, char **argv) {
+    config c = {.sets=64u, .dispatches=32u, .warmup=8u, .samples=12u};
+    for (int i = 1; i < argc; i++) {
+        if (!strcmp(argv[i], "-h") || !strcmp(argv[i], "--help")) {
+            usage(argv[0]); exit(0);
+        } else if (!strcmp(argv[i], "--sets")) {
+            c.sets = parse_u32(arg_value(&i, argc, argv), "--sets", 1u);
+        } else if (!strcmp(argv[i], "--dispatches")) {
+            c.dispatches = parse_u32(arg_value(&i, argc, argv), "--dispatches", 1u);
+        } else if (!strcmp(argv[i], "--warmup")) {
+            c.warmup = parse_u32(arg_value(&i, argc, argv), "--warmup", 0u);
+        } else if (!strcmp(argv[i], "--samples")) {
+            c.samples = parse_u32(arg_value(&i, argc, argv), "--samples", 2u);
+        } else {
+            usage(argv[0]); exit(2);
+        }
+    }
+    if (c.samples & 1u) {
+        fprintf(stderr, "metal-q4-prefill-pair-bench: --samples must be even\n");
+        exit(2);
+    }
+    return c;
+}
+
+static NSString *prelude(void) {
+    return @"#include <metal_stdlib>\nusing namespace metal;\n"
+            "#define MAX(x,y) ((x)>(y)?(x):(y))\n"
+            "#define MIN(x,y) ((x)<(y)?(x):(y))\n"
+            "#define SWAP(x,y) { auto t=(x); (x)=(y); (y)=t; }\n"
+            "#define QK8_0 32\n#ifndef QK_K\n#define QK_K 256\n#endif\n"
+            "#define N_SIMDWIDTH 32\n#define N_R0_Q8_0 2\n#define N_SG_Q8_0 4\n"
+            "#define FC_MUL_MV 600\n#define FC_MUL_MM 700\n#define FC_BIN 1300\n"
+            "#define FOR_UNROLL(x) _Pragma(\"clang loop unroll(full)\") for (x)\n"
+            "#define M_PI_F 3.14159265358979323846f\n"
+            "enum ds4_sort_order { DS4_SORT_ORDER_ASC, DS4_SORT_ORDER_DESC };\n"
+            "struct block_q8_0 { half d; int8_t qs[QK8_0]; };\n"
+            "struct block_q8_K { float d; int8_t qs[QK_K]; int16_t bsums[QK_K/16]; };\n";
+}
+
+static NSString *load_source(void) {
+    static const char *files[] = {
+        "metal/activations.metal", "metal/flash_attn.metal", "metal/dense.metal",
+        "metal/moe.metal", "metal/dsv4_hc.metal", "metal/unary.metal",
+        "metal/dsv4_kv.metal", "metal/dsv4_rope.metal", "metal/dsv4_misc.metal",
+        "metal/argsort.metal", "metal/cpy.metal", "metal/concat.metal",
+        "metal/get_rows.metal", "metal/sum_rows.metal", "metal/softmax.metal",
+        "metal/repeat.metal", "metal/glu.metal", "metal/norm.metal",
+        "metal/bin.metal", "metal/set_rows.metal",
+    };
+    NSString *root = getenv("DS4_SOURCE_ROOT")
+        ? [NSString stringWithUTF8String:getenv("DS4_SOURCE_ROOT")] : @".";
+    NSMutableString *s = [NSMutableString stringWithString:prelude()];
+    for (size_t i = 0; i < sizeof(files)/sizeof(files[0]); i++) {
+        NSString *p = [root stringByAppendingPathComponent:
+            [NSString stringWithUTF8String:files[i]]];
+        NSError *e = nil;
+        NSString *part = [NSString stringWithContentsOfFile:p
+                           encoding:NSUTF8StringEncoding error:&e];
+        if (!part) {
+            fprintf(stderr, "metal-q4-prefill-pair-bench: read %s: %s\n",
+                    files[i], [[e localizedDescription] UTF8String]);
+            return nil;
+        }
+        [s appendFormat:@"\n%@\n", part];
+    }
+    return s;
+}
+
+static id<MTLComputePipelineState> pipeline(id<MTLDevice> d,
+        id<MTLLibrary> l, NSString *name, bool mm) {
+    NSError *e = nil;
+    id<MTLFunction> fn = nil;
+    if (mm) {
+        bool no = false;
+        MTLFunctionConstantValues *v = [MTLFunctionConstantValues new];
+        [v setConstantValue:&no type:MTLDataTypeBool atIndex:700];
+        [v setConstantValue:&no type:MTLDataTypeBool atIndex:701];
+        fn = [l newFunctionWithName:name constantValues:v error:&e];
+    } else {
+        fn = [l newFunctionWithName:name];
+    }
+    if (!fn) {
+        fprintf(stderr, "metal-q4-prefill-pair-bench: function %s: %s\n",
+                [name UTF8String], e ? [[e localizedDescription] UTF8String] : "missing");
+        return nil;
+    }
+    id<MTLComputePipelineState> p = [d newComputePipelineStateWithFunction:fn error:&e];
+    if (!p) fprintf(stderr, "metal-q4-prefill-pair-bench: pipeline %s: %s\n",
+                    [name UTF8String], [[e localizedDescription] UTF8String]);
+    return p;
+}
+
+static uint32_t rng(uint32_t *s) { *s = *s * 1664525u + 1013904223u; return *s; }
+static void fill_q4(void *ptr, NSUInteger bytes, uint32_t seed) {
+    block_q4_K_host *b = ptr;
+    for (NSUInteger i = 0; i < bytes / sizeof(*b); i++) {
+        b[i].d = (uint16_t)(0x2400u | (rng(&seed) & 0x3ffu));
+        b[i].dmin = (uint16_t)(0x1c00u | (rng(&seed) & 0x3ffu));
+        for (size_t j=0;j<sizeof(b[i].scales);j++) b[i].scales[j]=(uint8_t)(rng(&seed)>>24);
+        for (size_t j=0;j<sizeof(b[i].qs);j++) b[i].qs[j]=(uint8_t)(rng(&seed)>>24);
+    }
+}
+static void fill_guard(id<MTLBuffer> b) {
+    uint32_t *p=b.contents; for (NSUInteger i=0;i<b.length/4;i++) p[i]=k_guard;
+}
+static void fill_payload(id<MTLBuffer> b, NSUInteger stride, uint32_t sets,
+                         NSUInteger bytes, uint32_t poison) {
+    for (uint32_t s = 0; s < sets; s++) {
+        uint32_t *p = (uint32_t *)((uint8_t *)b.contents +
+            (NSUInteger)s * stride + GUARD_BYTES);
+        for (NSUInteger i = 0; i < bytes / sizeof(*p); i++) p[i] = poison;
+    }
+}
+static void fill_x(float *x) {
+    uint32_t s=0x243f6a88u;
+    for (NSUInteger i=0;i<(NSUInteger)IN_DIM*MAX_TOKENS;i++)
+        x[i]=((int32_t)(rng(&s)&0xffffu)-32768)/32768.0f;
+}
+static NSUInteger align_up(NSUInteger x, NSUInteger a) { return (x+a-1)/a*a; }
+static id<MTLBuffer> alloc(id<MTLDevice> d, NSUInteger n, NSString *label) {
+    if (!n || n>d.maxBufferLength) return nil;
+    id<MTLBuffer> b=[d newBufferWithLength:n options:MTLResourceStorageModeShared];
+    b.label=label; return b;
+}
+
+static bool is_pre_m5_apple_silicon_name(const char *name) {
+    return name && !strncmp(name, "Apple M", 7) &&
+           name[7] >= '1' && name[7] <= '4' &&
+           (name[8] == '\0' || name[8] == ' ');
+}
+
+static bool init_fixture(fixture *f, const config *c, id<MTLDevice> device) {
+    f->device=device;
+    f->queue=[f->device newCommandQueue];
+    NSString *src=load_source(); if (!f->device || !f->queue || !src) return false;
+    NSError *e=nil;
+    id<MTLLibrary> l=[f->device newLibraryWithSource:src options:nil error:&e];
+    if (!l) { fprintf(stderr,"metal-q4-prefill-pair-bench: compile: %s\n",
+                      [[e localizedDescription] UTF8String]); return false; }
+    f->mm_f32=pipeline(f->device,l,@"kernel_mul_mm_q4_K_f32",true);
+    f->mm_f16=pipeline(f->device,l,@"kernel_mul_mm_q4_K_f16_rhs",true);
+    f->copy=pipeline(f->device,l,@"kernel_cpy_contig_f32_f16_4",false);
+    if (!f->mm_f32 || !f->mm_f16 || !f->copy ||
+        f->mm_f32.maxTotalThreadsPerThreadgroup<THREADS ||
+        f->mm_f16.maxTotalThreadsPerThreadgroup<THREADS) return false;
+    f->sets=c->sets;
+    f->w0_set_bytes=(IN_DIM/QK_K)*Q4_BLOCK_BYTES*OUT0_DIM;
+    f->w1_set_bytes=(IN_DIM/QK_K)*Q4_BLOCK_BYTES*OUT1_DIM;
+    f->x_off=f->rhs_off=GUARD_BYTES;
+    f->out0_stride=align_up(GUARD_BYTES+(NSUInteger)MAX_TOKENS*OUT0_DIM*4+GUARD_BYTES,256);
+    f->out1_stride=align_up(GUARD_BYTES+(NSUInteger)MAX_TOKENS*OUT1_DIM*4+GUARD_BYTES,256);
+    f->w0=alloc(f->device,f->w0_set_bytes*c->sets,@"pair-w0");
+    f->w1=alloc(f->device,f->w1_set_bytes*c->sets,@"pair-w1");
+    f->x=alloc(f->device,GUARD_BYTES+(NSUInteger)MAX_TOKENS*IN_DIM*4+GUARD_BYTES,@"pair-x");
+    f->rhs=alloc(f->device,GUARD_BYTES+(NSUInteger)MAX_TOKENS*IN_DIM*2+GUARD_BYTES,@"pair-rhs");
+    f->base0=alloc(f->device,f->out0_stride*c->sets,@"base0");
+    f->base1=alloc(f->device,f->out1_stride*c->sets,@"base1");
+    f->pair0=alloc(f->device,f->out0_stride*c->sets,@"pair0");
+    f->pair1=alloc(f->device,f->out1_stride*c->sets,@"pair1");
+    if (!f->w0||!f->w1||!f->x||!f->rhs||!f->base0||!f->base1||!f->pair0||!f->pair1) return false;
+    fill_q4(f->w0.contents,f->w0.length,0x41c64e6du);
+    fill_q4(f->w1.contents,f->w1.length,0x9e3779b9u);
+    fill_guard(f->x); fill_x((float *)((uint8_t *)f->x.contents+f->x_off));
+    f->x_snapshot=malloc(f->x.length); if (!f->x_snapshot) return false;
+    memcpy(f->x_snapshot,f->x.contents,f->x.length);
+    return true;
+}
+
+static mul_mm_args args(uint32_t out, uint32_t n, bool half_rhs) {
+    uint64_t row=(IN_DIM/QK_K)*Q4_BLOCK_BYTES, elem=half_rhs?2u:4u;
+    return (mul_mm_args){.ne00=IN_DIM,.ne02=1,.nb01=row,.nb02=row*out,.nb03=row*out,
+        .ne12=1,.nb10=elem,.nb11=(uint64_t)IN_DIM*elem,
+        .nb12=(uint64_t)IN_DIM*n*elem,.nb13=(uint64_t)IN_DIM*n*elem,
+        .ne0=(int32_t)out,.ne1=(int32_t)n,.r2=1,.r3=1};
+}
+
+static void encode_mm(fixture *f,id<MTLComputeCommandEncoder> e,bool half,
+        uint32_t n,uint32_t set,id<MTLBuffer> o0,id<MTLBuffer> o1) {
+    mul_mm_args a0=args(OUT0_DIM,n,half), a1=args(OUT1_DIM,n,half);
+    [e setComputePipelineState:half?f->mm_f16:f->mm_f32];
+    [e setThreadgroupMemoryLength:SMEM atIndex:0];
+    [e setBytes:&a0 length:sizeof(a0) atIndex:0];
+    [e setBuffer:f->w0 offset:set*f->w0_set_bytes atIndex:1];
+    [e setBuffer:half?f->rhs:f->x offset:half?f->rhs_off:f->x_off atIndex:2];
+    [e setBuffer:o0 offset:set*f->out0_stride+GUARD_BYTES atIndex:3];
+    [e dispatchThreadgroups:MTLSizeMake(n/32,OUT0_DIM/64,1)
+         threadsPerThreadgroup:MTLSizeMake(THREADS,1,1)];
+    [e setBytes:&a1 length:sizeof(a1) atIndex:0];
+    [e setBuffer:f->w1 offset:set*f->w1_set_bytes atIndex:1];
+    [e setBuffer:o1 offset:set*f->out1_stride+GUARD_BYTES atIndex:3];
+    [e dispatchThreadgroups:MTLSizeMake(n/32,OUT1_DIM/64,1)
+         threadsPerThreadgroup:MTLSizeMake(THREADS,1,1)];
+}
+
+static void encode_copy(fixture *f,id<MTLComputeCommandEncoder> e,uint32_t n) {
+    uint32_t elems=n*IN_DIM;
+    NSUInteger work=(elems+3u)/4u, nth=256u;
+    if (nth>f->copy.maxTotalThreadsPerThreadgroup) nth=f->copy.maxTotalThreadsPerThreadgroup;
+    [e setComputePipelineState:f->copy]; [e setBytes:&elems length:4 atIndex:0];
+    [e setBuffer:f->x offset:f->x_off atIndex:1]; [e setBuffer:f->rhs offset:f->rhs_off atIndex:2];
+    [e dispatchThreadgroups:MTLSizeMake((work+nth-1)/nth,1,1)
+         threadsPerThreadgroup:MTLSizeMake(nth,1,1)];
+}
+
+static bool finish(id<MTLCommandBuffer> cb,const char *label,double *secs) {
+    [cb commit]; [cb waitUntilCompleted];
+    if (cb.status!=MTLCommandBufferStatusCompleted) {
+        fprintf(stderr,"metal-q4-prefill-pair-bench: %s: %s\n",label,
+                cb.error?[[cb.error localizedDescription] UTF8String]:"failed"); return false;
+    }
+    double t=cb.GPUEndTime-cb.GPUStartTime; if (!(t>0)) return false;
+    if (secs) *secs=t; return true;
+}
+
+static bool workload(fixture *f,arm a,uint32_t n,uint32_t calls,uint32_t start,double *secs) {
+    id<MTLCommandBuffer> cb=[f->queue commandBuffer];
+    if (a==ARM_BASELINE) {
+        id<MTLComputeCommandEncoder> e=[cb computeCommandEncoder];
+        for(uint32_t i=0;i<calls;i++) { uint32_t s=(start+i)%f->sets;
+            encode_mm(f,e,false,n,s,f->base0,f->base1); }
+        [e endEncoding];
+    } else {
+        for(uint32_t i=0;i<calls;i++) { uint32_t s=(start+i)%f->sets;
+            id<MTLComputeCommandEncoder> c=[cb computeCommandEncoder]; encode_copy(f,c,n); [c endEncoding];
+            id<MTLComputeCommandEncoder> m=[cb computeCommandEncoder]; encode_mm(f,m,true,n,s,f->pair0,f->pair1); [m endEncoding];
+        }
+    }
+    return finish(cb,a==ARM_BASELINE?"baseline":"pair-f16",secs);
+}
+
+static bool guards(fixture *f,id<MTLBuffer> b,NSUInteger stride,NSUInteger payload,const char *name) {
+    uint32_t *p=b.contents;
+    for(uint32_t s=0;s<f->sets;s++) {
+        NSUInteger base=s*stride, begin=base+GUARD_BYTES, end=begin+payload;
+        for(NSUInteger x=base;x<begin;x+=4) if(p[x/4]!=k_guard) goto bad;
+        for(NSUInteger x=end;x<base+stride;x+=4) if(p[x/4]!=k_guard) goto bad;
+    }
+    return true;
+bad: fprintf(stderr,"metal-q4-prefill-pair-bench: %s canary changed\n",name); return false;
+}
+
+static bool oracle(fixture *f,uint32_t n) {
+    fill_guard(f->rhs); fill_guard(f->base0); fill_guard(f->base1); fill_guard(f->pair0); fill_guard(f->pair1);
+    NSUInteger b0=(NSUInteger)n*OUT0_DIM*4, b1=(NSUInteger)n*OUT1_DIM*4;
+    fill_payload(f->base0, f->out0_stride, f->sets, b0, k_baseline_poison);
+    fill_payload(f->base1, f->out1_stride, f->sets, b1, k_baseline_poison);
+    fill_payload(f->pair0, f->out0_stride, f->sets, b0, k_pair_poison);
+    fill_payload(f->pair1, f->out1_stride, f->sets, b1, k_pair_poison);
+    double ignored;
+    if (!workload(f,ARM_BASELINE,n,f->sets,0,&ignored) ||
+        !workload(f,ARM_PAIR_F16,n,f->sets,0,&ignored)) return false;
+    for(uint32_t s=0;s<f->sets;s++) {
+        uint8_t *a0=(uint8_t *)f->base0.contents+s*f->out0_stride+GUARD_BYTES;
+        uint8_t *p0=(uint8_t *)f->pair0.contents+s*f->out0_stride+GUARD_BYTES;
+        uint8_t *a1=(uint8_t *)f->base1.contents+s*f->out1_stride+GUARD_BYTES;
+        uint8_t *p1=(uint8_t *)f->pair1.contents+s*f->out1_stride+GUARD_BYTES;
+        if(memcmp(a0,p0,b0)||memcmp(a1,p1,b1)) { fprintf(stderr,"mismatch N=%u set=%u\n",n,s); return false; }
+    }
+    if(memcmp(f->x.contents,f->x_snapshot,f->x.length)) return false;
+    uint32_t *r=f->rhs.contents;
+    for(NSUInteger x=0;x<f->rhs_off;x+=4) if(r[x/4]!=k_guard) return false;
+    for(NSUInteger x=f->rhs_off+(NSUInteger)n*IN_DIM*2;x<f->rhs.length;x+=4) if(r[x/4]!=k_guard) return false;
+    if(!guards(f,f->base0,f->out0_stride,b0,"base0")||!guards(f,f->base1,f->out1_stride,b1,"base1")||
+       !guards(f,f->pair0,f->out0_stride,b0,"pair0")||!guards(f,f->pair1,f->out1_stride,b1,"pair1")) return false;
+    fprintf(stderr,"Metal Q4 prefill pair oracle N=%u: PASS (bit-exact, canaries intact)\n",n);
+    return true;
+}
+
+static int cmp(const void *a,const void *b){double x=*(double*)a,y=*(double*)b;return(x>y)-(x<y);}
+static double median(double *x,uint32_t n){qsort(x,n,sizeof(*x),cmp);return n&1?x[n/2]:(x[n/2-1]+x[n/2])*0.5;}
+
+static bool bench_case(fixture *f,const config *c,uint32_t n) {
+    double tmp;
+    for(int i=0;i<4&&c->warmup;i++) if(!workload(f,(i==0||i==3)?ARM_BASELINE:ARM_PAIR_F16,n,c->warmup,i*c->warmup,&tmp)) return false;
+    double *b=calloc(c->samples,sizeof(*b)),*p=calloc(c->samples,sizeof(*p)); if(!b||!p) return false;
+    uint32_t nb=0,np=0;
+    for(uint32_t cyc=0;cyc<c->samples/2;cyc++) {
+        arm abba[]={ARM_BASELINE,ARM_PAIR_F16,ARM_PAIR_F16,ARM_BASELINE};
+        arm baab[]={ARM_PAIR_F16,ARM_BASELINE,ARM_BASELINE,ARM_PAIR_F16}; arm *order=cyc&1?baab:abba;
+        for(int j=0;j<4;j++) { arm a=order[j]; double t;
+            uint32_t idx=a==ARM_BASELINE?nb:np;
+            if(!workload(f,a,n,c->dispatches,(uint64_t)idx*c->dispatches%f->sets,&t)){free(b);free(p);return false;}
+            if(a==ARM_BASELINE)b[nb++]=t;else p[np++]=t;
+        }
+    }
+    double bm=median(b,c->samples)*1e6/c->dispatches, pm=median(p,c->samples)*1e6/c->dispatches;
+    printf("  N=%-3u baseline 2xQ4/F32 %.3f us | pair copy+2xQ4/F16 %.3f us | %+.2f%%\n",
+           n,bm,pm,(bm-pm)*100.0/bm); free(b);free(p); return true;
+}
+
+int main(int argc,char **argv) { @autoreleasepool {
+    config c=parse_options(argc,argv);
+    id<MTLDevice> device=MTLCreateSystemDefaultDevice();
+    if(!device) {
+        fprintf(stderr,"metal-q4-prefill-pair-bench: SKIP: no Metal device\n");
+        return 0;
+    }
+    const char *device_name=[[device name] UTF8String];
+    fprintf(stderr,"metal-q4-prefill-pair-bench: device=%s\n",
+            device_name?device_name:"unknown");
+    if(!is_pre_m5_apple_silicon_name(device_name)) {
+        fprintf(stderr,
+                "metal-q4-prefill-pair-bench: SKIP: runtime candidate requires Apple M1-M4\n");
+        return 0;
+    }
+    fixture f={0}; if(!init_fixture(&f,&c,device)) return 1;
+    printf("Metal Q4 q_a/KV prefill pair, resident GPU-event-only\n");
+    printf("  4096->1024 + 4096->512, %.1f MiB weights, %u sets, %u calls/sample\n",
+           (double)(f.w0.length+f.w1.length)/1048576.0,c.sets,c.dispatches);
+    for(size_t i=0;i<sizeof(k_tokens)/sizeof(k_tokens[0]);i++) {
+        if(!oracle(&f,k_tokens[i])||!bench_case(&f,&c,k_tokens[i])) return 1;
+    }
+    printf("  correctness: bit-exact; input/RHS/output canaries intact\n");
+    printf("  scope: resident buffers, GPU timestamps; no SSD/model runtime/CPU wall time\n");
+    free(f.x_snapshot); return 0;
+} }
diff --git a/speed-bench/rocm_q4_prefill_bench.cpp b/speed-bench/rocm_q4_prefill_bench.cpp
new file mode 100644
index 0000000000..9f4cbf03de
--- /dev/null
+++ b/speed-bench/rocm_q4_prefill_bench.cpp
@@ -0,0 +1,2077 @@
+// SPDX-License-Identifier: MIT
+// Resident, GPU-event-only ROCm Q4_K prefill microbenchmark.
+
+#include "ds4_gpu.h"
+
+#include <hip/hip_runtime.h>
+
+#include <algorithm>
+#include <cerrno>
+#include <cmath>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <functional>
+#include <limits>
+#include <string>
+#include <sys/mman.h>
+#include <utility>
+#include <vector>
+
+/* Deliberately local to this harness: the backend entry performs only the
+ * selected kernel enqueue after this process has completed all validation. */
+extern "C" void ds4_rocm_bench_q8_K_quantize_enqueue(
+    void *out, const void *x, uint32_t in_dim, uint32_t n_rows,
+    int use_wave32);
+extern "C" const void *ds4_rocm_bench_q4_K_resident_weight_ptr(
+    const void *model_map, uint64_t model_size, uint64_t weight_offset,
+    uint64_t weight_bytes);
+extern "C" int ds4_rocm_bench_q4_K_wmma_enqueue(
+    void *out, const void *w, const void *x, uint32_t n_tok,
+    uint32_t n_groups, uint32_t in_dim, uint32_t out_dim,
+    uint64_t row_bytes, uint64_t x_token_stride,
+    uint64_t x_group_stride, uint64_t out_token_stride,
+    uint32_t row_tile, int load2);
+extern "C" int ds4_rocm_bench_q4_K_wmma_variant_enqueue(
+    void *out, const void *w, const void *x, uint32_t n_tok,
+    uint32_t n_groups, uint32_t in_dim, uint32_t out_dim,
+    uint64_t row_bytes, uint64_t x_token_stride,
+    uint64_t x_group_stride, uint64_t out_token_stride,
+    uint32_t row_tile, uint32_t k_tile, int load2);
+extern "C" int ds4_rocm_bench_q4_K_wmma_k128_enqueue(
+    void *out, const void *w, const void *x, uint32_t n_tok,
+    uint32_t n_groups, uint32_t in_dim, uint32_t out_dim,
+    uint64_t row_bytes, uint64_t x_token_stride,
+    uint64_t x_group_stride, uint64_t out_token_stride);
+extern "C" void ds4_rocm_test_q4_prefill_wmma_reset(void);
+extern "C" uint64_t ds4_rocm_test_q4_prefill_wmma_get_calls(void);
+extern "C" uint64_t ds4_rocm_test_q4_prefill_wmma_k64_get_calls(void);
+
+namespace {
+
+constexpr uint32_t kQ4Type = 12u;
+constexpr uint32_t kQkK = 256u;
+constexpr uint32_t kDenseK = 4096u;
+constexpr uint32_t kDenseM = 1024u;
+constexpr uint32_t kKvM = 512u;
+constexpr uint32_t kQbK = 1024u;
+constexpr uint32_t kQbM = 32768u;
+constexpr uint32_t kOutputGroups = 8u;
+constexpr uint32_t kOutputRank = 1024u;
+constexpr uint32_t kOutputLowDim = kOutputGroups * kOutputRank;
+constexpr uint32_t kOutputM = 4096u;
+constexpr uint32_t kDefaultSets = 4u;
+constexpr uint32_t kDefaultSamples = 8u;
+constexpr uint32_t kDefaultWarmup = 2u;
+constexpr uint32_t kRawQ8GuardWords = 64u;
+/* Catch a bad N-tail predicate anywhere in the final 64-token WMMA tile,
+ * including the widest q_b output used by this harness. */
+constexpr uint32_t kGuardWords = (64u - 1u) * kQbM;
+constexpr uint64_t kCompareChunk = 4u * 1024u * 1024u;
+
+constexpr const char *kPrefillEnable =
+    "DS4_ROCM_ENABLE_Q4_PREFILL_TILE8";
+constexpr const char *kPrefillDisable =
+    "DS4_ROCM_DISABLE_Q4_PREFILL_TILE8";
+constexpr const char *kPrefillRequire =
+    "DS4_ROCM_REQUIRE_Q4_PREFILL_TILE8";
+constexpr const char *kK1024Tile4Disable =
+    "DS4_ROCM_DISABLE_Q4_PREFILL_K1024_TILE4";
+constexpr const char *kK1024Tile4SsdEnable =
+    "DS4_ROCM_ENABLE_Q4_PREFILL_K1024_TILE4_SSD";
+constexpr const char *kK1024Tile4Require =
+    "DS4_ROCM_REQUIRE_Q4_PREFILL_K1024_TILE4";
+constexpr const char *kWmmaEnable =
+    "DS4_ROCM_ENABLE_Q4_PREFILL_WMMA";
+constexpr const char *kWmmaSsdEnable =
+    "DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_SSD";
+constexpr const char *kWmmaDisable =
+    "DS4_ROCM_DISABLE_Q4_PREFILL_WMMA";
+constexpr const char *kWmmaRequire =
+    "DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA";
+constexpr const char *kWmmaRowTile =
+    "DS4_ROCM_Q4_PREFILL_WMMA_ROW_TILE";
+constexpr const char *kWmmaK64 =
+    "DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_K64";
+constexpr const char *kWmmaK128Disable =
+    "DS4_ROCM_DISABLE_Q4_PREFILL_WMMA_K128";
+constexpr const char *kQ8Wave32Enable =
+    "DS4_ROCM_ENABLE_Q4_PREFILL_Q8_K_WAVE32";
+constexpr const char *kQ8Wave32Disable =
+    "DS4_ROCM_DISABLE_Q4_PREFILL_Q8_K_WAVE32";
+constexpr const char *kQ8Wave32Require =
+    "DS4_ROCM_REQUIRE_Q4_PREFILL_Q8_K_WAVE32";
+
+struct block_q4_K_host {
+    uint16_t d;
+    uint16_t dmin;
+    uint8_t scales[12];
+    uint8_t qs[kQkK / 2u];
+};
+
+static_assert(sizeof(block_q4_K_host) == 144u,
+              "Q4_K fixture must match the raw GGUF layout");
+
+struct block_q8_K_host {
+    float d;
+    int8_t qs[kQkK];
+    int16_t bsums[kQkK / 16u];
+};
+
+static_assert(sizeof(block_q8_K_host) == 292u,
+              "Q8_K fixture must match the ROCm activation layout");
+
+enum class bench_case {
+    all,
+    dense,
+    pair,
+    qb,
+    outb,
+    output,
+};
+
+struct config {
+    bench_case selected = bench_case::all;
+    std::vector<uint32_t> tokens = {
+        9u, 17u, 33u, 128u, 256u, 257u, 512u};
+    uint32_t sets = kDefaultSets;
+    uint32_t samples = kDefaultSamples;
+    uint32_t warmup = kDefaultWarmup;
+    bool wmma_supported = false;
+};
+
+struct weight_set {
+    uint64_t dense_offset = 0;
+    uint64_t kv_offset = 0;
+    uint64_t qb_offset = 0;
+    uint64_t output_a_offset = 0;
+    uint64_t output_b_offset = 0;
+};
+
+struct model_fixture {
+    uint8_t *data = nullptr;
+    uint64_t size = 0;
+    uint64_t resident_bytes = 0;
+    FILE *file = nullptr;
+    std::vector<weight_set> weights;
+    std::vector<uint64_t> span_offsets;
+    std::vector<uint64_t> span_sizes;
+
+    ~model_fixture() {
+        if (data && size != 0u) (void)munmap(data, static_cast<size_t>(size));
+        if (file) std::fclose(file);
+    }
+    model_fixture() = default;
+    model_fixture(const model_fixture &) = delete;
+    model_fixture &operator=(const model_fixture &) = delete;
+};
+
+struct tensor_owner {
+    ds4_gpu_tensor *ptr = nullptr;
+
+    explicit tensor_owner(uint64_t bytes) : ptr(ds4_gpu_tensor_alloc(bytes)) {}
+    ~tensor_owner() { ds4_gpu_tensor_free(ptr); }
+    tensor_owner(const tensor_owner &) = delete;
+    tensor_owner &operator=(const tensor_owner &) = delete;
+};
+
+struct env_snapshot {
+    const char *name;
+    bool existed;
+    std::string value;
+
+    explicit env_snapshot(const char *key)
+        : name(key), existed(std::getenv(key) != nullptr),
+          value(existed ? std::getenv(key) : "") {}
+    ~env_snapshot() {
+        if (existed) {
+            (void)setenv(name, value.c_str(), 1);
+        } else {
+            (void)unsetenv(name);
+        }
+    }
+    env_snapshot(const env_snapshot &) = delete;
+    env_snapshot &operator=(const env_snapshot &) = delete;
+};
+
+struct event_timer {
+    hipEvent_t begin = nullptr;
+    hipEvent_t end = nullptr;
+
+    event_timer() {
+        if (hipEventCreate(&begin) != hipSuccess ||
+            hipEventCreate(&end) != hipSuccess) {
+            std::fprintf(stderr,
+                         "rocm-q4-prefill-bench: HIP event allocation failed\n");
+            std::exit(1);
+        }
+    }
+    ~event_timer() {
+        if (begin) (void)hipEventDestroy(begin);
+        if (end) (void)hipEventDestroy(end);
+    }
+
+    bool measure(const std::function<bool()> &dispatch, float *milliseconds) {
+        if (hipEventRecord(begin, 0) != hipSuccess) return false;
+        if (!dispatch()) return false;
+        if (hipEventRecord(end, 0) != hipSuccess ||
+            hipEventSynchronize(end) != hipSuccess ||
+            hipEventElapsedTime(milliseconds, begin, end) != hipSuccess) {
+            return false;
+        }
+        /* Outside the measured interval: surface an immediate launch/config
+         * error from enqueue-only benchmark hooks instead of false-greening. */
+        return hipGetLastError() == hipSuccess;
+    }
+};
+
+struct arm {
+    const char *name;
+    std::function<void()> prepare;
+    std::function<bool(uint32_t)> dispatch;
+};
+
+struct stats {
+    double minimum = 0.0;
+    double median = 0.0;
+    double p95 = 0.0;
+    double mean = 0.0;
+};
+
+enum class benchmark_rate {
+    macs,
+    quantized_values,
+};
+
+uint64_t align_up(uint64_t value, uint64_t alignment) {
+    return (value + alignment - 1u) / alignment * alignment;
+}
+
+bool checked_mul(uint64_t a, uint64_t b, uint64_t *out) {
+    if (a != 0u && b > std::numeric_limits<uint64_t>::max() / a) return false;
+    *out = a * b;
+    return true;
+}
+
+uint32_t lcg_next(uint32_t *state) {
+    *state = *state * 1664525u + 1013904223u;
+    return *state;
+}
+
+void fill_q4(void *storage, uint64_t bytes, uint32_t seed) {
+    auto *blocks = static_cast<block_q4_K_host *>(storage);
+    const uint64_t count = bytes / sizeof(*blocks);
+    uint32_t state = seed;
+    for (uint64_t i = 0; i < count; i++) {
+        // Positive, finite FP16 scales. The payload is deterministic but does
+        // not need a CPU oracle: the benchmark compares production GPU paths.
+        blocks[i].d = static_cast<uint16_t>(0x2400u + (lcg_next(&state) & 0xffu));
+        blocks[i].dmin =
+            static_cast<uint16_t>(0x2000u + (lcg_next(&state) & 0xffu));
+        for (uint8_t &v : blocks[i].scales) {
+            v = static_cast<uint8_t>(lcg_next(&state) >> 24u);
+        }
+        for (uint8_t &v : blocks[i].qs) {
+            v = static_cast<uint8_t>(lcg_next(&state) >> 24u);
+        }
+    }
+}
+
+uint64_t q4_weight_bytes(uint32_t in_dim, uint32_t out_dim) {
+    return static_cast<uint64_t>(out_dim) * (in_dim / kQkK) *
+           sizeof(block_q4_K_host);
+}
+
+uint32_t shape_wmma_row_tile(uint32_t out_dim) {
+    return out_dim >= 8192u ? 256u : (out_dim >= 1024u ? 128u : 64u);
+}
+
+bool resolve_resident_weights(
+        const model_fixture &model,
+        uint64_t weight_set::*offset_member,
+        uint64_t weight_bytes,
+        std::vector<const void *> *resolved) {
+    resolved->resize(model.weights.size());
+    for (size_t i = 0; i < model.weights.size(); i++) {
+        const uint64_t offset = model.weights[i].*offset_member;
+        (*resolved)[i] = ds4_rocm_bench_q4_K_resident_weight_ptr(
+            model.data, model.size, offset, weight_bytes);
+        if (!(*resolved)[i]) {
+            std::fprintf(stderr,
+                         "rocm-q4-prefill-bench: weight set %zu is not "
+                         "physically device-resident\n",
+                         i);
+            return false;
+        }
+    }
+    return true;
+}
+
+bool make_model(model_fixture *model, uint32_t sets) {
+    constexpr uint64_t page = 4096u;
+    const uint64_t dense_bytes = q4_weight_bytes(kDenseK, kDenseM);
+    const uint64_t kv_bytes = q4_weight_bytes(kDenseK, kKvM);
+    const uint64_t qb_bytes = q4_weight_bytes(kQbK, kQbM);
+    const uint64_t output_a_bytes =
+        q4_weight_bytes(kDenseK, kOutputLowDim);
+    const uint64_t output_b_bytes =
+        q4_weight_bytes(kOutputLowDim, kOutputM);
+
+    model->weights.resize(sets);
+    model->span_offsets.reserve(static_cast<size_t>(sets) * 5u);
+    model->span_sizes.reserve(static_cast<size_t>(sets) * 5u);
+    uint64_t cursor = 0;
+    auto append = [&](uint64_t bytes) {
+        const uint64_t offset = align_up(cursor, page);
+        cursor = offset + bytes;
+        model->span_offsets.push_back(offset);
+        model->span_sizes.push_back(bytes);
+        model->resident_bytes += bytes;
+        return offset;
+    };
+    for (uint32_t i = 0; i < sets; i++) {
+        model->weights[i].dense_offset = append(dense_bytes);
+        model->weights[i].kv_offset = append(kv_bytes);
+        model->weights[i].qb_offset = append(qb_bytes);
+        model->weights[i].output_a_offset = append(output_a_bytes);
+        model->weights[i].output_b_offset = append(output_b_bytes);
+    }
+    model->size = align_up(cursor, page);
+    void *storage = nullptr;
+    if (posix_memalign(&storage, static_cast<size_t>(page),
+                       static_cast<size_t>(model->size)) != 0) {
+        return false;
+    }
+    auto *staging = static_cast<uint8_t *>(storage);
+    std::memset(staging, 0, static_cast<size_t>(model->size));
+    for (uint32_t i = 0; i < sets; i++) {
+        fill_q4(staging + model->weights[i].dense_offset, dense_bytes,
+                0x243f6a88u ^ (i * 0x9e3779b9u));
+        fill_q4(staging + model->weights[i].kv_offset, kv_bytes,
+                0x85a308d3u ^ (i * 0x7f4a7c15u));
+        fill_q4(staging + model->weights[i].qb_offset, qb_bytes,
+                0x13198a2eu ^ (i * 0x94d049bbu));
+        fill_q4(staging + model->weights[i].output_a_offset, output_a_bytes,
+                0x03707344u ^ (i * 0x369dea0fu));
+        fill_q4(staging + model->weights[i].output_b_offset, output_b_bytes,
+                0xa4093822u ^ (i * 0xdb4f0b91u));
+    }
+
+    FILE *file = std::tmpfile();
+    if (!file ||
+        std::fwrite(staging, 1u, static_cast<size_t>(model->size), file) !=
+            static_cast<size_t>(model->size) ||
+        std::fflush(file) != 0) {
+        std::free(staging);
+        if (file) std::fclose(file);
+        return false;
+    }
+    void *mapping = mmap(nullptr, static_cast<size_t>(model->size), PROT_READ,
+                         MAP_PRIVATE, fileno(file), 0);
+    std::free(staging);
+    if (mapping == MAP_FAILED) {
+        std::fclose(file);
+        return false;
+    }
+    model->data = static_cast<uint8_t *>(mapping);
+    model->file = file;
+    return true;
+}
+
+void fill_activation(std::vector<float> *values, uint32_t n_tokens,
+                     uint32_t in_dim) {
+    values->resize(static_cast<uint64_t>(n_tokens) * in_dim);
+    for (uint32_t token = 0; token < n_tokens; token++) {
+        for (uint32_t block = 0; block < in_dim / kQkK; block++) {
+            float *dst = values->data() +
+                         static_cast<uint64_t>(token) * in_dim + block * kQkK;
+            for (uint32_t i = 0; i < kQkK; i++) {
+                const int q = static_cast<int>((i * 73u + token * 37u +
+                                                block * 19u) % 241u) - 120;
+                /* Exercise real F32->F16 rounding in the direct-WMMA
+                 * scalar/load2 oracle; division by 32 made every fixture
+                 * value exactly representable in F16. */
+                dst[i] = static_cast<float>(q) / 37.0f;
+            }
+            dst[0] = ((token + block) & 1u) ? 127.0f / 32.0f
+                                             : -127.0f / 32.0f;
+        }
+    }
+}
+
+std::vector<uint32_t> guard_pattern(uint32_t words = kGuardWords) {
+    std::vector<uint32_t> guard(words);
+    for (uint32_t i = 0; i < words; i++) guard[i] = 0x7fc12000u + i;
+    return guard;
+}
+
+bool prepare_guard(ds4_gpu_tensor *tensor, uint64_t logical_bytes,
+                   uint32_t guard_words = kGuardWords) {
+    const std::vector<uint32_t> guard = guard_pattern(guard_words);
+    return ds4_gpu_tensor_write(tensor, logical_bytes, guard.data(),
+                                guard.size() * sizeof(guard[0])) != 0;
+}
+
+bool poison_output(ds4_gpu_tensor *tensor, uint64_t logical_bytes,
+                   uint32_t pattern,
+                   uint32_t guard_words = kGuardWords) {
+    if (!tensor || logical_bytes == 0u ||
+        logical_bytes % sizeof(uint32_t) != 0u) {
+        return false;
+    }
+    const uint64_t chunk_bytes = std::min(kCompareChunk, logical_bytes);
+    std::vector<uint32_t> poison(
+        static_cast<size_t>(chunk_bytes / sizeof(uint32_t)), pattern);
+    for (uint64_t offset = 0; offset < logical_bytes; offset += chunk_bytes) {
+        const uint64_t count = std::min(chunk_bytes, logical_bytes - offset);
+        if (!ds4_gpu_tensor_write(tensor, offset, poison.data(), count)) {
+            return false;
+        }
+    }
+    return prepare_guard(tensor, logical_bytes, guard_words);
+}
+
+bool check_guard(const ds4_gpu_tensor *tensor, uint64_t logical_bytes,
+                 const char *label,
+                 uint32_t guard_words = kGuardWords) {
+    const std::vector<uint32_t> expected = guard_pattern(guard_words);
+    std::vector<uint32_t> got(expected.size());
+    if (!ds4_gpu_tensor_read(tensor, logical_bytes, got.data(),
+                             got.size() * sizeof(got[0]))) {
+        std::fprintf(stderr, "%s: guard read failed\n", label);
+        return false;
+    }
+    if (got != expected) {
+        const auto mismatch = std::mismatch(got.begin(), got.end(),
+                                            expected.begin());
+        std::fprintf(stderr, "%s: output guard overwritten at word %zu\n",
+                     label,
+                     static_cast<size_t>(mismatch.first - got.begin()));
+        return false;
+    }
+    return true;
+}
+
+bool bitwise_equal(const ds4_gpu_tensor *a, const ds4_gpu_tensor *b,
+                   uint64_t bytes, const char *label) {
+    const uint64_t chunk = std::min(kCompareChunk, bytes);
+    std::vector<uint8_t> lhs(static_cast<size_t>(chunk));
+    std::vector<uint8_t> rhs(static_cast<size_t>(chunk));
+    for (uint64_t offset = 0; offset < bytes; offset += chunk) {
+        const uint64_t count = std::min(chunk, bytes - offset);
+        if (!ds4_gpu_tensor_read(a, offset, lhs.data(), count) ||
+            !ds4_gpu_tensor_read(b, offset, rhs.data(), count)) {
+            std::fprintf(stderr, "%s: oracle read failed\n", label);
+            return false;
+        }
+        if (std::memcmp(lhs.data(), rhs.data(), static_cast<size_t>(count)) != 0) {
+            uint64_t first = 0;
+            while (first < count && lhs[static_cast<size_t>(first)] ==
+                                         rhs[static_cast<size_t>(first)]) {
+                first++;
+            }
+            std::fprintf(stderr,
+                         "%s: bitwise mismatch at output byte %llu\n", label,
+                         static_cast<unsigned long long>(offset + first));
+            return false;
+        }
+    }
+    return true;
+}
+
+bool numerically_close(const ds4_gpu_tensor *got,
+                       const ds4_gpu_tensor *reference,
+                       uint64_t bytes,
+                       const char *label,
+                       float abs_tolerance = 2.0f,
+                       float rel_tolerance = 3.0e-2f,
+                       bool gate = true) {
+    if ((bytes % sizeof(float)) != 0u) return false;
+    const uint64_t chunk_bytes =
+        std::min(kCompareChunk, bytes) & ~(uint64_t)(sizeof(float) - 1u);
+    std::vector<float> lhs(static_cast<size_t>(chunk_bytes / sizeof(float)));
+    std::vector<float> rhs(static_cast<size_t>(chunk_bytes / sizeof(float)));
+    uint64_t failures = 0u;
+    uint64_t nonfinite = 0u;
+    uint64_t compared = 0u;
+    float max_abs = 0.0f;
+    float max_rel = 0.0f;
+    uint64_t worst = 0u;
+    for (uint64_t offset = 0u; offset < bytes; offset += chunk_bytes) {
+        const uint64_t count = std::min(chunk_bytes, bytes - offset);
+        if (!ds4_gpu_tensor_read(got, offset, lhs.data(), count) ||
+            !ds4_gpu_tensor_read(reference, offset, rhs.data(), count)) {
+            std::fprintf(stderr, "%s: oracle read failed\n", label);
+            return false;
+        }
+        const uint64_t values = count / sizeof(float);
+        for (uint64_t i = 0u; i < values; i++) {
+            if (!std::isfinite(lhs[(size_t)i]) ||
+                !std::isfinite(rhs[(size_t)i])) {
+                failures++;
+                nonfinite++;
+                continue;
+            }
+            const float diff = std::fabs(lhs[(size_t)i] - rhs[(size_t)i]);
+            const float rel = diff /
+                std::max(1.0f, std::fabs(rhs[(size_t)i]));
+            if (diff > max_abs) {
+                max_abs = diff;
+                worst = compared + i;
+            }
+            max_rel = std::max(max_rel, rel);
+            if (diff > abs_tolerance +
+                       rel_tolerance * std::fabs(rhs[(size_t)i])) {
+                failures++;
+            }
+        }
+        compared += values;
+    }
+    std::fprintf(stderr,
+                 "%s: failures=%llu/%llu nonfinite=%llu max_abs=%g "
+                 "max_rel=%g worst=%llu tolerance(abs=%g rel=%g) %s\n",
+                 label, (unsigned long long)failures,
+                 (unsigned long long)compared,
+                 (unsigned long long)nonfinite, max_abs, max_rel,
+                 (unsigned long long)worst, abs_tolerance, rel_tolerance,
+                 failures == 0u ? "PASS" :
+                 (gate || nonfinite != 0u ? "FAIL" : "DIAGNOSTIC"));
+    return nonfinite == 0u && (failures == 0u || !gate);
+}
+
+void select_legacy() {
+    (void)unsetenv(kPrefillEnable);
+    (void)setenv(kPrefillDisable, "1", 1);
+    (void)unsetenv(kPrefillRequire);
+    (void)unsetenv(kK1024Tile4Disable);
+    (void)unsetenv(kK1024Tile4SsdEnable);
+    (void)unsetenv(kK1024Tile4Require);
+    (void)unsetenv(kWmmaEnable);
+    (void)unsetenv(kWmmaSsdEnable);
+    (void)setenv(kWmmaDisable, "1", 1);
+    (void)unsetenv(kWmmaRequire);
+    (void)unsetenv(kWmmaRowTile);
+    (void)unsetenv(kWmmaK64);
+    (void)unsetenv(kWmmaK128Disable);
+    (void)unsetenv(kQ8Wave32Enable);
+    (void)unsetenv(kQ8Wave32Disable);
+    (void)unsetenv(kQ8Wave32Require);
+}
+
+void select_tile8(bool disable_k1024_tile4) {
+    (void)unsetenv(kPrefillEnable);
+    (void)unsetenv(kPrefillDisable);
+    (void)setenv(kPrefillRequire, "1", 1);
+    (void)unsetenv(kK1024Tile4SsdEnable);
+    (void)unsetenv(kK1024Tile4Require);
+    (void)unsetenv(kWmmaEnable);
+    (void)unsetenv(kWmmaSsdEnable);
+    (void)setenv(kWmmaDisable, "1", 1);
+    (void)unsetenv(kWmmaRequire);
+    (void)unsetenv(kWmmaRowTile);
+    (void)unsetenv(kWmmaK64);
+    (void)unsetenv(kWmmaK128Disable);
+    (void)unsetenv(kQ8Wave32Enable);
+    (void)unsetenv(kQ8Wave32Disable);
+    (void)unsetenv(kQ8Wave32Require);
+    if (disable_k1024_tile4) {
+        (void)setenv(kK1024Tile4Disable, "1", 1);
+    } else {
+        (void)unsetenv(kK1024Tile4Disable);
+    }
+}
+
+void select_k1024_tile4() {
+    select_tile8(false);
+    (void)setenv(kK1024Tile4Require, "1", 1);
+}
+
+void select_wmma_shape() {
+    select_tile8(false);
+    /* WMMA and TILE8 are separate strict contracts.  The candidate must not
+     * inherit REQUIRE_TILE8 from the baseline selector. */
+    (void)unsetenv(kPrefillRequire);
+    (void)setenv(kWmmaEnable, "1", 1);
+    (void)unsetenv(kWmmaSsdEnable);
+    (void)unsetenv(kWmmaDisable);
+    (void)setenv(kWmmaRequire, "1", 1);
+    (void)unsetenv(kK1024Tile4Require);
+    (void)unsetenv(kWmmaRowTile);
+    (void)unsetenv(kWmmaK64);
+    (void)unsetenv(kWmmaK128Disable);
+}
+
+void select_wmma_attention_a_tile8_b() {
+    select_tile8(false);
+    /* ENABLE is the production A-only request.  Do not use REQUIRE here:
+     * REQUIRE deliberately promotes the numerically compounded output-B
+     * direct-WMMA path for diagnostics. */
+    (void)unsetenv(kPrefillRequire);
+    (void)setenv(kWmmaEnable, "1", 1);
+    (void)unsetenv(kWmmaSsdEnable);
+    (void)unsetenv(kWmmaDisable);
+    (void)unsetenv(kWmmaRequire);
+    (void)unsetenv(kK1024Tile4Require);
+    (void)unsetenv(kWmmaRowTile);
+    (void)unsetenv(kWmmaK64);
+    (void)unsetenv(kWmmaK128Disable);
+    ds4_rocm_test_q4_prefill_wmma_reset();
+}
+
+double percentile(std::vector<double> sorted, double fraction) {
+    std::sort(sorted.begin(), sorted.end());
+    if (sorted.empty()) return 0.0;
+    const double position = fraction * static_cast<double>(sorted.size() - 1u);
+    const size_t lo = static_cast<size_t>(std::floor(position));
+    const size_t hi = static_cast<size_t>(std::ceil(position));
+    const double alpha = position - static_cast<double>(lo);
+    return sorted[lo] + (sorted[hi] - sorted[lo]) * alpha;
+}
+
+stats summarize(const std::vector<double> &samples) {
+    stats out;
+    out.minimum = *std::min_element(samples.begin(), samples.end());
+    out.median = percentile(samples, 0.5);
+    out.p95 = percentile(samples, 0.95);
+    for (double sample : samples) out.mean += sample;
+    out.mean /= static_cast<double>(samples.size());
+    return out;
+}
+
+bool benchmark_arms(const char *case_name, uint32_t n_tokens, uint32_t in_dim,
+                    uint32_t out_dim, const config &cfg, const arm &baseline,
+                    const arm &candidate,
+                    const std::function<bool()> &oracle_prepare,
+                    const std::function<bool()> &oracle,
+                    benchmark_rate rate = benchmark_rate::macs) {
+    // Validate every rotating weight set and prime reusable Q8_K scratch
+    // before any timed event.  This catches data-dependent path errors without
+    // admitting readback or comparison work into the HIP-event interval.
+    for (uint32_t set = 0; set < cfg.sets; set++) {
+        if (!oracle_prepare()) {
+            std::fprintf(stderr,
+                         "rocm-q4-prefill-bench: %s oracle poison failed "
+                         "for weight set %u\n",
+                         case_name, set);
+            return false;
+        }
+        baseline.prepare();
+        if (!baseline.dispatch(set) || !ds4_gpu_synchronize()) return false;
+        candidate.prepare();
+        if (!candidate.dispatch(set) || !ds4_gpu_synchronize()) return false;
+        if (!oracle()) {
+            std::fprintf(stderr,
+                         "rocm-q4-prefill-bench: %s oracle failed for "
+                         "weight set %u\n",
+                         case_name, set);
+            return false;
+        }
+    }
+
+    for (uint32_t i = 0; i < cfg.warmup; i++) {
+        const uint32_t set = i % cfg.sets;
+        baseline.prepare();
+        if (!baseline.dispatch(set) || !ds4_gpu_synchronize()) return false;
+        candidate.prepare();
+        if (!candidate.dispatch(set) || !ds4_gpu_synchronize()) return false;
+    }
+
+    event_timer timer;
+    std::vector<double> a_samples;
+    std::vector<double> b_samples;
+    a_samples.reserve(cfg.samples);
+    b_samples.reserve(cfg.samples);
+
+    auto take = [&](const arm &which, uint32_t set,
+                    std::vector<double> *samples) {
+        which.prepare();
+        float elapsed = 0.0f;
+        const bool ok = timer.measure(
+            [&]() { return which.dispatch(set); }, &elapsed);
+        if (!ok) {
+            std::fprintf(stderr,
+                         "rocm-q4-prefill-bench: %s/%s timed dispatch failed\n",
+                         case_name, which.name);
+            return false;
+        }
+        samples->push_back(static_cast<double>(elapsed));
+        return true;
+    };
+
+    // Each cycle contributes two samples per arm. Alternating ABBA/BAAB
+    // balances first/last position, while both arms see identical weight sets.
+    for (uint32_t cycle = 0; a_samples.size() < cfg.samples; cycle++) {
+        const uint32_t set0 = (cycle * 2u) % cfg.sets;
+        const uint32_t set1 = (cycle * 2u + 1u) % cfg.sets;
+        if ((cycle & 1u) == 0u) {
+            if (!take(baseline, set0, &a_samples) ||
+                !take(candidate, set0, &b_samples) ||
+                !take(candidate, set1, &b_samples) ||
+                !take(baseline, set1, &a_samples)) return false;
+        } else {
+            if (!take(candidate, set0, &b_samples) ||
+                !take(baseline, set0, &a_samples) ||
+                !take(baseline, set1, &a_samples) ||
+                !take(candidate, set1, &b_samples)) return false;
+        }
+    }
+
+    /* Re-run one set after the timed samples.  Event synchronization catches
+     * asynchronous launch failures, while this final readback also catches a
+     * geometry-dependent overwrite or wrong result that appears only after
+     * repeated launches. */
+    const uint32_t post_set = cfg.sets - 1u;
+    if (!oracle_prepare()) return false;
+    baseline.prepare();
+    if (!baseline.dispatch(post_set) || !ds4_gpu_synchronize()) return false;
+    candidate.prepare();
+    if (!candidate.dispatch(post_set) || !ds4_gpu_synchronize()) return false;
+    if (!oracle()) {
+        std::fprintf(stderr,
+                     "rocm-q4-prefill-bench: %s post-timing oracle failed\n",
+                     case_name);
+        return false;
+    }
+
+    const stats a = summarize(a_samples);
+    const stats b = summarize(b_samples);
+    std::vector<double> paired_delta;
+    paired_delta.reserve(a_samples.size());
+    for (size_t i = 0; i < a_samples.size(); i++) {
+        paired_delta.push_back((b_samples[i] / a_samples[i] - 1.0) * 100.0);
+    }
+    const double paired_median = percentile(paired_delta, 0.5);
+    const double median_delta = (b.median / a.median - 1.0) * 100.0;
+    const double speedup = (a.median / b.median - 1.0) * 100.0;
+    const double work = static_cast<double>(n_tokens) * in_dim *
+        (rate == benchmark_rate::macs ? out_dim : 1u);
+    const double a_rate = work / (a.median * 1.0e6);
+    const double b_rate = work / (b.median * 1.0e6);
+
+    if (rate == benchmark_rate::quantized_values) {
+        std::printf(
+            "DS4_ROCM_Q4_PREFILL_BENCH case=%s N=%u K=%u "
+            "baseline=%s candidate=%s samples=%u sets=%u "
+            "baseline_ms_p50=%.6f candidate_ms_p50=%.6f "
+            "baseline_ms_min=%.6f candidate_ms_min=%.6f "
+            "baseline_ms_p95=%.6f candidate_ms_p95=%.6f "
+            "baseline_gvalue_s=%.3f candidate_gvalue_s=%.3f "
+            "candidate_delta_pct=%.3f paired_delta_pct_p50=%.3f "
+            "speedup_pct=%.3f\n",
+            case_name, n_tokens, in_dim, baseline.name, candidate.name,
+            cfg.samples, cfg.sets, a.median, b.median, a.minimum, b.minimum,
+            a.p95, b.p95, a_rate, b_rate, median_delta, paired_median,
+            speedup);
+    } else {
+        std::printf(
+            "DS4_ROCM_Q4_PREFILL_BENCH case=%s N=%u K=%u M=%u "
+            "baseline=%s candidate=%s samples=%u sets=%u "
+            "baseline_ms_p50=%.6f candidate_ms_p50=%.6f "
+            "baseline_ms_min=%.6f candidate_ms_min=%.6f "
+            "baseline_ms_p95=%.6f candidate_ms_p95=%.6f "
+            "baseline_gmac_s=%.3f candidate_gmac_s=%.3f "
+            "candidate_delta_pct=%.3f paired_delta_pct_p50=%.3f "
+            "speedup_pct=%.3f\n",
+            case_name, n_tokens, in_dim, out_dim, baseline.name,
+            candidate.name, cfg.samples, cfg.sets, a.median, b.median,
+            a.minimum, b.minimum, a.p95, b.p95, a_rate, b_rate,
+            median_delta, paired_median, speedup);
+    }
+    std::fflush(stdout);
+    return true;
+}
+
+bool allocate_io(uint32_t n_tokens, uint32_t in_dim, uint64_t out_elements,
+                 tensor_owner *x, tensor_owner *out_a, tensor_owner *out_b) {
+    std::vector<float> activation;
+    fill_activation(&activation, n_tokens, in_dim);
+    if (!x->ptr || !out_a->ptr || !out_b->ptr ||
+        !ds4_gpu_tensor_write(x->ptr, 0, activation.data(),
+                              activation.size() * sizeof(float))) {
+        return false;
+    }
+    const uint64_t logical_bytes = out_elements * sizeof(float);
+    return poison_output(out_a->ptr, logical_bytes, 0x7fc10001u) &&
+           poison_output(out_b->ptr, logical_bytes, 0x7fc20002u);
+}
+
+bool run_q8_quantizer(const config &cfg, ds4_gpu_tensor *x,
+                      uint32_t n_tokens) {
+    int active_device = -1;
+    hipDeviceProp_t properties{};
+    if (hipGetDevice(&active_device) != hipSuccess || active_device < 0 ||
+        hipGetDeviceProperties(&properties, active_device) != hipSuccess ||
+        properties.warpSize != 32 ||
+        std::strncmp(properties.gcnArchName, "gfx1151", 7u) != 0) {
+        std::fprintf(stderr,
+                     "dense_q8_wave32 N=%u: active device is not "
+                     "gfx1151 wave32\n",
+                     n_tokens);
+        return false;
+    }
+
+    uint64_t block_count = 0;
+    uint64_t logical_bytes = 0;
+    uint64_t x_bytes = 0;
+    if (!x ||
+        !checked_mul(n_tokens, kDenseK / kQkK, &block_count) ||
+        !checked_mul(block_count, sizeof(block_q8_K_host), &logical_bytes) ||
+        !checked_mul(static_cast<uint64_t>(n_tokens) * kDenseK,
+                     sizeof(float), &x_bytes) ||
+        logical_bytes > std::numeric_limits<uint64_t>::max() -
+                            kRawQ8GuardWords * sizeof(uint32_t) ||
+        ds4_gpu_tensor_bytes(x) < x_bytes) {
+        std::fprintf(stderr,
+                     "dense_q8_wave32 N=%u: raw tensor size overflow\n",
+                     n_tokens);
+        return false;
+    }
+
+    const uint64_t allocation_bytes = logical_bytes +
+        kRawQ8GuardWords * sizeof(uint32_t);
+    tensor_owner canonical(allocation_bytes);
+    tensor_owner wave32(allocation_bytes);
+    if (!canonical.ptr || !wave32.ptr) {
+        std::fprintf(stderr,
+                     "dense_q8_wave32 N=%u: raw tensor allocation failed\n",
+                     n_tokens);
+        return false;
+    }
+
+    /* contents() synchronizes; resolve all raw device pointers before any
+     * event is recorded so the timed callbacks contain one launch only. */
+    const void *x_device = ds4_gpu_tensor_contents(x);
+    void *canonical_device = ds4_gpu_tensor_contents(canonical.ptr);
+    void *wave32_device = ds4_gpu_tensor_contents(wave32.ptr);
+    if (!x_device || !canonical_device || !wave32_device) {
+        std::fprintf(stderr,
+                     "dense_q8_wave32 N=%u: device pointer resolution failed\n",
+                     n_tokens);
+        return false;
+    }
+
+    const arm q8_canonical = {
+        "q8_canonical_raw", []() {},
+        [&](uint32_t) {
+            ds4_rocm_bench_q8_K_quantize_enqueue(
+                canonical_device, x_device, kDenseK, n_tokens, 0);
+            return true;
+        }};
+    const arm q8_wave32 = {
+        "q8_wave32_raw", []() {},
+        [&](uint32_t) {
+            ds4_rocm_bench_q8_K_quantize_enqueue(
+                wave32_device, x_device, kDenseK, n_tokens, 1);
+            return true;
+        }};
+
+    config q8_cfg = cfg;
+    q8_cfg.sets = 1u;  // Raw quantization has no rotating weight set.
+    return benchmark_arms(
+        "dense_q8_wave32", n_tokens, kDenseK, 1u, q8_cfg,
+        q8_canonical, q8_wave32,
+        [&]() {
+            return poison_output(canonical.ptr, logical_bytes, 0x5a5a5a5au,
+                                 kRawQ8GuardWords) &&
+                   poison_output(wave32.ptr, logical_bytes, 0xa5a5a5a5u,
+                                 kRawQ8GuardWords);
+        },
+        [&]() {
+            return bitwise_equal(canonical.ptr, wave32.ptr, logical_bytes,
+                                 "raw canonical vs wave32 Q8_K") &&
+                   check_guard(canonical.ptr, logical_bytes,
+                               "raw canonical Q8_K", kRawQ8GuardWords) &&
+                   check_guard(wave32.ptr, logical_bytes,
+                               "raw wave32 Q8_K", kRawQ8GuardWords);
+        },
+        benchmark_rate::quantized_values);
+}
+
+bool run_dense(const model_fixture &model, const config &cfg,
+               uint32_t n_tokens) {
+    uint64_t out_elements = 0;
+    if (!checked_mul(n_tokens, kDenseM, &out_elements)) return false;
+    const uint64_t logical_bytes = out_elements * sizeof(float);
+    const uint64_t allocation_bytes = logical_bytes +
+                                      kGuardWords * sizeof(uint32_t);
+    tensor_owner x(static_cast<uint64_t>(n_tokens) * kDenseK * sizeof(float));
+    tensor_owner legacy(allocation_bytes);
+    tensor_owner tiled(allocation_bytes);
+    if (!allocate_io(n_tokens, kDenseK, out_elements, &x, &legacy, &tiled)) {
+        std::fprintf(stderr, "dense N=%u: tensor setup failed\n", n_tokens);
+        return false;
+    }
+    const arm baseline = {
+        "legacy", select_legacy,
+        [&](uint32_t set) {
+            return ds4_gpu_matmul_quant_tensor(
+                       legacy.ptr, model.data, model.size,
+                       model.weights[set].dense_offset, kQ4Type, kDenseK,
+                       kDenseM, x.ptr, n_tokens) != 0;
+        }};
+    const arm candidate = {
+        "tile8", []() { select_tile8(false); },
+        [&](uint32_t set) {
+            return ds4_gpu_matmul_quant_tensor(
+                       tiled.ptr, model.data, model.size,
+                       model.weights[set].dense_offset, kQ4Type, kDenseK,
+                       kDenseM, x.ptr, n_tokens) != 0;
+        }};
+    if (!benchmark_arms(
+            "dense", n_tokens, kDenseK, kDenseM, cfg, baseline, candidate,
+            [&]() {
+                return poison_output(legacy.ptr, logical_bytes, 0x7fc10001u) &&
+                       poison_output(tiled.ptr, logical_bytes, 0x7fc20002u);
+            },
+            [&]() {
+                return bitwise_equal(legacy.ptr, tiled.ptr, logical_bytes,
+                                     "dense legacy vs tile8") &&
+                       check_guard(legacy.ptr, logical_bytes,
+                                   "dense legacy oracle") &&
+                       check_guard(tiled.ptr, logical_bytes,
+                                   "dense tile8 oracle");
+            })) return false;
+    if (!bitwise_equal(legacy.ptr, tiled.ptr, logical_bytes,
+                       "dense legacy vs tile8") ||
+        !check_guard(legacy.ptr, logical_bytes, "dense legacy") ||
+        !check_guard(tiled.ptr, logical_bytes, "dense tile8")) {
+        return false;
+    }
+    if (!cfg.wmma_supported) return true;
+    if (!run_q8_quantizer(cfg, x.ptr, n_tokens)) return false;
+
+    if (n_tokens < 256u) return true;
+
+    const arm wmma_baseline = {
+        "tile8", []() { select_tile8(false); },
+        [&](uint32_t set) {
+            return ds4_gpu_matmul_quant_tensor(
+                       legacy.ptr, model.data, model.size,
+                       model.weights[set].dense_offset, kQ4Type, kDenseK,
+                       kDenseM, x.ptr, n_tokens) != 0;
+        }};
+    const arm wmma_candidate = {
+        "wmma_shape", select_wmma_shape,
+        [&](uint32_t set) {
+            return ds4_gpu_matmul_quant_tensor(
+                       tiled.ptr, model.data, model.size,
+                       model.weights[set].dense_offset, kQ4Type, kDenseK,
+                       kDenseM, x.ptr, n_tokens) != 0;
+        }};
+    if (!benchmark_arms(
+        "dense_wmma", n_tokens, kDenseK, kDenseM, cfg, wmma_baseline,
+        wmma_candidate,
+        [&]() {
+            return poison_output(legacy.ptr, logical_bytes, 0x7fc10001u) &&
+                   poison_output(tiled.ptr, logical_bytes, 0x7fc20002u);
+        },
+        [&]() {
+            return numerically_close(tiled.ptr, legacy.ptr, logical_bytes,
+                                     "dense WMMA shape vs TILE8") &&
+                   check_guard(legacy.ptr, logical_bytes,
+                               "dense WMMA baseline oracle") &&
+                   check_guard(tiled.ptr, logical_bytes,
+                               "dense WMMA shape oracle");
+        })) return false;
+
+    void *const x_device = ds4_gpu_tensor_contents(x.ptr);
+    void *const rows64_device = ds4_gpu_tensor_contents(legacy.ptr);
+    void *const shape_device = ds4_gpu_tensor_contents(tiled.ptr);
+    std::vector<const void *> dense_weights;
+    if (!x_device || !rows64_device || !shape_device ||
+        !resolve_resident_weights(
+            model, &weight_set::dense_offset,
+            q4_weight_bytes(kDenseK, kDenseM), &dense_weights)) {
+        std::fprintf(stderr,
+                     "dense_wmma_rows N=%u: direct setup failed\n", n_tokens);
+        return false;
+    }
+    const uint64_t row_bytes = q4_weight_bytes(kDenseK, 1u);
+    const arm geometry_baseline = {
+        "wmma_rows64_scalar", []() {},
+        [&](uint32_t set) {
+            return ds4_rocm_bench_q4_K_wmma_enqueue(
+                       rows64_device, dense_weights[set], x_device,
+                       n_tokens, 1u, kDenseK, kDenseM, row_bytes,
+                       kDenseK, 0u, kDenseM, 64u, 0) != 0;
+        }};
+    const arm geometry_candidate = {
+        "wmma_shape_scalar", []() {},
+        [&](uint32_t set) {
+            return ds4_rocm_bench_q4_K_wmma_enqueue(
+                       shape_device, dense_weights[set], x_device,
+                       n_tokens, 1u, kDenseK, kDenseM, row_bytes,
+                       kDenseK, 0u, kDenseM,
+                       shape_wmma_row_tile(kDenseM), 0) != 0;
+        }};
+    if (!benchmark_arms(
+        "dense_wmma_rows", n_tokens, kDenseK, kDenseM, cfg,
+        geometry_baseline, geometry_candidate,
+        [&]() {
+            return poison_output(legacy.ptr, logical_bytes, 0x7fc10001u) &&
+                   poison_output(tiled.ptr, logical_bytes, 0x7fc20002u);
+        },
+        [&]() {
+            return bitwise_equal(legacy.ptr, tiled.ptr, logical_bytes,
+                                 "dense WMMA rows64 vs shape") &&
+                   check_guard(legacy.ptr, logical_bytes,
+                               "dense WMMA rows64 oracle") &&
+                   check_guard(tiled.ptr, logical_bytes,
+                               "dense WMMA shape oracle");
+        })) return false;
+
+    const uint32_t fixed_row_tile = shape_wmma_row_tile(kDenseM);
+    const arm k32_baseline = {
+        "wmma_k32_rows128_load2", []() {},
+        [&](uint32_t set) {
+            return ds4_rocm_bench_q4_K_wmma_variant_enqueue(
+                       rows64_device, dense_weights[set], x_device,
+                       n_tokens, 1u, kDenseK, kDenseM, row_bytes,
+                       kDenseK, 0u, kDenseM, fixed_row_tile, 32u, 1) != 0;
+        }};
+    const arm k64_candidate = {
+        "wmma_k64p80_rows128_load2", []() {},
+        [&](uint32_t set) {
+            return ds4_rocm_bench_q4_K_wmma_variant_enqueue(
+                       shape_device, dense_weights[set], x_device,
+                       n_tokens, 1u, kDenseK, kDenseM, row_bytes,
+                       kDenseK, 0u, kDenseM, fixed_row_tile, 64u, 1) != 0;
+        }};
+    return benchmark_arms(
+        "dense_wmma_k32_k64", n_tokens, kDenseK, kDenseM, cfg,
+        k32_baseline, k64_candidate,
+        [&]() {
+            return poison_output(legacy.ptr, logical_bytes, 0x7fc10001u) &&
+                   poison_output(tiled.ptr, logical_bytes, 0x7fc20002u);
+        },
+        [&]() {
+            return bitwise_equal(legacy.ptr, tiled.ptr, logical_bytes,
+                                 "dense WMMA K32 vs K64/P80") &&
+                   check_guard(legacy.ptr, logical_bytes,
+                               "dense WMMA K32 oracle") &&
+                   check_guard(tiled.ptr, logical_bytes,
+                               "dense WMMA K64/P80 oracle");
+        });
+}
+
+bool run_pair(const model_fixture &model, const config &cfg,
+              uint32_t n_tokens) {
+    uint64_t out0_elements = 0, out1_elements = 0;
+    if (!checked_mul(n_tokens, kDenseM, &out0_elements) ||
+        !checked_mul(n_tokens, kKvM, &out1_elements)) return false;
+    const uint64_t out0_bytes = out0_elements * sizeof(float);
+    const uint64_t out1_bytes = out1_elements * sizeof(float);
+    const uint64_t guard_bytes = kGuardWords * sizeof(uint32_t);
+    tensor_owner x(static_cast<uint64_t>(n_tokens) * kDenseK * sizeof(float));
+    tensor_owner separate0(out0_bytes + guard_bytes);
+    tensor_owner separate1(out1_bytes + guard_bytes);
+    tensor_owner pair0(out0_bytes + guard_bytes);
+    tensor_owner pair1(out1_bytes + guard_bytes);
+    std::vector<float> activation;
+    fill_activation(&activation, n_tokens, kDenseK);
+    if (!x.ptr || !separate0.ptr || !separate1.ptr || !pair0.ptr || !pair1.ptr ||
+        !ds4_gpu_tensor_write(x.ptr, 0, activation.data(),
+                              activation.size() * sizeof(float)) ||
+        !prepare_guard(separate0.ptr, out0_bytes) ||
+        !prepare_guard(separate1.ptr, out1_bytes) ||
+        !prepare_guard(pair0.ptr, out0_bytes) ||
+        !prepare_guard(pair1.ptr, out1_bytes)) {
+        std::fprintf(stderr, "pair N=%u: tensor setup failed\n", n_tokens);
+        return false;
+    }
+    const arm baseline = {
+        "two_dense_tile8", []() { select_tile8(false); },
+        [&](uint32_t set) {
+            return ds4_gpu_matmul_quant_tensor(
+                       separate0.ptr, model.data, model.size,
+                       model.weights[set].dense_offset, kQ4Type, kDenseK,
+                       kDenseM, x.ptr, n_tokens) != 0 &&
+                   ds4_gpu_matmul_quant_tensor(
+                       separate1.ptr, model.data, model.size,
+                       model.weights[set].kv_offset, kQ4Type, kDenseK, kKvM,
+                       x.ptr, n_tokens) != 0;
+        }};
+    const arm candidate = {
+        "pair_tile8", []() { select_tile8(false); },
+        [&](uint32_t set) {
+            return ds4_gpu_matmul_q4_K_pair_tensor(
+                       pair0.ptr, pair1.ptr, model.data, model.size,
+                       model.weights[set].dense_offset,
+                       model.weights[set].kv_offset, kDenseK, kDenseM, kKvM,
+                       x.ptr, n_tokens) == 1;
+        }};
+    if (!benchmark_arms(
+            "pair", n_tokens, kDenseK, kDenseM + kKvM, cfg, baseline,
+            candidate,
+            [&]() {
+                return poison_output(separate0.ptr, out0_bytes, 0x7fc10001u) &&
+                       poison_output(separate1.ptr, out1_bytes, 0x7fc20002u) &&
+                       poison_output(pair0.ptr, out0_bytes, 0x7fc30003u) &&
+                       poison_output(pair1.ptr, out1_bytes, 0x7fc40004u);
+            },
+            [&]() {
+                return bitwise_equal(separate0.ptr, pair0.ptr, out0_bytes,
+                                     "pair q_a output") &&
+                       bitwise_equal(separate1.ptr, pair1.ptr, out1_bytes,
+                                     "pair kv output") &&
+                       check_guard(separate0.ptr, out0_bytes,
+                                   "pair separate q_a oracle") &&
+                       check_guard(separate1.ptr, out1_bytes,
+                                   "pair separate kv oracle") &&
+                       check_guard(pair0.ptr, out0_bytes,
+                                   "pair fused q_a oracle") &&
+                       check_guard(pair1.ptr, out1_bytes,
+                                   "pair fused kv oracle");
+            })) return false;
+    return bitwise_equal(separate0.ptr, pair0.ptr, out0_bytes,
+                         "pair q_a output") &&
+           bitwise_equal(separate1.ptr, pair1.ptr, out1_bytes,
+                         "pair kv output") &&
+           check_guard(separate0.ptr, out0_bytes, "pair separate q_a") &&
+           check_guard(separate1.ptr, out1_bytes, "pair separate kv") &&
+           check_guard(pair0.ptr, out0_bytes, "pair fused q_a") &&
+           check_guard(pair1.ptr, out1_bytes, "pair fused kv");
+}
+
+bool run_qb(const model_fixture &model, const config &cfg,
+            uint32_t n_tokens) {
+    uint64_t out_elements = 0;
+    if (!checked_mul(n_tokens, kQbM, &out_elements)) return false;
+    const uint64_t logical_bytes = out_elements * sizeof(float);
+    const uint64_t allocation_bytes = logical_bytes +
+                                      kGuardWords * sizeof(uint32_t);
+    tensor_owner x(static_cast<uint64_t>(n_tokens) * kQbK * sizeof(float));
+    tensor_owner tile8(allocation_bytes);
+    tensor_owner tile4(allocation_bytes);
+    if (!allocate_io(n_tokens, kQbK, out_elements, &x, &tile8, &tile4)) {
+        std::fprintf(stderr, "q_b N=%u: tensor setup failed\n", n_tokens);
+        return false;
+    }
+    const arm baseline = {
+        "tile8", []() { select_tile8(true); },
+        [&](uint32_t set) {
+            return ds4_gpu_matmul_quant_tensor(
+                       tile8.ptr, model.data, model.size,
+                       model.weights[set].qb_offset, kQ4Type, kQbK, kQbM,
+                       x.ptr, n_tokens) != 0;
+        }};
+    const arm candidate = {
+        "tile4", select_k1024_tile4,
+        [&](uint32_t set) {
+            return ds4_gpu_matmul_quant_tensor(
+                       tile4.ptr, model.data, model.size,
+                       model.weights[set].qb_offset, kQ4Type, kQbK, kQbM,
+                       x.ptr, n_tokens) != 0;
+        }};
+    if (!benchmark_arms(
+            "q_b", n_tokens, kQbK, kQbM, cfg, baseline, candidate,
+            [&]() {
+                return poison_output(tile8.ptr, logical_bytes, 0x7fc10001u) &&
+                       poison_output(tile4.ptr, logical_bytes, 0x7fc20002u);
+            },
+            [&]() {
+                return bitwise_equal(tile8.ptr, tile4.ptr, logical_bytes,
+                                     "q_b tile8 vs tile4") &&
+                       check_guard(tile8.ptr, logical_bytes,
+                                   "q_b tile8 oracle") &&
+                       check_guard(tile4.ptr, logical_bytes,
+                                   "q_b tile4 oracle");
+            })) return false;
+    if (!bitwise_equal(tile8.ptr, tile4.ptr, logical_bytes,
+                       "q_b tile8 vs tile4") ||
+        !check_guard(tile8.ptr, logical_bytes, "q_b tile8") ||
+        !check_guard(tile4.ptr, logical_bytes, "q_b tile4")) {
+        return false;
+    }
+    if (!cfg.wmma_supported || n_tokens < 256u) return true;
+
+    const arm wmma_baseline = {
+        "tile4", select_k1024_tile4,
+        [&](uint32_t set) {
+            return ds4_gpu_matmul_quant_tensor(
+                       tile8.ptr, model.data, model.size,
+                       model.weights[set].qb_offset, kQ4Type, kQbK, kQbM,
+                       x.ptr, n_tokens) != 0;
+        }};
+    const arm wmma_candidate = {
+        "wmma_shape", select_wmma_shape,
+        [&](uint32_t set) {
+            return ds4_gpu_matmul_quant_tensor(
+                       tile4.ptr, model.data, model.size,
+                       model.weights[set].qb_offset, kQ4Type, kQbK, kQbM,
+                       x.ptr, n_tokens) != 0;
+        }};
+    if (!benchmark_arms(
+        "q_b_wmma", n_tokens, kQbK, kQbM, cfg, wmma_baseline,
+        wmma_candidate,
+        [&]() {
+            return poison_output(tile8.ptr, logical_bytes, 0x7fc10001u) &&
+                   poison_output(tile4.ptr, logical_bytes, 0x7fc20002u);
+        },
+        [&]() {
+            return numerically_close(tile4.ptr, tile8.ptr, logical_bytes,
+                                     "q_b WMMA shape vs TILE4") &&
+                   check_guard(tile8.ptr, logical_bytes,
+                               "q_b WMMA baseline oracle") &&
+                   check_guard(tile4.ptr, logical_bytes,
+                               "q_b WMMA shape oracle");
+        })) return false;
+
+    void *const x_device = ds4_gpu_tensor_contents(x.ptr);
+    void *const rows64_device = ds4_gpu_tensor_contents(tile8.ptr);
+    void *const shape_device = ds4_gpu_tensor_contents(tile4.ptr);
+    std::vector<const void *> qb_weights;
+    if (!x_device || !rows64_device || !shape_device ||
+        !resolve_resident_weights(
+            model, &weight_set::qb_offset,
+            q4_weight_bytes(kQbK, kQbM), &qb_weights)) {
+        std::fprintf(stderr,
+                     "q_b_wmma_rows N=%u: direct setup failed\n", n_tokens);
+        return false;
+    }
+    const uint64_t row_bytes = q4_weight_bytes(kQbK, 1u);
+    const arm geometry_baseline = {
+        "wmma_rows64_scalar", []() {},
+        [&](uint32_t set) {
+            return ds4_rocm_bench_q4_K_wmma_enqueue(
+                       rows64_device, qb_weights[set], x_device,
+                       n_tokens, 1u, kQbK, kQbM, row_bytes,
+                       kQbK, 0u, kQbM, 64u, 0) != 0;
+        }};
+    const arm rows128_candidate = {
+        "wmma_rows128_scalar", []() {},
+        [&](uint32_t set) {
+            return ds4_rocm_bench_q4_K_wmma_enqueue(
+                       shape_device, qb_weights[set], x_device,
+                       n_tokens, 1u, kQbK, kQbM, row_bytes,
+                       kQbK, 0u, kQbM, 128u, 0) != 0;
+        }};
+    if (!benchmark_arms(
+        "q_b_wmma_rows64_128", n_tokens, kQbK, kQbM, cfg,
+        geometry_baseline, rows128_candidate,
+        [&]() {
+            return poison_output(tile8.ptr, logical_bytes, 0x7fc10001u) &&
+                   poison_output(tile4.ptr, logical_bytes, 0x7fc20002u);
+        },
+        [&]() {
+            return bitwise_equal(tile8.ptr, tile4.ptr, logical_bytes,
+                                 "q_b WMMA rows64 vs rows128") &&
+                   check_guard(tile8.ptr, logical_bytes,
+                               "q_b WMMA rows64 oracle") &&
+                   check_guard(tile4.ptr, logical_bytes,
+                               "q_b WMMA rows128 oracle");
+        })) return false;
+
+    const arm rows128_baseline = {
+        "wmma_rows128_scalar", []() {},
+        [&](uint32_t set) {
+            return ds4_rocm_bench_q4_K_wmma_enqueue(
+                       rows64_device, qb_weights[set], x_device,
+                       n_tokens, 1u, kQbK, kQbM, row_bytes,
+                       kQbK, 0u, kQbM, 128u, 0) != 0;
+        }};
+    const arm rows256_candidate = {
+        "wmma_rows256_scalar", []() {},
+        [&](uint32_t set) {
+            return ds4_rocm_bench_q4_K_wmma_enqueue(
+                       shape_device, qb_weights[set], x_device,
+                       n_tokens, 1u, kQbK, kQbM, row_bytes,
+                       kQbK, 0u, kQbM, 256u, 0) != 0;
+        }};
+    if (!benchmark_arms(
+        "q_b_wmma_rows128_256", n_tokens, kQbK, kQbM, cfg,
+        rows128_baseline, rows256_candidate,
+        [&]() {
+            return poison_output(tile8.ptr, logical_bytes, 0x7fc10001u) &&
+                   poison_output(tile4.ptr, logical_bytes, 0x7fc20002u);
+        },
+        [&]() {
+            return bitwise_equal(tile8.ptr, tile4.ptr, logical_bytes,
+                                 "q_b WMMA rows128 vs rows256") &&
+                   check_guard(tile8.ptr, logical_bytes,
+                               "q_b WMMA rows128 oracle") &&
+                   check_guard(tile4.ptr, logical_bytes,
+                               "q_b WMMA rows256 oracle");
+        })) return false;
+
+    const arm rows128_scalar = {
+        "wmma_rows128_scalar", []() {},
+        [&](uint32_t set) {
+            return ds4_rocm_bench_q4_K_wmma_enqueue(
+                       rows64_device, qb_weights[set], x_device,
+                       n_tokens, 1u, kQbK, kQbM, row_bytes,
+                       kQbK, 0u, kQbM, 128u, 0) != 0;
+        }};
+    const arm rows128_load2 = {
+        "wmma_rows128_load2", []() {},
+        [&](uint32_t set) {
+            return ds4_rocm_bench_q4_K_wmma_enqueue(
+                       shape_device, qb_weights[set], x_device,
+                       n_tokens, 1u, kQbK, kQbM, row_bytes,
+                       kQbK, 0u, kQbM, 128u, 1) != 0;
+        }};
+    if (!benchmark_arms(
+        "q_b_wmma_load2_128", n_tokens, kQbK, kQbM, cfg,
+        rows128_scalar, rows128_load2,
+        [&]() {
+            return poison_output(tile8.ptr, logical_bytes, 0x7fc10001u) &&
+                   poison_output(tile4.ptr, logical_bytes, 0x7fc20002u);
+        },
+        [&]() {
+            return bitwise_equal(tile8.ptr, tile4.ptr, logical_bytes,
+                                 "q_b WMMA rows128 scalar vs load2") &&
+                   check_guard(tile8.ptr, logical_bytes,
+                               "q_b WMMA rows128 scalar oracle") &&
+                   check_guard(tile4.ptr, logical_bytes,
+                               "q_b WMMA rows128 load2 oracle");
+        })) return false;
+
+    const arm rows256_scalar = {
+        "wmma_rows256_scalar", []() {},
+        [&](uint32_t set) {
+            return ds4_rocm_bench_q4_K_wmma_enqueue(
+                       rows64_device, qb_weights[set], x_device,
+                       n_tokens, 1u, kQbK, kQbM, row_bytes,
+                       kQbK, 0u, kQbM, 256u, 0) != 0;
+        }};
+    const arm rows256_load2 = {
+        "wmma_rows256_load2", []() {},
+        [&](uint32_t set) {
+            return ds4_rocm_bench_q4_K_wmma_enqueue(
+                       shape_device, qb_weights[set], x_device,
+                       n_tokens, 1u, kQbK, kQbM, row_bytes,
+                       kQbK, 0u, kQbM, 256u, 1) != 0;
+        }};
+    if (!benchmark_arms(
+        "q_b_wmma_load2_256", n_tokens, kQbK, kQbM, cfg,
+        rows256_scalar, rows256_load2,
+        [&]() {
+            return poison_output(tile8.ptr, logical_bytes, 0x7fc10001u) &&
+                   poison_output(tile4.ptr, logical_bytes, 0x7fc20002u);
+        },
+        [&]() {
+            return bitwise_equal(tile8.ptr, tile4.ptr, logical_bytes,
+                                 "q_b WMMA rows256 scalar vs load2") &&
+                   check_guard(tile8.ptr, logical_bytes,
+                               "q_b WMMA rows256 scalar oracle") &&
+                   check_guard(tile4.ptr, logical_bytes,
+                               "q_b WMMA rows256 load2 oracle");
+        })) return false;
+
+    const arm k32_baseline = {
+        "wmma_k32_rows256_load2", []() {},
+        [&](uint32_t set) {
+            return ds4_rocm_bench_q4_K_wmma_variant_enqueue(
+                       rows64_device, qb_weights[set], x_device,
+                       n_tokens, 1u, kQbK, kQbM, row_bytes,
+                       kQbK, 0u, kQbM, 256u, 32u, 1) != 0;
+        }};
+    const arm k64_candidate = {
+        "wmma_k64p80_rows256_load2", []() {},
+        [&](uint32_t set) {
+            return ds4_rocm_bench_q4_K_wmma_variant_enqueue(
+                       shape_device, qb_weights[set], x_device,
+                       n_tokens, 1u, kQbK, kQbM, row_bytes,
+                       kQbK, 0u, kQbM, 256u, 64u, 1) != 0;
+        }};
+    if (!benchmark_arms(
+        "q_b_wmma_k32_k64", n_tokens, kQbK, kQbM, cfg,
+        k32_baseline, k64_candidate,
+        [&]() {
+            return poison_output(tile8.ptr, logical_bytes, 0x7fc10001u) &&
+                   poison_output(tile4.ptr, logical_bytes, 0x7fc20002u);
+        },
+        [&]() {
+            return bitwise_equal(tile8.ptr, tile4.ptr, logical_bytes,
+                                 "q_b WMMA K32 vs K64/P80") &&
+                   check_guard(tile8.ptr, logical_bytes,
+                               "q_b WMMA K32 oracle") &&
+                   check_guard(tile4.ptr, logical_bytes,
+                               "q_b WMMA K64/P80 oracle");
+        })) return false;
+
+    const arm k64_baseline = {
+        "wmma_k64p80_rows256_load2", []() {},
+        [&](uint32_t set) {
+            return ds4_rocm_bench_q4_K_wmma_variant_enqueue(
+                       rows64_device, qb_weights[set], x_device,
+                       n_tokens, 1u, kQbK, kQbM, row_bytes,
+                       kQbK, 0u, kQbM, 256u, 64u, 1) != 0;
+        }};
+    const arm k128_candidate = {
+        "wmma_k128p144_rows256_load4", []() {},
+        [&](uint32_t set) {
+            return ds4_rocm_bench_q4_K_wmma_k128_enqueue(
+                       shape_device, qb_weights[set], x_device,
+                       n_tokens, 1u, kQbK, kQbM, row_bytes,
+                       kQbK, 0u, kQbM) != 0;
+        }};
+    return benchmark_arms(
+        "q_b_wmma_k64_k128", n_tokens, kQbK, kQbM, cfg,
+        k64_baseline, k128_candidate,
+        [&]() {
+            return poison_output(tile8.ptr, logical_bytes, 0x7fc10001u) &&
+                   poison_output(tile4.ptr, logical_bytes, 0x7fc20002u);
+        },
+        [&]() {
+            return bitwise_equal(tile8.ptr, tile4.ptr, logical_bytes,
+                                 "q_b WMMA K64/P80 vs K128/P144") &&
+                   check_guard(tile8.ptr, logical_bytes,
+                               "q_b WMMA K64/P80 oracle") &&
+                   check_guard(tile4.ptr, logical_bytes,
+                               "q_b WMMA K128/P144 oracle");
+        });
+}
+
+bool run_output_b(const model_fixture &model, const config &cfg,
+                  uint32_t n_tokens) {
+    if (!cfg.wmma_supported || n_tokens < 256u) return true;
+    uint64_t out_elements = 0;
+    if (!checked_mul(n_tokens, kOutputM, &out_elements)) return false;
+    const uint64_t logical_bytes = out_elements * sizeof(float);
+    const uint64_t allocation_bytes = logical_bytes +
+                                      kGuardWords * sizeof(uint32_t);
+    tensor_owner x(
+        static_cast<uint64_t>(n_tokens) * kOutputLowDim * sizeof(float));
+    tensor_owner tile8(allocation_bytes);
+    tensor_owner wmma(allocation_bytes);
+    if (!allocate_io(n_tokens, kOutputLowDim, out_elements,
+                     &x, &tile8, &wmma)) {
+        std::fprintf(stderr, "output_b N=%u: tensor setup failed\n", n_tokens);
+        return false;
+    }
+
+    const arm baseline = {
+        "tile8", []() { select_tile8(false); },
+        [&](uint32_t set) {
+            return ds4_gpu_matmul_quant_tensor(
+                       tile8.ptr, model.data, model.size,
+                       model.weights[set].output_b_offset, kQ4Type,
+                       kOutputLowDim, kOutputM, x.ptr, n_tokens) != 0;
+        }};
+    const arm candidate = {
+        "wmma_shape", select_wmma_shape,
+        [&](uint32_t set) {
+            return ds4_gpu_matmul_quant_tensor(
+                       wmma.ptr, model.data, model.size,
+                       model.weights[set].output_b_offset, kQ4Type,
+                       kOutputLowDim, kOutputM, x.ptr, n_tokens) != 0;
+        }};
+    if (!benchmark_arms(
+        "output_b_wmma", n_tokens, kOutputLowDim, kOutputM, cfg,
+        baseline, candidate,
+        [&]() {
+            return poison_output(tile8.ptr, logical_bytes, 0x7fc10001u) &&
+                   poison_output(wmma.ptr, logical_bytes, 0x7fc20002u);
+        },
+        [&]() {
+            return numerically_close(wmma.ptr, tile8.ptr, logical_bytes,
+                                     "output_b WMMA shape vs TILE8") &&
+                   check_guard(tile8.ptr, logical_bytes,
+                               "output_b TILE8 oracle") &&
+                   check_guard(wmma.ptr, logical_bytes,
+                               "output_b WMMA shape oracle");
+        })) return false;
+
+    void *const x_device = ds4_gpu_tensor_contents(x.ptr);
+    void *const rows64_device = ds4_gpu_tensor_contents(tile8.ptr);
+    void *const shape_device = ds4_gpu_tensor_contents(wmma.ptr);
+    std::vector<const void *> output_b_weights;
+    if (!x_device || !rows64_device || !shape_device ||
+        !resolve_resident_weights(
+            model, &weight_set::output_b_offset,
+            q4_weight_bytes(kOutputLowDim, kOutputM), &output_b_weights)) {
+        std::fprintf(stderr,
+                     "output_b_wmma_rows N=%u: direct setup failed\n",
+                     n_tokens);
+        return false;
+    }
+    const uint64_t row_bytes = q4_weight_bytes(kOutputLowDim, 1u);
+    const arm geometry_baseline = {
+        "wmma_rows64_scalar", []() {},
+        [&](uint32_t set) {
+            return ds4_rocm_bench_q4_K_wmma_enqueue(
+                       rows64_device, output_b_weights[set], x_device,
+                       n_tokens, 1u, kOutputLowDim, kOutputM, row_bytes,
+                       kOutputLowDim, 0u, kOutputM, 64u, 0) != 0;
+        }};
+    const arm geometry_candidate = {
+        "wmma_shape_scalar", []() {},
+        [&](uint32_t set) {
+            return ds4_rocm_bench_q4_K_wmma_enqueue(
+                       shape_device, output_b_weights[set], x_device,
+                       n_tokens, 1u, kOutputLowDim, kOutputM, row_bytes,
+                       kOutputLowDim, 0u, kOutputM,
+                       shape_wmma_row_tile(kOutputM), 0) != 0;
+        }};
+    if (!benchmark_arms(
+        "output_b_wmma_rows", n_tokens, kOutputLowDim, kOutputM, cfg,
+        geometry_baseline, geometry_candidate,
+        [&]() {
+            return poison_output(tile8.ptr, logical_bytes, 0x7fc10001u) &&
+                   poison_output(wmma.ptr, logical_bytes, 0x7fc20002u);
+        },
+        [&]() {
+            return bitwise_equal(tile8.ptr, wmma.ptr, logical_bytes,
+                                 "output_b WMMA rows64 vs shape") &&
+                   check_guard(tile8.ptr, logical_bytes,
+                               "output_b WMMA rows64 oracle") &&
+                   check_guard(wmma.ptr, logical_bytes,
+                               "output_b WMMA shape oracle");
+        })) return false;
+
+    const uint32_t fixed_row_tile = shape_wmma_row_tile(kOutputM);
+    const arm k32_baseline = {
+        "wmma_k32_rows128_load2", []() {},
+        [&](uint32_t set) {
+            return ds4_rocm_bench_q4_K_wmma_variant_enqueue(
+                       rows64_device, output_b_weights[set], x_device,
+                       n_tokens, 1u, kOutputLowDim, kOutputM, row_bytes,
+                       kOutputLowDim, 0u, kOutputM, fixed_row_tile,
+                       32u, 1) != 0;
+        }};
+    const arm k64_candidate = {
+        "wmma_k64p80_rows128_load2", []() {},
+        [&](uint32_t set) {
+            return ds4_rocm_bench_q4_K_wmma_variant_enqueue(
+                       shape_device, output_b_weights[set], x_device,
+                       n_tokens, 1u, kOutputLowDim, kOutputM, row_bytes,
+                       kOutputLowDim, 0u, kOutputM, fixed_row_tile,
+                       64u, 1) != 0;
+        }};
+    return benchmark_arms(
+        "output_b_wmma_k32_k64", n_tokens, kOutputLowDim, kOutputM, cfg,
+        k32_baseline, k64_candidate,
+        [&]() {
+            return poison_output(tile8.ptr, logical_bytes, 0x7fc10001u) &&
+                   poison_output(wmma.ptr, logical_bytes, 0x7fc20002u);
+        },
+        [&]() {
+            return bitwise_equal(tile8.ptr, wmma.ptr, logical_bytes,
+                                 "output_b WMMA K32 vs K64/P80") &&
+                   check_guard(tile8.ptr, logical_bytes,
+                               "output_b WMMA K32 oracle") &&
+                   check_guard(wmma.ptr, logical_bytes,
+                               "output_b WMMA K64/P80 oracle");
+        });
+}
+
+bool run_attention_output(const model_fixture &model, const config &cfg,
+                          uint32_t n_tokens) {
+    if (!cfg.wmma_supported || n_tokens < 256u) return true;
+    uint64_t heads_elements = 0;
+    uint64_t low_elements = 0;
+    uint64_t out_elements = 0;
+    if (!checked_mul(n_tokens,
+                     static_cast<uint64_t>(kOutputGroups) * kDenseK,
+                     &heads_elements) ||
+        !checked_mul(n_tokens, kOutputLowDim, &low_elements) ||
+        !checked_mul(n_tokens, kOutputM, &out_elements)) {
+        return false;
+    }
+    const uint64_t low_bytes = low_elements * sizeof(float);
+    const uint64_t out_bytes = out_elements * sizeof(float);
+    const uint64_t guard_bytes = kGuardWords * sizeof(uint32_t);
+    tensor_owner heads(heads_elements * sizeof(float));
+    tensor_owner tile8_low(low_bytes + guard_bytes);
+    tensor_owner tile8_out(out_bytes + guard_bytes);
+    tensor_owner wmma_low(low_bytes + guard_bytes);
+    tensor_owner wmma_out(out_bytes + guard_bytes);
+    tensor_owner replay_out(out_bytes + guard_bytes);
+    std::vector<float> activation;
+    fill_activation(&activation, n_tokens, kOutputGroups * kDenseK);
+    if (!heads.ptr || !tile8_low.ptr || !tile8_out.ptr || !wmma_low.ptr ||
+        !wmma_out.ptr || !replay_out.ptr ||
+        !ds4_gpu_tensor_write(heads.ptr, 0, activation.data(),
+                              activation.size() * sizeof(float))) {
+        std::fprintf(stderr,
+                     "attention_output N=%u: tensor setup failed\n", n_tokens);
+        return false;
+    }
+
+    auto poison_outputs = [&]() {
+        return poison_output(tile8_low.ptr, low_bytes, 0x7fc10001u) &&
+               poison_output(tile8_out.ptr, out_bytes, 0x7fc20002u) &&
+               poison_output(wmma_low.ptr, low_bytes, 0x7fc30003u) &&
+               poison_output(wmma_out.ptr, out_bytes, 0x7fc40004u);
+    };
+    auto poison_production = [&]() {
+        return poison_outputs() &&
+               poison_output(replay_out.ptr, out_bytes, 0x7fc50005u);
+    };
+    if (!poison_production()) return false;
+
+    uint32_t production_candidate_set = 0u;
+
+    const arm baseline = {
+        "tile8_ab", []() { select_tile8(false); },
+        [&](uint32_t set) {
+            return ds4_gpu_attention_output_q4_K_batch_tensor(
+                       tile8_out.ptr, tile8_low.ptr, nullptr, nullptr,
+                       model.data, model.size,
+                       model.weights[set].output_a_offset,
+                       model.weights[set].output_b_offset, kQ4Type,
+                       kDenseK, kOutputRank, kOutputGroups, kOutputM,
+                       heads.ptr, n_tokens) > 0;
+        }};
+    const arm candidate = {
+        "wmma_a_k64p80_b_tile8", select_wmma_attention_a_tile8_b,
+        [&](uint32_t set) {
+            production_candidate_set = set;
+            return ds4_gpu_attention_output_q4_K_batch_tensor(
+                       wmma_out.ptr, wmma_low.ptr, nullptr, nullptr,
+                       model.data, model.size,
+                       model.weights[set].output_a_offset,
+                       model.weights[set].output_b_offset, kQ4Type,
+                       kDenseK, kOutputRank, kOutputGroups, kOutputM,
+                       heads.ptr, n_tokens) > 0;
+        }};
+    if (!benchmark_arms(
+        "attention_output_a_wmma_b_tile8", n_tokens, kOutputLowDim,
+        kDenseK + kOutputM, cfg, baseline, candidate, poison_production,
+        [&]() {
+            const uint64_t wmma_calls =
+                ds4_rocm_test_q4_prefill_wmma_get_calls();
+            const uint64_t k64_calls =
+                ds4_rocm_test_q4_prefill_wmma_k64_get_calls();
+            bool oracle_ok = wmma_calls == 1u && k64_calls == 1u;
+            std::fprintf(stderr,
+                         "output production dispatch: WMMA=%llu/1 "
+                         "K64=%llu/1 %s\n",
+                         (unsigned long long)wmma_calls,
+                         (unsigned long long)k64_calls,
+                         oracle_ok ? "PASS" : "FAIL");
+            oracle_ok = numerically_close(
+                wmma_low.ptr, tile8_low.ptr, low_bytes,
+                "output_a grouped WMMA shape vs TILE8") && oracle_ok;
+
+            /* A changes the values presented to B, so comparing the composed
+             * candidate against an all-TILE8 run conflates A's deliberate F16
+             * boundary with B correctness.  Keep the delta visible, but do
+             * not make it the production-path gate. */
+            oracle_ok = numerically_close(
+                wmma_out.ptr, tile8_out.ptr, out_bytes,
+                "output A-WMMA/B-TILE8 vs all-TILE8 (diagnostic only)",
+                16.0f, 8.0e-2f, false) && oracle_ok;
+
+            /* Hard B oracle: replay exact TILE8 with the very same WMMA-low
+             * intermediate consumed by the production candidate. */
+            select_tile8(false);
+            const bool replay_ok =
+                ds4_gpu_matmul_quant_tensor(
+                    replay_out.ptr, model.data, model.size,
+                    model.weights[production_candidate_set].output_b_offset,
+                    kQ4Type, kOutputLowDim, kOutputM, wmma_low.ptr,
+                    n_tokens) != 0 &&
+                ds4_gpu_synchronize();
+            if (!replay_ok) {
+                std::fprintf(stderr,
+                             "output_b same-low TILE8 replay dispatch FAIL\n");
+                oracle_ok = false;
+            } else {
+                const uint64_t replay_wmma_calls =
+                    ds4_rocm_test_q4_prefill_wmma_get_calls();
+                const uint64_t replay_k64_calls =
+                    ds4_rocm_test_q4_prefill_wmma_k64_get_calls();
+                if (replay_wmma_calls != wmma_calls ||
+                    replay_k64_calls != k64_calls) {
+                    std::fprintf(stderr,
+                                 "output_b same-low TILE8 replay used WMMA: "
+                                 "WMMA=%llu/%llu K64=%llu/%llu FAIL\n",
+                                 (unsigned long long)replay_wmma_calls,
+                                 (unsigned long long)wmma_calls,
+                                 (unsigned long long)replay_k64_calls,
+                                 (unsigned long long)k64_calls);
+                    oracle_ok = false;
+                }
+                oracle_ok = bitwise_equal(
+                    wmma_out.ptr, replay_out.ptr, out_bytes,
+                    "output_b production TILE8 vs same-low TILE8 replay") &&
+                    oracle_ok;
+            }
+            oracle_ok = check_guard(
+                tile8_low.ptr, low_bytes, "output_a TILE8 oracle") &&
+                oracle_ok;
+            oracle_ok = check_guard(
+                wmma_low.ptr, low_bytes, "output_a WMMA shape oracle") &&
+                oracle_ok;
+            oracle_ok = check_guard(
+                tile8_out.ptr, out_bytes, "output_b all-TILE8 oracle") &&
+                oracle_ok;
+            oracle_ok = check_guard(
+                wmma_out.ptr, out_bytes,
+                "output_b production A-WMMA/B-TILE8 oracle") && oracle_ok;
+            oracle_ok = check_guard(
+                replay_out.ptr, out_bytes,
+                "output_b same-low TILE8 replay oracle") && oracle_ok;
+            return oracle_ok;
+        })) return false;
+
+    void *const heads_device = ds4_gpu_tensor_contents(heads.ptr);
+    void *const rows64_low_device = ds4_gpu_tensor_contents(tile8_low.ptr);
+    void *const rows64_out_device = ds4_gpu_tensor_contents(tile8_out.ptr);
+    void *const shape_low_device = ds4_gpu_tensor_contents(wmma_low.ptr);
+    void *const shape_out_device = ds4_gpu_tensor_contents(wmma_out.ptr);
+    std::vector<const void *> output_a_weights;
+    std::vector<const void *> output_b_weights;
+    if (!heads_device || !rows64_low_device || !rows64_out_device ||
+        !shape_low_device || !shape_out_device ||
+        !resolve_resident_weights(
+            model, &weight_set::output_a_offset,
+            q4_weight_bytes(kDenseK, kOutputLowDim), &output_a_weights) ||
+        !resolve_resident_weights(
+            model, &weight_set::output_b_offset,
+            q4_weight_bytes(kOutputLowDim, kOutputM), &output_b_weights)) {
+        std::fprintf(stderr,
+                     "attention_output_ab_wmma_rows N=%u: direct setup failed\n",
+                     n_tokens);
+        return false;
+    }
+    const uint64_t row_a_bytes = q4_weight_bytes(kDenseK, 1u);
+    const uint64_t row_b_bytes = q4_weight_bytes(kOutputLowDim, 1u);
+    const arm geometry_baseline = {
+        "wmma_rows64_ab_scalar", []() {},
+        [&](uint32_t set) {
+            return ds4_rocm_bench_q4_K_wmma_enqueue(
+                       rows64_low_device, output_a_weights[set], heads_device,
+                       n_tokens, kOutputGroups, kDenseK, kOutputRank,
+                       row_a_bytes, kOutputGroups * kDenseK, kDenseK,
+                       kOutputLowDim, 64u, 0) != 0 &&
+                   ds4_rocm_bench_q4_K_wmma_enqueue(
+                       rows64_out_device, output_b_weights[set],
+                       rows64_low_device, n_tokens, 1u, kOutputLowDim,
+                       kOutputM, row_b_bytes, kOutputLowDim, 0u, kOutputM,
+                       64u, 0) != 0;
+        }};
+    const arm geometry_candidate = {
+        "wmma_shape_ab_scalar", []() {},
+        [&](uint32_t set) {
+            return ds4_rocm_bench_q4_K_wmma_enqueue(
+                       shape_low_device, output_a_weights[set], heads_device,
+                       n_tokens, kOutputGroups, kDenseK, kOutputRank,
+                       row_a_bytes, kOutputGroups * kDenseK, kDenseK,
+                       kOutputLowDim,
+                       shape_wmma_row_tile(kOutputRank), 0) != 0 &&
+                   ds4_rocm_bench_q4_K_wmma_enqueue(
+                       shape_out_device, output_b_weights[set],
+                       shape_low_device, n_tokens, 1u, kOutputLowDim,
+                       kOutputM, row_b_bytes, kOutputLowDim, 0u, kOutputM,
+                       shape_wmma_row_tile(kOutputM), 0) != 0;
+        }};
+    if (!benchmark_arms(
+        "attention_output_ab_wmma_rows", n_tokens, kOutputLowDim,
+        kDenseK + kOutputM, cfg, geometry_baseline, geometry_candidate,
+        poison_outputs,
+        [&]() {
+            return bitwise_equal(tile8_low.ptr, wmma_low.ptr, low_bytes,
+                                 "output_a WMMA rows64 vs shape") &&
+                   bitwise_equal(tile8_out.ptr, wmma_out.ptr, out_bytes,
+                                 "output_a+b WMMA rows64 vs shape") &&
+                   check_guard(tile8_low.ptr, low_bytes,
+                               "output_a WMMA rows64 oracle") &&
+                   check_guard(wmma_low.ptr, low_bytes,
+                               "output_a WMMA shape oracle") &&
+                   check_guard(tile8_out.ptr, out_bytes,
+                               "output_b WMMA rows64 oracle") &&
+                   check_guard(wmma_out.ptr, out_bytes,
+                               "output_b WMMA shape oracle");
+        })) return false;
+
+    const uint32_t fixed_row_a_tile = shape_wmma_row_tile(kOutputRank);
+    const uint32_t fixed_row_b_tile = shape_wmma_row_tile(kOutputM);
+    const arm k32_baseline = {
+        "wmma_k32_ab_rows128_load2", []() {},
+        [&](uint32_t set) {
+            return ds4_rocm_bench_q4_K_wmma_variant_enqueue(
+                       rows64_low_device, output_a_weights[set], heads_device,
+                       n_tokens, kOutputGroups, kDenseK, kOutputRank,
+                       row_a_bytes, kOutputGroups * kDenseK, kDenseK,
+                       kOutputLowDim, fixed_row_a_tile, 32u, 1) != 0 &&
+                   ds4_rocm_bench_q4_K_wmma_variant_enqueue(
+                       rows64_out_device, output_b_weights[set],
+                       rows64_low_device, n_tokens, 1u, kOutputLowDim,
+                       kOutputM, row_b_bytes, kOutputLowDim, 0u, kOutputM,
+                       fixed_row_b_tile, 32u, 1) != 0;
+        }};
+    const arm k64_candidate = {
+        "wmma_k64p80_ab_rows128_load2", []() {},
+        [&](uint32_t set) {
+            return ds4_rocm_bench_q4_K_wmma_variant_enqueue(
+                       shape_low_device, output_a_weights[set], heads_device,
+                       n_tokens, kOutputGroups, kDenseK, kOutputRank,
+                       row_a_bytes, kOutputGroups * kDenseK, kDenseK,
+                       kOutputLowDim, fixed_row_a_tile, 64u, 1) != 0 &&
+                   ds4_rocm_bench_q4_K_wmma_variant_enqueue(
+                       shape_out_device, output_b_weights[set],
+                       shape_low_device, n_tokens, 1u, kOutputLowDim,
+                       kOutputM, row_b_bytes, kOutputLowDim, 0u, kOutputM,
+                       fixed_row_b_tile, 64u, 1) != 0;
+        }};
+    return benchmark_arms(
+        "attention_output_ab_wmma_k32_k64", n_tokens, kOutputLowDim,
+        kDenseK + kOutputM, cfg, k32_baseline, k64_candidate, poison_outputs,
+        [&]() {
+            return bitwise_equal(tile8_low.ptr, wmma_low.ptr, low_bytes,
+                                 "output_a WMMA K32 vs K64/P80") &&
+                   bitwise_equal(tile8_out.ptr, wmma_out.ptr, out_bytes,
+                                 "output_a+b WMMA K32 vs K64/P80") &&
+                   check_guard(tile8_low.ptr, low_bytes,
+                               "output_a WMMA K32 oracle") &&
+                   check_guard(wmma_low.ptr, low_bytes,
+                               "output_a WMMA K64/P80 oracle") &&
+                   check_guard(tile8_out.ptr, out_bytes,
+                               "output_b WMMA K32 oracle") &&
+                   check_guard(wmma_out.ptr, out_bytes,
+                               "output_b WMMA K64/P80 oracle");
+        });
+}
+
+void usage(FILE *stream, const char *argv0) {
+    std::fprintf(
+        stream,
+        "usage: %s [options]\n\n"
+        "Resident ROCm Q4_K prefill kernel A/B (HIP event timing only).\n\n"
+        "  --case all|dense|pair|qb|outb|output\n"
+        "                              comparison to run (default: all)\n"
+        "  --tokens N[,N...]          token counts, each 9..4096\n"
+        "  --full                     use 9,16,17,31,32,33,128,256,257,512,4096\n"
+        "  --sets N                   rotating resident weight sets (default: %u)\n"
+        "  --samples N                samples/arm, multiple of 4 (default: %u)\n"
+        "  --warmup N                 untimed dispatches/arm (default: %u)\n"
+        "  -h, --help                 show this help\n\n"
+        "dense compares legacy/TILE8, times the raw canonical/wave32 Q8_K\n"
+        "quantizer kernels on gfx1151, and TILE8/direct WMMA there for N>=256\n"
+        "at K=4096,M=1024. pair compares\n"
+        "two TILE8 projections with\n"
+        "the fused K=4096,M=(1024+512) path. qb adds TILE4/direct WMMA at the\n"
+        "K=1024,M=32768 shape. outb isolates K=8192,M=4096; output measures\n"
+        "the production grouped output_a plus output_b API as all-TILE8 vs\n"
+        "A-WMMA/B-TILE8. Its hard B oracle replays TILE8 on the same low; the\n"
+        "composed all-TILE8 delta is diagnostic only. Other WMMA comparisons\n"
+        "use a finite/toleranced oracle because their F16 boundary is not\n"
+        "bit-identical to the Q8_K activation path. At N>=256 the raw direct\n"
+        "arms also compare K32 with K64/P80 at fixed production geometry and\n"
+        "load2; q_b additionally compares K64/P80 with the default K128/P144\n"
+        "load4 stage. Both staged-kernel checks require bitwise output.\n",
+        argv0, kDefaultSets, kDefaultSamples, kDefaultWarmup);
+}
+
+uint32_t parse_u32(const char *text, const char *option, uint32_t minimum,
+                   uint32_t maximum) {
+    char *end = nullptr;
+    errno = 0;
+    const unsigned long value = std::strtoul(text, &end, 10);
+    if (errno != 0 || !text[0] || !end || *end || value < minimum ||
+        value > maximum) {
+        std::fprintf(stderr, "invalid %s: %s\n", option, text);
+        std::exit(2);
+    }
+    return static_cast<uint32_t>(value);
+}
+
+const char *need_value(int *index, int argc, char **argv) {
+    if (*index + 1 >= argc) {
+        std::fprintf(stderr, "%s requires a value\n", argv[*index]);
+        std::exit(2);
+    }
+    return argv[++*index];
+}
+
+std::vector<uint32_t> parse_tokens(const char *text) {
+    std::vector<uint32_t> result;
+    const char *cursor = text;
+    while (*cursor) {
+        const char *comma = std::strchr(cursor, ',');
+        const std::string item(cursor,
+                               comma ? static_cast<size_t>(comma - cursor)
+                                     : std::strlen(cursor));
+        result.push_back(parse_u32(item.c_str(), "--tokens", 9u, 4096u));
+        if (!comma) break;
+        cursor = comma + 1;
+        if (!*cursor) {
+            std::fprintf(stderr, "invalid --tokens: trailing comma\n");
+            std::exit(2);
+        }
+    }
+    if (result.empty()) {
+        std::fprintf(stderr, "--tokens cannot be empty\n");
+        std::exit(2);
+    }
+    std::sort(result.begin(), result.end());
+    result.erase(std::unique(result.begin(), result.end()), result.end());
+    return result;
+}
+
+config parse_options(int argc, char **argv) {
+    config cfg;
+    for (int i = 1; i < argc; i++) {
+        if (!std::strcmp(argv[i], "-h") || !std::strcmp(argv[i], "--help")) {
+            usage(stdout, argv[0]);
+            std::exit(0);
+        } else if (!std::strcmp(argv[i], "--case")) {
+            const char *value = need_value(&i, argc, argv);
+            if (!std::strcmp(value, "all")) cfg.selected = bench_case::all;
+            else if (!std::strcmp(value, "dense")) cfg.selected = bench_case::dense;
+            else if (!std::strcmp(value, "pair")) cfg.selected = bench_case::pair;
+            else if (!std::strcmp(value, "qb")) cfg.selected = bench_case::qb;
+            else if (!std::strcmp(value, "outb")) cfg.selected = bench_case::outb;
+            else if (!std::strcmp(value, "output")) cfg.selected = bench_case::output;
+            else {
+                std::fprintf(stderr, "invalid --case: %s\n", value);
+                std::exit(2);
+            }
+        } else if (!std::strcmp(argv[i], "--tokens")) {
+            cfg.tokens = parse_tokens(need_value(&i, argc, argv));
+        } else if (!std::strcmp(argv[i], "--full")) {
+            cfg.tokens = {
+                9u, 16u, 17u, 31u, 32u, 33u, 128u, 256u, 257u, 512u,
+                4096u};
+        } else if (!std::strcmp(argv[i], "--sets")) {
+            cfg.sets = parse_u32(need_value(&i, argc, argv), "--sets", 1u, 32u);
+        } else if (!std::strcmp(argv[i], "--samples")) {
+            cfg.samples =
+                parse_u32(need_value(&i, argc, argv), "--samples", 4u, 1000u);
+        } else if (!std::strcmp(argv[i], "--warmup")) {
+            cfg.warmup =
+                parse_u32(need_value(&i, argc, argv), "--warmup", 0u, 100u);
+        } else {
+            std::fprintf(stderr, "unknown option: %s\n", argv[i]);
+            usage(stderr, argv[0]);
+            std::exit(2);
+        }
+    }
+    if ((cfg.samples % 4u) != 0u) {
+        std::fprintf(stderr,
+                     "--samples must be a multiple of 4 for ABBA/BAAB balance\n");
+        std::exit(2);
+    }
+    return cfg;
+}
+
+bool includes(bench_case selected, bench_case wanted) {
+    return selected == bench_case::all || selected == wanted;
+}
+
+}  // namespace
+
+int main(int argc, char **argv) {
+    config cfg = parse_options(argc, argv);
+    env_snapshot enable_guard(kPrefillEnable);
+    env_snapshot disable_guard(kPrefillDisable);
+    env_snapshot require_guard(kPrefillRequire);
+    env_snapshot tile4_guard(kK1024Tile4Disable);
+    env_snapshot tile4_ssd_guard(kK1024Tile4SsdEnable);
+    env_snapshot tile4_require_guard(kK1024Tile4Require);
+    env_snapshot wmma_enable_guard(kWmmaEnable);
+    env_snapshot wmma_ssd_enable_guard(kWmmaSsdEnable);
+    env_snapshot wmma_disable_guard(kWmmaDisable);
+    env_snapshot wmma_require_guard(kWmmaRequire);
+    env_snapshot wmma_row_tile_guard(kWmmaRowTile);
+    env_snapshot wmma_k64_guard(kWmmaK64);
+    env_snapshot wmma_k128_disable_guard(kWmmaK128Disable);
+    env_snapshot q8_wave32_enable_guard(kQ8Wave32Enable);
+    env_snapshot q8_wave32_disable_guard(kQ8Wave32Disable);
+    env_snapshot q8_wave32_require_guard(kQ8Wave32Require);
+    (void)unsetenv(kQ8Wave32Enable);
+    (void)unsetenv(kQ8Wave32Disable);
+    (void)unsetenv(kQ8Wave32Require);
+    (void)unsetenv(kWmmaK128Disable);
+
+    int device_count = 0;
+    hipError_t hip_rc = hipGetDeviceCount(&device_count);
+    if (hip_rc != hipSuccess || device_count <= 0) {
+        std::fprintf(stderr,
+                     "rocm-q4-prefill-bench: no visible HIP device (%s)\n",
+                     hip_rc == hipSuccess ? "device count is zero"
+                                          : hipGetErrorString(hip_rc));
+        return 77;
+    }
+    hipDeviceProp_t properties{};
+    if (hipGetDeviceProperties(&properties, 0) != hipSuccess) {
+        std::fprintf(stderr,
+                     "rocm-q4-prefill-bench: cannot query device properties\n");
+        return 1;
+    }
+    cfg.wmma_supported = properties.warpSize == 32 &&
+        std::strncmp(properties.gcnArchName, "gfx1151", 7u) == 0;
+    const bool wmma_only_case =
+        cfg.selected == bench_case::outb || cfg.selected == bench_case::output;
+    const bool has_wmma_tokens = std::any_of(
+        cfg.tokens.begin(), cfg.tokens.end(),
+        [](uint32_t n_tokens) { return n_tokens >= 256u; });
+    if (wmma_only_case && (!cfg.wmma_supported || !has_wmma_tokens)) {
+        std::fprintf(stderr,
+                     "rocm-q4-prefill-bench: SKIP (%s requires gfx1151 "
+                     "wave32 and at least one N>=256 sample)\n",
+                     cfg.selected == bench_case::outb ? "outb" : "output");
+        return 77;
+    }
+    if (!ds4_gpu_init()) {
+        std::fprintf(stderr, "rocm-q4-prefill-bench: ds4_gpu_init failed\n");
+        return 1;
+    }
+
+    bool ok = true;
+    model_fixture model;
+    if (!make_model(&model, cfg.sets)) {
+        std::fprintf(stderr, "rocm-q4-prefill-bench: model fixture allocation failed\n");
+        ok = false;
+    }
+    if (ok) {
+        ds4_gpu_set_ssd_streaming(false);
+        const uint64_t max_tensor = std::max(
+            q4_weight_bytes(kOutputLowDim, kOutputM),
+            q4_weight_bytes(kDenseK, kOutputLowDim));
+        if (!ds4_gpu_set_model_fd(fileno(model.file)) ||
+            !ds4_gpu_set_model_map_spans(
+                model.data, model.size, model.span_offsets.data(),
+                model.span_sizes.data(),
+                static_cast<uint32_t>(model.span_offsets.size()), max_tensor) ||
+            !ds4_gpu_synchronize()) {
+            std::fprintf(stderr,
+                         "rocm-q4-prefill-bench: device-resident weight copy failed\n");
+            ok = false;
+        }
+    }
+
+    if (ok) {
+        std::printf(
+            "DS4_ROCM_Q4_PREFILL_SETUP device=%s arch=%s warp=%d sets=%u "
+            "resident_mib=%.2f timing=hip_events ssd_streaming=off "
+            "wmma_rowtiles=%s wmma_loaders=%s wmma_k_stages=%s "
+            "wmma_k_default=%s q8_wave32=%s\n",
+            properties.name, properties.gcnArchName, properties.warpSize,
+            cfg.sets, static_cast<double>(model.resident_bytes) / 1048576.0,
+            cfg.wmma_supported ? "64,128,256" : "skipped",
+            cfg.wmma_supported ? "scalar,load2,load4" : "skipped",
+            cfg.wmma_supported ? "32,64p80,128p144" : "skipped",
+            cfg.wmma_supported ? "128p144@rows256,64p80@rows64/128"
+                               : "skipped",
+            cfg.wmma_supported ? "available" : "skipped");
+        std::fflush(stdout);
+        for (uint32_t n_tokens : cfg.tokens) {
+            if (includes(cfg.selected, bench_case::dense)) {
+                ok = run_dense(model, cfg, n_tokens) && ok;
+            }
+            if (ok && includes(cfg.selected, bench_case::pair)) {
+                ok = run_pair(model, cfg, n_tokens) && ok;
+            }
+            if (ok && includes(cfg.selected, bench_case::qb)) {
+                ok = run_qb(model, cfg, n_tokens) && ok;
+            }
+            if (ok && includes(cfg.selected, bench_case::outb)) {
+                ok = run_output_b(model, cfg, n_tokens) && ok;
+            }
+            if (ok && includes(cfg.selected, bench_case::output)) {
+                ok = run_attention_output(model, cfg, n_tokens) && ok;
+            }
+            if (!ok) break;
+        }
+    }
+
+    (void)ds4_gpu_set_model_fd(-1);
+    ds4_gpu_cleanup();
+    std::fprintf(stderr, "rocm-q4-prefill-bench: %s\n", ok ? "PASS" : "FAIL");
+    return ok ? 0 : 1;
+}
diff --git a/tests/cuda_q4_gb10_fast_matrix.sh b/tests/cuda_q4_gb10_fast_matrix.sh
new file mode 100755
index 0000000000..41458dcb0d
--- /dev/null
+++ b/tests/cuda_q4_gb10_fast_matrix.sh
@@ -0,0 +1,480 @@
+#!/bin/sh
+# Isolate the numerical effect of each AProjQ4 GB10 fast-path component.
+#
+# Usage:
+#   tests/cuda_q4_gb10_fast_matrix.sh MODEL MANIFEST [OUTPUT_DIR]
+#
+# Every non-oracle arm is run through score_official.  The script deliberately
+# uses separate processes:
+# these switches are cached during CUDA/MMQ initialization and cannot be
+# compared safely in one process.
+
+set -eu
+
+usage() {
+    cat >&2 <<'EOF'
+usage: tests/cuda_q4_gb10_fast_matrix.sh MODEL MANIFEST [OUTPUT_DIR]
+
+Environment:
+  DS4_BIN                         ds4 executable (default: ./ds4)
+  DS4_CUDA_Q4_MATRIX_SCORER       score_official executable
+  DS4_CUDA_Q4_MATRIX_CTX          context size (default: 4096)
+  DS4_CUDA_Q4_MATRIX_TOKENS       smoke continuation length (default: 32)
+  DS4_CUDA_Q4_MATRIX_TOP_K        smoke top-logprobs (default: 128)
+  DS4_CUDA_Q4_MATRIX_PROMPT       deterministic smoke prompt
+  DS4_CUDA_Q4_MATRIX_SSD_STREAMING  0 or 1 (default: 0)
+  DS4_CUDA_Q4_MATRIX_SSD_CACHE    expert count or NGB; required with streaming
+  DS4_CUDA_Q4_MATRIX_SSD_PRELOAD  optional expert preload count
+  DS4_CUDA_Q4_MATRIX_DECODE_GRAPHS default, 0, or 1
+  DS4_CUDA_Q4_MATRIX_SKIP_PARITY  0 or 1 (default: 0; skip is incomplete QA)
+
+The output directory must not already contain matrix results.  Exit status is
+zero only when tensor-oracle coverage passes, scorer rows match, and every
+recorded smoke result is byte-identical to the umbrella rollback.  A numerical
+difference is preserved in *.diff and causes a nonzero exit after all arms
+have completed.
+EOF
+}
+
+if [ "$#" -lt 2 ] || [ "$#" -gt 3 ]; then
+    usage
+    exit 2
+fi
+
+MODEL=$1
+MANIFEST=$2
+OUT_DIR=${3:-}
+DS4_BIN=${DS4_BIN:-./ds4}
+SCORER=${DS4_CUDA_Q4_MATRIX_SCORER:-gguf-tools/quality-testing/score_official}
+CTX=${DS4_CUDA_Q4_MATRIX_CTX:-4096}
+TOKENS=${DS4_CUDA_Q4_MATRIX_TOKENS:-32}
+TOP_K=${DS4_CUDA_Q4_MATRIX_TOP_K:-128}
+PROMPT=${DS4_CUDA_Q4_MATRIX_PROMPT:-Write a complete Python quicksort function with comments.}
+SSD_STREAMING=${DS4_CUDA_Q4_MATRIX_SSD_STREAMING:-0}
+SSD_CACHE=${DS4_CUDA_Q4_MATRIX_SSD_CACHE:-}
+SSD_PRELOAD=${DS4_CUDA_Q4_MATRIX_SSD_PRELOAD:-}
+DECODE_GRAPHS=${DS4_CUDA_Q4_MATRIX_DECODE_GRAPHS:-default}
+SKIP_PARITY=${DS4_CUDA_Q4_MATRIX_SKIP_PARITY:-0}
+
+case "$CTX:$TOKENS:$TOP_K" in
+    *[!0-9:]*|:*|*::*|*:|0:*|*:0:*|*:0)
+        echo "q4-gb10-matrix: ctx, tokens, and top-k must be positive integers" >&2
+        exit 2
+        ;;
+esac
+if [ "$TOP_K" -gt 128 ]; then
+    echo "q4-gb10-matrix: top-k cannot exceed the ds4 dump limit (128)" >&2
+    exit 2
+fi
+case "$SSD_STREAMING" in
+    0|1) ;;
+    *) echo "q4-gb10-matrix: SSD_STREAMING must be 0 or 1" >&2; exit 2 ;;
+esac
+case "$SKIP_PARITY" in
+    0|1) ;;
+    *) echo "q4-gb10-matrix: SKIP_PARITY must be 0 or 1" >&2; exit 2 ;;
+esac
+case "$DECODE_GRAPHS" in
+    default|0|1) ;;
+    *) echo "q4-gb10-matrix: DECODE_GRAPHS must be default, 0, or 1" >&2; exit 2 ;;
+esac
+if [ "$SSD_STREAMING" = 1 ] && [ -z "$SSD_CACHE" ]; then
+    echo "q4-gb10-matrix: set DS4_CUDA_Q4_MATRIX_SSD_CACHE when streaming" >&2
+    exit 2
+fi
+if [ "$SSD_STREAMING" = 0 ] && { [ -n "$SSD_CACHE" ] || [ -n "$SSD_PRELOAD" ]; }; then
+    echo "q4-gb10-matrix: SSD cache/preload requires SSD_STREAMING=1" >&2
+    exit 2
+fi
+if [ ! -x "$DS4_BIN" ]; then
+    echo "q4-gb10-matrix: ds4 executable not found: $DS4_BIN" >&2
+    exit 2
+fi
+if [ ! -r "$MODEL" ]; then
+    echo "q4-gb10-matrix: model is not readable: $MODEL" >&2
+    exit 2
+fi
+if [ ! -r "$MANIFEST" ]; then
+    echo "q4-gb10-matrix: manifest is not readable: $MANIFEST" >&2
+    exit 2
+fi
+if [ ! -x "$SCORER" ]; then
+    echo "q4-gb10-matrix: scorer not found: $SCORER" >&2
+    echo "build it with: make gguf-tools/quality-testing/score_official" >&2
+    exit 2
+fi
+
+if [ -z "$OUT_DIR" ]; then
+    OUT_DIR=$(mktemp -d "${TMPDIR:-/tmp}/ds4-q4-gb10-matrix.XXXXXX")
+else
+    mkdir -p "$OUT_DIR"
+fi
+if [ -e "$OUT_DIR/umbrella_control.log" ] ||
+   [ -e "$OUT_DIR/umbrella_control.json" ] ||
+   [ -e "$OUT_DIR/umbrella_control.tsv" ]; then
+    echo "q4-gb10-matrix: output directory already contains matrix results: $OUT_DIR" >&2
+    exit 2
+fi
+
+case "$DECODE_GRAPHS" in
+    default) GRAPH_ENV=; GRAPH_LOG_ENV= ;;
+    0) GRAPH_ENV=DS4_CUDA_DECODE_GRAPHS=0; GRAPH_LOG_ENV= ;;
+    1) GRAPH_ENV=DS4_CUDA_DECODE_GRAPHS=1; GRAPH_LOG_ENV=DS4_CUDA_DECODE_GRAPH_LOG=1 ;;
+esac
+
+# Strip every selector that could leak from the caller and turn an apparently
+# isolated arm into a compound experiment.  Arm-specific assignments follow
+# these -u options.
+clean_env() {
+    env \
+        -u DS4_CUDA_MMQ \
+        -u DS4_CUDA_DISABLE_Q4_DENSE_PAIR \
+        -u DS4_CUDA_NO_Q4_GB10_FAST \
+        -u DS4_CUDA_NO_Q4_DENSE_SCRATCH \
+        -u DS4_CUDA_NO_Q4_GROUPED_ATTN_A \
+        -u DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH \
+        -u DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_BATCH \
+        -u DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_BATCH \
+        -u DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_PREFILL \
+        -u DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL \
+        -u DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_PREFILL \
+        -u DS4_CUDA_ENABLE_Q4_GROUPED_ATTN_A_SINGLE_GRID \
+        -u DS4_CUDA_DISABLE_Q4_GROUPED_ATTN_A_SINGLE_GRID \
+        -u DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_SINGLE_GRID \
+        -u DS4_CUDA_NO_Q4_GROUPED_ATTN_A_Q81 \
+        -u DS4_CUDA_REQUIRE_Q4_GROUPED_ATTN_A_Q81 \
+        -u DS4_CUDA_Q4_GROUPED_ATTN_A_ORACLE \
+        -u DS4_CUDA_DISABLE_Q4_ATTN_OUT_HC_FUSE \
+        -u DS4_CUDA_ENABLE_Q4_ATTN_OUT_HC_FUSE \
+        -u DS4_CUDA_Q4_ATTN_OUT_HC_Q8K_EXPERIMENT \
+        -u DS4_CUDA_Q4_ATTN_OUT_HC_ORACLE \
+        -u DS4_CUDA_NO_Q4_K1024_PERSISTENT \
+        -u DS4_CUDA_ENABLE_Q4_K1024_PERSISTENT \
+        -u DS4_CUDA_REQUIRE_Q4_K1024_PERSISTENT \
+        -u DS4_CUDA_Q4_MMQ_16WARP \
+        -u DS4_CUDA_NO_Q4_MMQ_16WARP \
+        -u DS4_CUDA_REQUIRE_Q4_MMQ_16WARP \
+        -u DS4_CUDA_DECODE_GRAPHS \
+        -u DS4_CUDA_DECODE_GRAPH_LOG \
+        "$@"
+}
+
+check_gb10_log() {
+    log=$1
+    cuda_count=$(grep -c 'ds4: CUDA backend initialized on ' "$log" || true)
+    if [ "$cuda_count" -ne 1 ] ||
+       ! grep -Eq 'ds4: CUDA backend initialized on .*\(sm_121\) dev=' "$log"; then
+        echo "q4-gb10-matrix: $log did not initialize exactly one sm_121 CUDA GPU" >&2
+        return 1
+    fi
+}
+
+check_graph_log() {
+    arm=$1
+    log=$2
+    case "$arm" in
+        grouped_oracle|hc_oracle)
+            if ! grep -q 'decode graph capture disabled for Q4 attention oracle' "$log"; then
+                echo "q4-gb10-matrix: $arm did not prove oracle graph exclusion" >&2
+                return 1
+            fi
+            ;;
+        *)
+            case "$DECODE_GRAPHS" in
+                0)
+                    if ! grep -q 'decode graph capture disabled' "$log"; then
+                        echo "q4-gb10-matrix: $arm did not prove graphs-off dispatch" >&2
+                        return 1
+                    fi
+                    ;;
+                1)
+                    if ! grep -q 'ds4: decode graph captured ' "$log"; then
+                        echo "q4-gb10-matrix: $arm had zero decode-graph captures" >&2
+                        return 1
+                    fi
+                    if grep -Eq 'decode graph (capture|instantiate|first launch|replay) failed' "$log"; then
+                        echo "q4-gb10-matrix: $arm reported a decode-graph failure" >&2
+                        return 1
+                    fi
+                    ;;
+            esac
+            ;;
+    esac
+}
+
+run_smoke() {
+    arm=$1
+    shift
+    json="$OUT_DIR/$arm.json"
+    stdout="$OUT_DIR/$arm.stdout"
+    log="$OUT_DIR/$arm.log"
+    echo "q4-gb10-matrix: smoke arm=$arm"
+
+    if [ "$SSD_STREAMING" = 1 ]; then
+        if [ -n "$SSD_PRELOAD" ]; then
+            clean_env ${GRAPH_ENV:+"$GRAPH_ENV"} ${GRAPH_LOG_ENV:+"$GRAPH_LOG_ENV"} \
+                "$@" "$DS4_BIN" \
+                --cuda -m "$MODEL" --ctx "$CTX" --tokens "$TOKENS" \
+                --nothink --temp 0 --dump-logprobs "$json" \
+                --logprobs-top-k "$TOP_K" --ssd-streaming \
+                --ssd-streaming-cache-experts "$SSD_CACHE" \
+                --ssd-streaming-preload-experts "$SSD_PRELOAD" \
+                -p "$PROMPT" >"$stdout" 2>"$log"
+        else
+            clean_env ${GRAPH_ENV:+"$GRAPH_ENV"} ${GRAPH_LOG_ENV:+"$GRAPH_LOG_ENV"} \
+                "$@" "$DS4_BIN" \
+                --cuda -m "$MODEL" --ctx "$CTX" --tokens "$TOKENS" \
+                --nothink --temp 0 --dump-logprobs "$json" \
+                --logprobs-top-k "$TOP_K" --ssd-streaming \
+                --ssd-streaming-cache-experts "$SSD_CACHE" \
+                -p "$PROMPT" >"$stdout" 2>"$log"
+        fi
+    else
+        clean_env ${GRAPH_ENV:+"$GRAPH_ENV"} ${GRAPH_LOG_ENV:+"$GRAPH_LOG_ENV"} \
+            "$@" "$DS4_BIN" \
+            --cuda -m "$MODEL" --ctx "$CTX" --tokens "$TOKENS" \
+            --nothink --temp 0 --dump-logprobs "$json" \
+            --logprobs-top-k "$TOP_K" -p "$PROMPT" \
+            >"$stdout" 2>"$log"
+    fi
+    check_gb10_log "$log"
+    check_graph_log "$arm" "$log"
+    if [ ! -s "$json" ]; then
+        echo "q4-gb10-matrix: arm $arm produced no logprob dump" >&2
+        return 1
+    fi
+}
+
+run_score() {
+    arm=$1
+    shift
+    tsv="$OUT_DIR/$arm.tsv"
+    log="$OUT_DIR/$arm.score.log"
+    echo "q4-gb10-matrix: quality arm=$arm"
+
+    if [ "$SSD_STREAMING" = 1 ]; then
+        if [ -n "$SSD_PRELOAD" ]; then
+            clean_env ${GRAPH_ENV:+"$GRAPH_ENV"} ${GRAPH_LOG_ENV:+"$GRAPH_LOG_ENV"} \
+                "$@" "$SCORER" \
+                "$MODEL" "$MANIFEST" "$tsv" "$CTX" --ssd-streaming \
+                --ssd-streaming-cache-experts "$SSD_CACHE" \
+                --ssd-streaming-preload-experts "$SSD_PRELOAD" \
+                >"$OUT_DIR/$arm.score.stdout" 2>"$log"
+        else
+            clean_env ${GRAPH_ENV:+"$GRAPH_ENV"} ${GRAPH_LOG_ENV:+"$GRAPH_LOG_ENV"} \
+                "$@" "$SCORER" \
+                "$MODEL" "$MANIFEST" "$tsv" "$CTX" --ssd-streaming \
+                --ssd-streaming-cache-experts "$SSD_CACHE" \
+                >"$OUT_DIR/$arm.score.stdout" 2>"$log"
+        fi
+    else
+        clean_env ${GRAPH_ENV:+"$GRAPH_ENV"} ${GRAPH_LOG_ENV:+"$GRAPH_LOG_ENV"} \
+            "$@" "$SCORER" \
+            "$MODEL" "$MANIFEST" "$tsv" "$CTX" \
+            >"$OUT_DIR/$arm.score.stdout" 2>"$log"
+    fi
+    check_gb10_log "$log"
+    check_graph_log "$arm" "$log"
+    if [ ! -s "$tsv" ]; then
+        echo "q4-gb10-matrix: quality arm $arm produced no TSV" >&2
+        return 1
+    fi
+}
+
+field() {
+    printf '%s\n' "$1" | tr ' ' '\n' | awk -F= -v key="$2" '
+        $1 == key { gsub(/[^0-9].*$/, "", $2); print $2; exit }
+    '
+}
+
+check_grouped_oracle() {
+    log=$1
+    line=$(grep 'ds4: CUDA Q4 grouped attention-A oracle:' "$log" | tail -n 1 || true)
+    calls=$(field "$line" calls)
+    mismatches=$(field "$line" mismatches)
+    skips=$(field "$line" skips)
+    if [ -z "$line" ] || [ -z "$calls" ] || [ "$calls" -le 0 ] ||
+       [ "$skips" != 0 ]; then
+        echo "q4-gb10-matrix: grouped oracle coverage failed: ${line:-missing summary}" >&2
+        return 1
+    fi
+    if [ "$mismatches" != 0 ]; then
+        echo "q4-gb10-matrix: grouped oracle found mismatches: $line" >&2
+        oracle_mismatch_arms="$oracle_mismatch_arms grouped"
+    fi
+}
+
+check_hc_oracle() {
+    log=$1
+    line=$(grep 'ds4: CUDA Q4 attention-output/HC oracle:' "$log" | tail -n 1 || true)
+    calls=$(field "$line" calls)
+    mismatches=$(field "$line" epilogue_mismatches)
+    skips=$(field "$line" skips)
+    if [ -z "$line" ] || [ -z "$calls" ] || [ "$calls" -le 0 ] ||
+       [ "$skips" != 0 ]; then
+        echo "q4-gb10-matrix: HC epilogue oracle coverage failed: ${line:-missing summary}" >&2
+        return 1
+    fi
+    if [ "$mismatches" != 0 ]; then
+        echo "q4-gb10-matrix: HC epilogue oracle found mismatches: $line" >&2
+        oracle_mismatch_arms="$oracle_mismatch_arms hc_epilogue"
+    fi
+}
+
+LOCAL_ROLLBACK="DS4_CUDA_NO_Q4_DENSE_SCRATCH=1
+DS4_CUDA_NO_Q4_GROUPED_ATTN_A=1
+DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH=1
+DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL=1
+DS4_CUDA_NO_Q4_GROUPED_ATTN_A_Q81=1
+DS4_CUDA_DISABLE_Q4_ATTN_OUT_HC_FUSE=1
+DS4_CUDA_NO_Q4_K1024_PERSISTENT=1"
+
+# Do not stop at the first numerical difference: completing all arms is what
+# identifies a single culprit versus an interaction.  Execution/coverage
+# failures still stop immediately because subsequent comparisons would lie.
+if [ "$SKIP_PARITY" = 0 ]; then
+    echo "q4-gb10-matrix: synthetic MMQ parity"
+    clean_env make test-mmq-parity-cuda CUDA_ARCH=sm_121 \
+        >"$OUT_DIR/mmq-parity.log" 2>&1
+else
+    echo "q4-gb10-matrix: WARNING synthetic parity skipped (result is incomplete)" >&2
+fi
+
+# `set -- $LOCAL_ROLLBACK` intentionally splits the newline-delimited list
+# into environment assignments. Values and names contain no shell metacharacters.
+set -- $LOCAL_ROLLBACK
+run_smoke umbrella_control DS4_CUDA_NO_Q4_GB10_FAST=1
+run_smoke local_control "$@"
+run_smoke scratch_only \
+    DS4_CUDA_NO_Q4_GROUPED_ATTN_A=1 \
+    DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH=1 \
+    DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL=1 \
+    DS4_CUDA_DISABLE_Q4_ATTN_OUT_HC_FUSE=1 \
+    DS4_CUDA_NO_Q4_K1024_PERSISTENT=1
+run_smoke grouped_only \
+    DS4_CUDA_NO_Q4_DENSE_SCRATCH=1 \
+    DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH=1 \
+    DS4_CUDA_DISABLE_Q4_ATTN_OUT_HC_FUSE=1 \
+    DS4_CUDA_NO_Q4_K1024_PERSISTENT=1
+run_smoke hc_only \
+    DS4_CUDA_NO_Q4_DENSE_SCRATCH=1 \
+    DS4_CUDA_NO_Q4_GROUPED_ATTN_A=1 \
+    DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH=1 \
+    DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL=1 \
+    DS4_CUDA_NO_Q4_K1024_PERSISTENT=1
+run_smoke default_fast DS4_CUDA_NO_Q4_K1024_PERSISTENT=1
+run_smoke grouped_q81_rollback \
+    DS4_CUDA_NO_Q4_GROUPED_ATTN_A_Q81=1 \
+    DS4_CUDA_NO_Q4_K1024_PERSISTENT=1
+run_smoke grouped_prefill_rollback \
+    DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL=1 \
+    DS4_CUDA_NO_Q4_K1024_PERSISTENT=1
+
+status=0
+oracle_mismatch_arms=
+run_smoke grouped_oracle \
+    DS4_CUDA_NO_Q4_DENSE_SCRATCH=1 \
+    DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH=1 \
+    DS4_CUDA_DISABLE_Q4_ATTN_OUT_HC_FUSE=1 \
+    DS4_CUDA_NO_Q4_K1024_PERSISTENT=1 \
+    DS4_CUDA_Q4_GROUPED_ATTN_A_ORACLE=1
+check_grouped_oracle "$OUT_DIR/grouped_oracle.log"
+
+run_smoke hc_oracle \
+    DS4_CUDA_NO_Q4_DENSE_SCRATCH=1 \
+    DS4_CUDA_NO_Q4_GROUPED_ATTN_A=1 \
+    DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH=1 \
+    DS4_CUDA_NO_Q4_K1024_PERSISTENT=1 \
+    DS4_CUDA_Q4_ATTN_OUT_HC_ORACLE=1
+check_hc_oracle "$OUT_DIR/hc_oracle.log"
+
+if [ -n "$oracle_mismatch_arms" ]; then
+    status=1
+fi
+changed_smoke=
+compare_smoke() {
+    arm=$1
+    if cmp -s "$OUT_DIR/umbrella_control.json" "$OUT_DIR/$arm.json"; then
+        echo "q4-gb10-matrix: smoke $arm: EXACT"
+    else
+        echo "q4-gb10-matrix: smoke $arm: DIFFERENT"
+        diff -u "$OUT_DIR/umbrella_control.json" "$OUT_DIR/$arm.json" \
+            >"$OUT_DIR/$arm.diff" || true
+        changed_smoke="$changed_smoke $arm"
+        status=1
+    fi
+}
+
+for arm in local_control scratch_only grouped_only hc_only default_fast \
+           grouped_q81_rollback grouped_prefill_rollback \
+           grouped_oracle hc_oracle; do
+    compare_smoke "$arm"
+done
+
+changed_quality=
+run_score umbrella_control DS4_CUDA_NO_Q4_GB10_FAST=1
+set -- $LOCAL_ROLLBACK
+run_score local_control "$@"
+run_score scratch_only \
+    DS4_CUDA_NO_Q4_GROUPED_ATTN_A=1 \
+    DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH=1 \
+    DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL=1 \
+    DS4_CUDA_DISABLE_Q4_ATTN_OUT_HC_FUSE=1 \
+    DS4_CUDA_NO_Q4_K1024_PERSISTENT=1
+run_score grouped_only \
+    DS4_CUDA_NO_Q4_DENSE_SCRATCH=1 \
+    DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH=1 \
+    DS4_CUDA_DISABLE_Q4_ATTN_OUT_HC_FUSE=1 \
+    DS4_CUDA_NO_Q4_K1024_PERSISTENT=1
+run_score hc_only \
+    DS4_CUDA_NO_Q4_DENSE_SCRATCH=1 \
+    DS4_CUDA_NO_Q4_GROUPED_ATTN_A=1 \
+    DS4_CUDA_NO_Q4_GROUPED_ATTN_A_BATCH=1 \
+    DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL=1 \
+    DS4_CUDA_NO_Q4_K1024_PERSISTENT=1
+run_score default_fast DS4_CUDA_NO_Q4_K1024_PERSISTENT=1
+run_score grouped_q81_rollback \
+    DS4_CUDA_NO_Q4_GROUPED_ATTN_A_Q81=1 \
+    DS4_CUDA_NO_Q4_K1024_PERSISTENT=1
+run_score grouped_prefill_rollback \
+    DS4_CUDA_NO_Q4_GROUPED_ATTN_A_PREFILL=1 \
+    DS4_CUDA_NO_Q4_K1024_PERSISTENT=1
+
+for arm in local_control scratch_only grouped_only hc_only default_fast \
+           grouped_q81_rollback grouped_prefill_rollback; do
+    if cmp -s "$OUT_DIR/umbrella_control.tsv" "$OUT_DIR/$arm.tsv"; then
+        echo "q4-gb10-matrix: quality $arm: EXACT"
+    else
+        echo "q4-gb10-matrix: quality $arm: DIFFERENT"
+        python3 gguf-tools/quality-testing/compare_scores.py \
+            "$OUT_DIR/umbrella_control.tsv" "$OUT_DIR/$arm.tsv" \
+            >"$OUT_DIR/$arm.comparison.txt"
+        changed_quality="$changed_quality $arm"
+        status=1
+    fi
+done
+
+if [ "$SKIP_PARITY" != 0 ]; then
+    status=1
+fi
+
+{
+    echo "output_dir=$OUT_DIR"
+    echo "decode_graphs=$DECODE_GRAPHS"
+    echo "smoke_differences=${changed_smoke# }"
+    echo "quality_differences=${changed_quality# }"
+    echo "oracle_mismatches=${oracle_mismatch_arms# }"
+    echo "quality_manifest=$MANIFEST"
+    if [ "$SKIP_PARITY" = 0 ]; then
+        echo "synthetic_parity=pass"
+    else
+        echo "synthetic_parity=skipped"
+    fi
+    if [ "$status" -eq 0 ]; then
+        echo "promotion_gate=pass"
+    else
+        echo "promotion_gate=blocked"
+    fi
+} | tee "$OUT_DIR/summary.txt"
+
+exit "$status"
diff --git a/tests/ds4_test.c b/tests/ds4_test.c
index cf8bca2c52..e7271ae6cb 100644
--- a/tests/ds4_test.c
+++ b/tests/ds4_test.c
@@ -863,6 +863,185 @@ static void test_metal_store_raw_kv_batch_wrap(void) {
     ds4_gpu_tensor_free(raw);
 }
 
+#if defined(__APPLE__)
+typedef struct {
+    volatile uint32_t *sequence_count;
+    volatile uint32_t *sequence;
+    volatile uint32_t  calls;
+    uint32_t           marker;
+} test_metal_progress_callback_ctx;
+
+static void test_metal_progress_callback(void *opaque) {
+    test_metal_progress_callback_ctx *ctx = opaque;
+    const uint32_t slot = __atomic_fetch_add(
+        ctx->sequence_count, 1u, __ATOMIC_ACQ_REL);
+    if (slot < 2u) {
+        __atomic_store_n(
+            ctx->sequence + slot, ctx->marker, __ATOMIC_RELEASE);
+    }
+    (void)__atomic_fetch_add(&ctx->calls, 1u, __ATOMIC_ACQ_REL);
+}
+
+static void test_metal_flush_commands_progress_exact(void) {
+    enum {
+        n = 257,
+        guard = 19,
+        alloc_n = n + guard,
+    };
+    const uint64_t active_bytes = (uint64_t)n * sizeof(float);
+    const uint64_t alloc_bytes = (uint64_t)alloc_n * sizeof(float);
+    const uint32_t tmp_poison = 0x7fc1a500u;
+    const uint32_t out_poison = 0x7fc25a00u;
+    float a_host[alloc_n];
+    float b_host[alloc_n];
+    float c_host[alloc_n];
+    float tmp_host[alloc_n];
+    float out_host[alloc_n];
+    float expected_tmp[n];
+    float expected_out[n];
+
+    for (uint32_t i = 0; i < alloc_n; i++) {
+        a_host[i] = (float)((int)(i % 17u) - 8) * 0.5f;
+        b_host[i] = (float)((int)((i * 5u) % 23u) - 11) * 0.25f;
+        c_host[i] = (float)((int)((i * 7u) % 29u) - 14) * 0.125f;
+        const uint32_t tmp_bits = tmp_poison + (i & 0xffu);
+        const uint32_t out_bits = out_poison + (i & 0xffu);
+        memcpy(tmp_host + i, &tmp_bits, sizeof(tmp_bits));
+        memcpy(out_host + i, &out_bits, sizeof(out_bits));
+        if (i < n) {
+            expected_tmp[i] = a_host[i] + b_host[i];
+            expected_out[i] = expected_tmp[i] + c_host[i];
+        }
+    }
+
+    ds4_gpu_tensor *a = ds4_gpu_tensor_alloc(alloc_bytes);
+    ds4_gpu_tensor *b = ds4_gpu_tensor_alloc(alloc_bytes);
+    ds4_gpu_tensor *c = ds4_gpu_tensor_alloc(alloc_bytes);
+    ds4_gpu_tensor *tmp_base = ds4_gpu_tensor_alloc(alloc_bytes);
+    ds4_gpu_tensor *out_base = ds4_gpu_tensor_alloc(alloc_bytes);
+    ds4_gpu_tensor *tmp = tmp_base
+        ? ds4_gpu_tensor_view(tmp_base, 0u, active_bytes) : NULL;
+    ds4_gpu_tensor *out = out_base
+        ? ds4_gpu_tensor_view(out_base, 0u, active_bytes) : NULL;
+    TEST_ASSERT(a && b && c && tmp_base && out_base && tmp && out);
+
+    volatile uint32_t sequence_count = 0u;
+    volatile uint32_t sequence[2] = {0u, 0u};
+    test_metal_progress_callback_ctx callback0 = {
+        .sequence_count = &sequence_count,
+        .sequence = sequence,
+        .calls = 0u,
+        .marker = 1u,
+    };
+    test_metal_progress_callback_ctx callback1 = {
+        .sequence_count = &sequence_count,
+        .sequence = sequence,
+        .calls = 0u,
+        .marker = 2u,
+    };
+
+    bool submitted = false;
+    if (a && b && c && tmp_base && out_base && tmp && out) {
+        TEST_ASSERT(ds4_gpu_tensor_write(a, 0u, a_host, alloc_bytes) != 0);
+        TEST_ASSERT(ds4_gpu_tensor_write(b, 0u, b_host, alloc_bytes) != 0);
+        TEST_ASSERT(ds4_gpu_tensor_write(c, 0u, c_host, alloc_bytes) != 0);
+        TEST_ASSERT(ds4_gpu_tensor_write(
+                        tmp_base, 0u, tmp_host, alloc_bytes) != 0);
+        TEST_ASSERT(ds4_gpu_tensor_write(
+                        out_base, 0u, out_host, alloc_bytes) != 0);
+        TEST_ASSERT(ds4_gpu_commands_active() == 0);
+
+        const int begun = ds4_gpu_begin_commands();
+        TEST_ASSERT(begun != 0);
+        const int first = begun
+            ? ds4_gpu_add_tensor(tmp, a, b, n) : 0;
+        TEST_ASSERT(first != 0);
+        const int flushed0 = first
+            ? ds4_gpu_flush_commands_progress(
+                  test_metal_progress_callback, &callback0)
+            : 0;
+        TEST_ASSERT(flushed0 != 0);
+        const int second = flushed0
+            ? ds4_gpu_add_tensor(out, tmp, c, n) : 0;
+        TEST_ASSERT(second != 0);
+        const int flushed1 = second
+            ? ds4_gpu_flush_commands_progress(
+                  test_metal_progress_callback, &callback1)
+            : 0;
+        TEST_ASSERT(flushed1 != 0);
+        submitted = flushed1 != 0;
+
+        const int drained = ds4_gpu_commands_active()
+            ? ds4_gpu_end_commands() : 0;
+        TEST_ASSERT(drained != 0);
+        TEST_ASSERT(ds4_gpu_commands_active() == 0);
+    }
+
+    test_float_compare_stats tmp_stats = {0};
+    test_float_compare_stats out_stats = {0};
+    size_t guard_mismatches = 0u;
+    if (submitted && tmp_base && out_base) {
+        TEST_ASSERT(ds4_gpu_tensor_read(
+                        tmp_base, 0u, tmp_host, alloc_bytes) != 0);
+        TEST_ASSERT(ds4_gpu_tensor_read(
+                        out_base, 0u, out_host, alloc_bytes) != 0);
+        tmp_stats = test_compare_float_bits(expected_tmp, tmp_host, n);
+        out_stats = test_compare_float_bits(expected_out, out_host, n);
+        for (uint32_t i = n; i < alloc_n; i++) {
+            uint32_t tmp_bits = 0u;
+            uint32_t out_bits = 0u;
+            memcpy(&tmp_bits, tmp_host + i, sizeof(tmp_bits));
+            memcpy(&out_bits, out_host + i, sizeof(out_bits));
+            if (tmp_bits != tmp_poison + (i & 0xffu) ||
+                out_bits != out_poison + (i & 0xffu)) {
+                guard_mismatches++;
+            }
+        }
+    }
+
+    const uint32_t calls0 =
+        __atomic_load_n(&callback0.calls, __ATOMIC_ACQUIRE);
+    const uint32_t calls1 =
+        __atomic_load_n(&callback1.calls, __ATOMIC_ACQUIRE);
+    const uint32_t completed =
+        __atomic_load_n(&sequence_count, __ATOMIC_ACQUIRE);
+    const uint32_t observed0 =
+        __atomic_load_n(sequence + 0u, __ATOMIC_ACQUIRE);
+    const uint32_t observed1 =
+        __atomic_load_n(sequence + 1u, __ATOMIC_ACQUIRE);
+    const bool distinct_contexts =
+        (observed0 == 1u && observed1 == 2u) ||
+        (observed0 == 2u && observed1 == 1u);
+    fprintf(stderr,
+            "ds4-test: Metal progress flush exact callbacks=%u/%u "
+            "completed=%u order=%u,%u tmp=%zu out=%zu guard=%zu\n",
+            calls0, calls1, completed, observed0, observed1,
+            tmp_stats.mismatch_count, out_stats.mismatch_count,
+            guard_mismatches);
+    TEST_ASSERT(calls0 == 1u);
+    TEST_ASSERT(calls1 == 1u);
+    TEST_ASSERT(completed == 2u);
+    /* Completion-handler ordering across different command buffers is not a
+     * public contract.  The dependent tensor result proves GPU submission
+     * order; here only prove that both distinct contexts were delivered. */
+    TEST_ASSERT(distinct_contexts);
+    TEST_ASSERT(tmp_stats.mismatch_count == 0u && tmp_stats.max_ulp == 0u);
+    TEST_ASSERT(out_stats.mismatch_count == 0u && out_stats.max_ulp == 0u);
+    TEST_ASSERT(guard_mismatches == 0u);
+
+    if (ds4_gpu_commands_active()) {
+        TEST_ASSERT(ds4_gpu_end_commands() != 0);
+    }
+    ds4_gpu_tensor_free(out);
+    ds4_gpu_tensor_free(tmp);
+    ds4_gpu_tensor_free(out_base);
+    ds4_gpu_tensor_free(tmp_base);
+    ds4_gpu_tensor_free(c);
+    ds4_gpu_tensor_free(b);
+    ds4_gpu_tensor_free(a);
+}
+#endif
+
 static void test_dspark_cache_window_crop(void) {
     TEST_ASSERT(ds4_test_dspark_cache_window_crop());
 }
@@ -1007,6 +1186,1070 @@ static void test_metal_q8_0_decode_pair_exact(void) {
 }
 
 #if defined(__APPLE__)
+static void test_fill_q4_K_weights(uint8_t *weights,
+                                   uint32_t in_dim,
+                                   uint32_t out_dim,
+                                   uint32_t seed) {
+    /* GGUF Q4_K block: f16 d, f16 dmin, 12 packed scales, 128 quants. */
+    const uint32_t block_elems = 256u;
+    const uint32_t block_bytes = 144u;
+    TEST_ASSERT(weights != NULL);
+    TEST_ASSERT((in_dim % block_elems) == 0u);
+    if (!weights || (in_dim % block_elems) != 0u) return;
+
+    const uint32_t blocks_per_row = in_dim / block_elems;
+    for (uint32_t row = 0; row < out_dim; row++) {
+        for (uint32_t block = 0; block < blocks_per_row; block++) {
+            uint8_t *q = weights +
+                ((uint64_t)row * blocks_per_row + block) * block_bytes;
+            const uint32_t key =
+                seed + row * 1009u + block * 313u + (row ^ (block * 17u));
+            const uint16_t d = test_float_to_f16(
+                0.0025f + (float)(key % 13u) / 4096.0f);
+            const uint16_t dmin = test_float_to_f16(
+                0.0010f + (float)((key >> 3u) % 7u) / 8192.0f);
+            memcpy(q + 0u, &d, sizeof(d));
+            memcpy(q + 2u, &dmin, sizeof(dmin));
+            for (uint32_t i = 0; i < 12u; i++) {
+                q[4u + i] = (uint8_t)(
+                    1u + ((key + i * 23u + (i ^ row) * 5u) % 0xfeu));
+            }
+            for (uint32_t i = 0; i < 128u; i++) {
+                q[16u + i] = (uint8_t)(
+                    key + i * 37u + (i >> 2u) * 11u + row * 3u);
+            }
+        }
+    }
+}
+
+static void test_metal_q8_0_decode_rows_exact(void) {
+    /* The exact-N output head must preserve the canonical one-row Q8_0
+     * arithmetic for every speculative row, including the output tail. */
+    const uint32_t in_dim = 4096u;
+    const uint32_t out_dim = 259u;
+    const uint32_t max_rows = 5u;
+    const uint64_t page = (uint64_t)getpagesize();
+    const uint64_t row_bytes = (uint64_t)(in_dim / 32u) * 34u;
+    const uint64_t weight_bytes = (uint64_t)out_dim * row_bytes;
+    const uint64_t weight_alloc = test_round_up_u64(weight_bytes, page);
+    const uint64_t x_row_bytes = (uint64_t)in_dim * sizeof(float);
+    const uint64_t out_row_bytes = (uint64_t)out_dim * sizeof(float);
+    const uint64_t x_bytes = (uint64_t)max_rows * x_row_bytes;
+    const uint64_t out_bytes = (uint64_t)max_rows * out_row_bytes;
+
+    void *weights_raw = NULL;
+    TEST_ASSERT(posix_memalign(&weights_raw,
+                               (size_t)page,
+                               (size_t)weight_alloc) == 0);
+    if (!weights_raw) return;
+    memset(weights_raw, 0, (size_t)weight_alloc);
+    test_fill_q8_0_weights(weights_raw, in_dim, out_dim, 173u);
+
+    ds4_gpu_tensor *x = ds4_gpu_tensor_alloc(x_bytes);
+    ds4_gpu_tensor *reference = ds4_gpu_tensor_alloc(out_bytes);
+    ds4_gpu_tensor *batched = ds4_gpu_tensor_alloc(out_bytes);
+    TEST_ASSERT(x != NULL);
+    TEST_ASSERT(reference != NULL);
+    TEST_ASSERT(batched != NULL);
+    if (!x || !reference || !batched) {
+        ds4_gpu_tensor_free(x);
+        ds4_gpu_tensor_free(reference);
+        ds4_gpu_tensor_free(batched);
+        free(weights_raw);
+        return;
+    }
+
+    float *x_host = malloc((size_t)x_bytes);
+    float *reference_host = malloc((size_t)out_bytes);
+    float *batched_host = malloc((size_t)out_bytes);
+    TEST_ASSERT(x_host != NULL);
+    TEST_ASSERT(reference_host != NULL);
+    TEST_ASSERT(batched_host != NULL);
+    if (!x_host || !reference_host || !batched_host) {
+        free(x_host);
+        free(reference_host);
+        free(batched_host);
+        ds4_gpu_tensor_free(x);
+        ds4_gpu_tensor_free(reference);
+        ds4_gpu_tensor_free(batched);
+        free(weights_raw);
+        return;
+    }
+
+    for (uint32_t row = 0; row < max_rows; row++) {
+        for (uint32_t i = 0; i < in_dim; i++) {
+            const uint32_t mix =
+                i * 29u + row * 101u + ((i + row * 17u) ^ (i >> 3u)) * 7u;
+            x_host[(uint64_t)row * in_dim + i] =
+                (float)((int)(mix % 191u) - 95) / 113.0f;
+        }
+    }
+
+    TEST_ASSERT(ds4_gpu_tensor_write(x, 0, x_host, x_bytes) != 0);
+    TEST_ASSERT(ds4_gpu_set_model_map(weights_raw, weight_alloc) != 0);
+    ds4_gpu_set_quality(false);
+
+    for (uint32_t row = 0; row < max_rows; row++) {
+        ds4_gpu_tensor *x_row = ds4_gpu_tensor_view(
+                x, (uint64_t)row * x_row_bytes, x_row_bytes);
+        ds4_gpu_tensor *out_row = ds4_gpu_tensor_view(
+                reference, (uint64_t)row * out_row_bytes, out_row_bytes);
+        TEST_ASSERT(x_row != NULL);
+        TEST_ASSERT(out_row != NULL);
+        if (x_row && out_row) {
+            TEST_ASSERT(ds4_gpu_matmul_q8_0_tensor(
+                            out_row, weights_raw, weight_alloc, 0,
+                            in_dim, out_dim, x_row, 1) != 0);
+        }
+        ds4_gpu_tensor_free(x_row);
+        ds4_gpu_tensor_free(out_row);
+    }
+    TEST_ASSERT(ds4_gpu_tensor_read(
+                    reference, 0, reference_host, out_bytes) != 0);
+
+    for (uint32_t n_rows = 2u; n_rows <= max_rows; n_rows++) {
+        memset(batched_host, 0xa5, (size_t)out_bytes);
+        TEST_ASSERT(ds4_gpu_tensor_write(
+                        batched, 0, batched_host, out_bytes) != 0);
+        TEST_ASSERT(ds4_gpu_matmul_q8_0_decode_rows_exact_tensor(
+                        batched, weights_raw, weight_alloc, 0,
+                        in_dim, out_dim, x, n_rows) != 0);
+        TEST_ASSERT(ds4_gpu_tensor_read(
+                        batched, 0, batched_host, out_bytes) != 0);
+
+        const size_t compared = (size_t)n_rows * out_dim;
+        const test_float_compare_stats stats = test_compare_float_bits(
+                reference_host, batched_host, compared);
+        if (stats.mismatch_count != 0) {
+            fprintf(stderr,
+                    "ds4-test: Metal Q8_0 exact-%u rows mismatches=%zu/%zu "
+                    "max_ulp=%u max_abs=%g\n",
+                    n_rows,
+                    stats.mismatch_count,
+                    compared,
+                    stats.max_ulp,
+                    stats.max_abs);
+        }
+        TEST_ASSERT(memcmp(reference_host,
+                           batched_host,
+                           compared * sizeof(float)) == 0);
+    }
+
+    free(x_host);
+    free(reference_host);
+    free(batched_host);
+    ds4_gpu_tensor_free(x);
+    ds4_gpu_tensor_free(reference);
+    ds4_gpu_tensor_free(batched);
+    free(weights_raw);
+}
+
+static void test_fill_q8_0_constant_weights(uint8_t *weights,
+                                             uint32_t in_dim,
+                                             uint32_t out_dim,
+                                             int8_t quant) {
+    const uint32_t blocks = in_dim / 32u;
+    const uint64_t row_bytes = (uint64_t)blocks * 34u;
+    const uint16_t scale_bits = test_float_to_f16(1.0f / 256.0f);
+    for (uint32_t row = 0; row < out_dim; row++) {
+        uint8_t *dst = weights + (uint64_t)row * row_bytes;
+        for (uint32_t block = 0; block < blocks; block++) {
+            memcpy(dst + (uint64_t)block * 34u,
+                   &scale_bits,
+                   sizeof(scale_bits));
+            memset(dst + (uint64_t)block * 34u + 2u,
+                   (unsigned char)quant,
+                   32u);
+        }
+    }
+}
+
+static bool test_metal_q8_attention_output_static_batch_exact_case(
+        uint32_t n_tokens) {
+    const int failures_before = test_failures;
+    /*
+     * The production AProjQ8 static kernel receives a flattened z coordinate:
+     *
+     *     pair = token * n_groups + group
+     *
+     * Only pair % n_groups may select Woa.  Keep a second, sign-inverted Woa
+     * immediately after the real one so the old pair-as-group bug is a safe,
+     * deterministic wrong read for token 1 instead of an out-of-bounds Metal
+     * access.  The public batch API also runs the small Q8 output projection;
+     * both its low intermediate and final output must match the generic direct
+     * kernel bit for bit.
+     */
+    const uint32_t group_dim = 4096u;
+    const uint32_t rank = 1024u;
+    const uint32_t n_groups = 8u;
+    const uint32_t low_dim = n_groups * rank;
+    const uint32_t out_dim = 32u;
+    const uint32_t alloc_tokens = n_tokens + 1u;
+    const uint64_t page = (uint64_t)getpagesize();
+    const uint64_t row_a_bytes = (uint64_t)(group_dim / 32u) * 34u;
+    const uint64_t group_a_bytes = (uint64_t)rank * row_a_bytes;
+    const uint64_t out_a_bytes = (uint64_t)n_groups * group_a_bytes;
+    const uint64_t shadow_a_offset = out_a_bytes;
+    const uint64_t out_b_offset = 2u * out_a_bytes;
+    const uint64_t row_b_bytes = (uint64_t)(low_dim / 32u) * 34u;
+    const uint64_t out_b_bytes = (uint64_t)out_dim * row_b_bytes;
+    const uint64_t model_bytes =
+        test_round_up_u64(out_b_offset + out_b_bytes, page);
+    const uint64_t heads_bytes =
+        (uint64_t)alloc_tokens * n_groups * group_dim * sizeof(float);
+    const uint64_t low_bytes =
+        (uint64_t)alloc_tokens * low_dim * sizeof(float);
+    const uint64_t out_bytes =
+        (uint64_t)alloc_tokens * out_dim * sizeof(float);
+    const uint64_t active_low_bytes =
+        (uint64_t)n_tokens * low_dim * sizeof(float);
+    const uint64_t active_out_bytes =
+        (uint64_t)n_tokens * out_dim * sizeof(float);
+    const char *disable_direct_env =
+        "DS4_METAL_DISABLE_ATTN_OUT_LOW_DIRECT";
+    const char *disable_ports_env =
+        "DS4_METAL_DISABLE_PRE_M5_DECODE_PORTS";
+    const char *disable_static_env =
+        "DS4_METAL_DISABLE_PRE_M5_ATTN_OUT_LOW_Q8_STATIC";
+
+    _Static_assert(4096u / 32u * 34u == 4352u,
+                   "production AProjQ8 row size changed");
+    _Static_assert(8u * 1024u * 4352u == 35651584u,
+                   "production AProjQ8 Woa size changed");
+
+    char *saved_disable_direct = test_save_env(disable_direct_env);
+    char *saved_disable_ports = test_save_env(disable_ports_env);
+    char *saved_disable_static = test_save_env(disable_static_env);
+    void *model_raw = NULL;
+    float *heads_host = NULL;
+    float *reference_low_host = NULL;
+    float *candidate_low_host = NULL;
+    float *reference_out_host = NULL;
+    float *candidate_out_host = NULL;
+    ds4_gpu_tensor *heads = NULL;
+    ds4_gpu_tensor *reference_low = NULL;
+    ds4_gpu_tensor *candidate_low = NULL;
+    ds4_gpu_tensor *reference_out = NULL;
+    ds4_gpu_tensor *candidate_out = NULL;
+    ds4_gpu_tensor *group_tmp = NULL;
+    ds4_gpu_tensor *low_tmp = NULL;
+
+    TEST_ASSERT(row_a_bytes == 4352u);
+    TEST_ASSERT(out_a_bytes == 35651584u);
+    TEST_ASSERT(posix_memalign(
+                    &model_raw, (size_t)page, (size_t)model_bytes) == 0);
+    if (!model_raw) goto cleanup;
+    memset(model_raw, 0, (size_t)model_bytes);
+    for (uint32_t group = 0; group < n_groups; group++) {
+        test_fill_q8_0_constant_weights(
+            (uint8_t *)model_raw + (uint64_t)group * group_a_bytes,
+            group_dim,
+            rank,
+            (int8_t)(group + 1u));
+        test_fill_q8_0_constant_weights(
+            (uint8_t *)model_raw + shadow_a_offset +
+                (uint64_t)group * group_a_bytes,
+            group_dim,
+            rank,
+            (int8_t)-(int8_t)(group + 1u));
+    }
+    test_fill_q8_0_constant_weights(
+        (uint8_t *)model_raw + out_b_offset,
+        low_dim,
+        out_dim,
+        1);
+
+    heads_host = malloc((size_t)heads_bytes);
+    reference_low_host = malloc((size_t)low_bytes);
+    candidate_low_host = malloc((size_t)low_bytes);
+    reference_out_host = malloc((size_t)out_bytes);
+    candidate_out_host = malloc((size_t)out_bytes);
+    heads = ds4_gpu_tensor_alloc(heads_bytes);
+    reference_low = ds4_gpu_tensor_alloc(low_bytes);
+    candidate_low = ds4_gpu_tensor_alloc(low_bytes);
+    reference_out = ds4_gpu_tensor_alloc(out_bytes);
+    candidate_out = ds4_gpu_tensor_alloc(out_bytes);
+    group_tmp = ds4_gpu_tensor_alloc(
+        (uint64_t)n_tokens * group_dim * sizeof(float));
+    low_tmp = ds4_gpu_tensor_alloc(
+        (uint64_t)n_tokens * rank * sizeof(float));
+    TEST_ASSERT(heads_host && reference_low_host && candidate_low_host &&
+                reference_out_host && candidate_out_host && heads &&
+                reference_low && candidate_low && reference_out &&
+                candidate_out && group_tmp && low_tmp);
+    if (!heads_host || !reference_low_host || !candidate_low_host ||
+        !reference_out_host || !candidate_out_host || !heads ||
+        !reference_low || !candidate_low || !reference_out ||
+        !candidate_out || !group_tmp || !low_tmp) {
+        goto cleanup;
+    }
+
+    for (uint64_t i = 0; i < heads_bytes / sizeof(float); i++) {
+        const uint32_t token = (uint32_t)(i / ((uint64_t)n_groups * group_dim));
+        const uint32_t key =
+            (uint32_t)i * 17u + token * 131u + ((uint32_t)i >> 4u);
+        heads_host[i] = (float)(1u + key % 13u) / 128.0f;
+    }
+    memset(reference_low_host, 0xa5, (size_t)low_bytes);
+    memset(candidate_low_host, 0xa5, (size_t)low_bytes);
+    memset(reference_out_host, 0xa5, (size_t)out_bytes);
+    memset(candidate_out_host, 0xa5, (size_t)out_bytes);
+    TEST_ASSERT(ds4_gpu_tensor_write(
+                    heads, 0, heads_host, heads_bytes) != 0);
+    TEST_ASSERT(ds4_gpu_tensor_write(
+                    reference_low, 0, reference_low_host, low_bytes) != 0);
+    TEST_ASSERT(ds4_gpu_tensor_write(
+                    candidate_low, 0, candidate_low_host, low_bytes) != 0);
+    TEST_ASSERT(ds4_gpu_tensor_write(
+                    reference_out, 0, reference_out_host, out_bytes) != 0);
+    TEST_ASSERT(ds4_gpu_tensor_write(
+                    candidate_out, 0, candidate_out_host, out_bytes) != 0);
+    TEST_ASSERT(ds4_gpu_set_model_map(model_raw, model_bytes) != 0);
+    ds4_gpu_set_quality(false);
+
+    TEST_ASSERT(unsetenv(disable_direct_env) == 0);
+    TEST_ASSERT(unsetenv(disable_ports_env) == 0);
+    TEST_ASSERT(unsetenv(disable_static_env) == 0);
+    ds4_gpu_test_set_flags(DS4_GPU_TEST_ATTN_OUT_LOW_Q8_STATIC);
+    TEST_ASSERT(ds4_gpu_attention_output_q8_batch_tensor(
+                    candidate_out,
+                    candidate_low,
+                    group_tmp,
+                    low_tmp,
+                    model_raw,
+                    model_bytes,
+                    0,
+                    out_b_offset,
+                    group_dim,
+                    rank,
+                    n_groups,
+                    out_dim,
+                    heads,
+                    n_tokens) == 1);
+
+    ds4_gpu_test_set_flags(0);
+    TEST_ASSERT(setenv(disable_static_env, "1", 1) == 0);
+    TEST_ASSERT(ds4_gpu_attention_output_q8_batch_tensor(
+                    reference_out,
+                    reference_low,
+                    group_tmp,
+                    low_tmp,
+                    model_raw,
+                    model_bytes,
+                    0,
+                    out_b_offset,
+                    group_dim,
+                    rank,
+                    n_groups,
+                    out_dim,
+                    heads,
+                    n_tokens) == 1);
+
+    TEST_ASSERT(ds4_gpu_tensor_read(
+                    reference_low, 0, reference_low_host, low_bytes) != 0);
+    TEST_ASSERT(ds4_gpu_tensor_read(
+                    candidate_low, 0, candidate_low_host, low_bytes) != 0);
+    TEST_ASSERT(ds4_gpu_tensor_read(
+                    reference_out, 0, reference_out_host, out_bytes) != 0);
+    TEST_ASSERT(ds4_gpu_tensor_read(
+                    candidate_out, 0, candidate_out_host, out_bytes) != 0);
+
+    const size_t low_count = (size_t)n_tokens * low_dim;
+    const size_t out_count = (size_t)n_tokens * out_dim;
+    const test_float_compare_stats low_stats = test_compare_float_bits(
+        reference_low_host, candidate_low_host, low_count);
+    const test_float_compare_stats out_stats = test_compare_float_bits(
+        reference_out_host, candidate_out_host, out_count);
+    size_t tail_byte_mismatch = 0;
+    for (uint64_t i = active_low_bytes; i < low_bytes; i++) {
+        if (((const uint8_t *)reference_low_host)[i] != 0xa5u) {
+            tail_byte_mismatch++;
+        }
+        if (((const uint8_t *)candidate_low_host)[i] != 0xa5u) {
+            tail_byte_mismatch++;
+        }
+    }
+    for (uint64_t i = active_out_bytes; i < out_bytes; i++) {
+        if (((const uint8_t *)reference_out_host)[i] != 0xa5u) {
+            tail_byte_mismatch++;
+        }
+        if (((const uint8_t *)candidate_out_host)[i] != 0xa5u) {
+            tail_byte_mismatch++;
+        }
+    }
+    fprintf(stderr,
+            "ds4-test: Metal Q8 static attention-output exact-%u "
+            "low=%zu/%zu max_ulp=%u max_abs=%g "
+            "out=%zu/%zu max_ulp=%u max_abs=%g tail_bytes=%zu\n",
+            n_tokens,
+            low_stats.mismatch_count,
+            low_count,
+            low_stats.max_ulp,
+            low_stats.max_abs,
+            out_stats.mismatch_count,
+            out_count,
+            out_stats.max_ulp,
+            out_stats.max_abs,
+            tail_byte_mismatch);
+    TEST_ASSERT(low_stats.mismatch_count == 0 && low_stats.max_ulp == 0);
+    TEST_ASSERT(out_stats.mismatch_count == 0 && out_stats.max_ulp == 0);
+    TEST_ASSERT(tail_byte_mismatch == 0);
+
+cleanup:
+    ds4_gpu_test_set_flags(0);
+    ds4_gpu_tensor_free(low_tmp);
+    ds4_gpu_tensor_free(group_tmp);
+    ds4_gpu_tensor_free(candidate_out);
+    ds4_gpu_tensor_free(reference_out);
+    ds4_gpu_tensor_free(candidate_low);
+    ds4_gpu_tensor_free(reference_low);
+    ds4_gpu_tensor_free(heads);
+    free(candidate_out_host);
+    free(reference_out_host);
+    free(candidate_low_host);
+    free(reference_low_host);
+    free(heads_host);
+    free(model_raw);
+    test_restore_env(disable_static_env, saved_disable_static);
+    test_restore_env(disable_ports_env, saved_disable_ports);
+    test_restore_env(disable_direct_env, saved_disable_direct);
+    return test_failures == failures_before;
+}
+
+static void test_metal_q8_attention_output_static_batch_exact(void) {
+    /* The shadow Woa makes the old bug safe at N=2; stop there on failure. */
+    if (test_metal_q8_attention_output_static_batch_exact_case(2u)) {
+        (void)test_metal_q8_attention_output_static_batch_exact_case(31u);
+    }
+}
+
+static void test_metal_q4_attention_output_tiny_batch_exact_case(
+        uint32_t out_b_type) {
+    /*
+     * The opt-in AProjQ4 tiny batch must be the exact composition of the two
+     * canonical one-row projections for both deployed output quantizations.
+     * Keep a sixth poisoned row in every destination so each N=2..5 case also
+     * checks dispatch bounds.
+     */
+    TEST_ASSERT(out_b_type == 8u || out_b_type == 12u);
+    if (out_b_type != 8u && out_b_type != 12u) return;
+    const uint32_t group_dim = 256u;
+    const uint32_t rank = 128u;
+    const uint32_t n_groups = 2u;
+    const uint32_t low_dim = n_groups * rank;
+    const uint32_t out_dim = 67u;
+    const uint32_t max_rows = 5u;
+    const uint32_t alloc_rows = max_rows + 1u;
+    const uint64_t page = (uint64_t)getpagesize();
+    const uint64_t row_a_bytes = (uint64_t)(group_dim / 256u) * 144u;
+    const uint64_t out_a_bytes =
+        (uint64_t)n_groups * rank * row_a_bytes;
+    const uint64_t out_b_offset = test_round_up_u64(out_a_bytes, page);
+    const uint64_t row_b_bytes = out_b_type == 8u
+        ? (uint64_t)(low_dim / 32u) * 34u
+        : (uint64_t)(low_dim / 256u) * 144u;
+    const uint64_t out_b_bytes = (uint64_t)out_dim * row_b_bytes;
+    const uint64_t model_bytes =
+        test_round_up_u64(out_b_offset + out_b_bytes, page);
+    const uint64_t heads_row_bytes =
+        (uint64_t)n_groups * group_dim * sizeof(float);
+    const uint64_t low_row_bytes = (uint64_t)low_dim * sizeof(float);
+    const uint64_t out_row_bytes = (uint64_t)out_dim * sizeof(float);
+    const uint64_t heads_bytes = (uint64_t)alloc_rows * heads_row_bytes;
+    const uint64_t low_bytes = (uint64_t)alloc_rows * low_row_bytes;
+    const uint64_t out_bytes = (uint64_t)alloc_rows * out_row_bytes;
+    const char *enable_env =
+        "DS4_METAL_ENABLE_Q4_ATTN_OUT_TINY_BATCH";
+    const char *disable_env =
+        "DS4_METAL_DISABLE_Q4_ATTN_OUT_TINY_BATCH";
+    const char *require_env =
+        "DS4_METAL_REQUIRE_Q4_ATTN_OUT_TINY_BATCH";
+    const char *disable_classic_env =
+        "DS4_METAL_DISABLE_Q4_MV_CLASSIC";
+
+    char *saved_enable = test_save_env(enable_env);
+    char *saved_disable = test_save_env(disable_env);
+    char *saved_require = test_save_env(require_env);
+    char *saved_disable_classic = test_save_env(disable_classic_env);
+    void *model_raw = NULL;
+    float *heads_host = NULL;
+    float *reference_low_host = NULL;
+    float *reference_out_host = NULL;
+    float *candidate_low_host = NULL;
+    float *candidate_out_host = NULL;
+    ds4_gpu_tensor *heads = NULL;
+    ds4_gpu_tensor *reference_low = NULL;
+    ds4_gpu_tensor *reference_out = NULL;
+    ds4_gpu_tensor *candidate_low = NULL;
+    ds4_gpu_tensor *candidate_out = NULL;
+    ds4_gpu_tensor *group_tmp = NULL;
+    ds4_gpu_tensor *low_tmp = NULL;
+
+    TEST_ASSERT(posix_memalign(
+                    &model_raw, (size_t)page, (size_t)model_bytes) == 0);
+    if (!model_raw) goto cleanup;
+    memset(model_raw, 0, (size_t)model_bytes);
+    test_fill_q4_K_weights((uint8_t *)model_raw,
+                           group_dim,
+                           n_groups * rank,
+                           211u);
+    if (out_b_type == 8u) {
+        test_fill_q8_0_weights(
+            (uint8_t *)model_raw + out_b_offset, low_dim, out_dim, 307u);
+    } else {
+        test_fill_q4_K_weights(
+            (uint8_t *)model_raw + out_b_offset, low_dim, out_dim, 307u);
+    }
+
+    heads_host = malloc((size_t)heads_bytes);
+    reference_low_host = malloc((size_t)low_bytes);
+    reference_out_host = malloc((size_t)out_bytes);
+    candidate_low_host = malloc((size_t)low_bytes);
+    candidate_out_host = malloc((size_t)out_bytes);
+    heads = ds4_gpu_tensor_alloc(heads_bytes);
+    reference_low = ds4_gpu_tensor_alloc(low_bytes);
+    reference_out = ds4_gpu_tensor_alloc(out_bytes);
+    candidate_low = ds4_gpu_tensor_alloc(low_bytes);
+    candidate_out = ds4_gpu_tensor_alloc(out_bytes);
+    group_tmp = ds4_gpu_tensor_alloc(
+        (uint64_t)alloc_rows * group_dim * sizeof(float));
+    low_tmp = ds4_gpu_tensor_alloc(
+        (uint64_t)alloc_rows * rank * sizeof(float));
+    TEST_ASSERT(heads_host && reference_low_host && reference_out_host &&
+                candidate_low_host && candidate_out_host && heads &&
+                reference_low && reference_out && candidate_low &&
+                candidate_out && group_tmp && low_tmp);
+    if (!heads_host || !reference_low_host || !reference_out_host ||
+        !candidate_low_host || !candidate_out_host || !heads ||
+        !reference_low || !reference_out || !candidate_low ||
+        !candidate_out || !group_tmp || !low_tmp) {
+        goto cleanup;
+    }
+
+    for (uint32_t row = 0; row < alloc_rows; row++) {
+        for (uint32_t i = 0; i < n_groups * group_dim; i++) {
+            const uint32_t key =
+                i * 41u + row * 271u + ((i >> 2u) ^ (row * 19u));
+            heads_host[(uint64_t)row * n_groups * group_dim + i] =
+                (float)((int)(key % 257u) - 128) / 137.0f;
+        }
+    }
+    memset(reference_low_host, 0, (size_t)low_bytes);
+    memset(reference_out_host, 0, (size_t)out_bytes);
+    TEST_ASSERT(ds4_gpu_tensor_write(
+                    heads, 0, heads_host, heads_bytes) != 0);
+    TEST_ASSERT(ds4_gpu_tensor_write(
+                    reference_low, 0, reference_low_host, low_bytes) != 0);
+    TEST_ASSERT(ds4_gpu_tensor_write(
+                    reference_out, 0, reference_out_host, out_bytes) != 0);
+    TEST_ASSERT(ds4_gpu_set_model_map(model_raw, model_bytes) != 0);
+    ds4_gpu_set_quality(false);
+    TEST_ASSERT(unsetenv(disable_classic_env) == 0);
+
+    for (uint32_t row = 0; row < max_rows; row++) {
+        ds4_gpu_tensor *heads_row = ds4_gpu_tensor_view(
+            heads, (uint64_t)row * heads_row_bytes, heads_row_bytes);
+        ds4_gpu_tensor *low_row = ds4_gpu_tensor_view(
+            reference_low, (uint64_t)row * low_row_bytes, low_row_bytes);
+        ds4_gpu_tensor *out_row = ds4_gpu_tensor_view(
+            reference_out, (uint64_t)row * out_row_bytes, out_row_bytes);
+        TEST_ASSERT(heads_row && low_row && out_row);
+        if (heads_row && low_row && out_row) {
+            TEST_ASSERT(ds4_gpu_attention_output_low_q4_K_slice_tensor(
+                            low_row,
+                            model_raw,
+                            model_bytes,
+                            0,
+                            group_dim,
+                            rank,
+                            0,
+                            n_groups,
+                            heads_row,
+                            0) != 0);
+            if (out_b_type == 8u) {
+                TEST_ASSERT(ds4_gpu_matmul_q8_0_tensor(
+                                out_row,
+                                model_raw,
+                                model_bytes,
+                                out_b_offset,
+                                low_dim,
+                                out_dim,
+                                low_row,
+                                1) != 0);
+            } else {
+                TEST_ASSERT(ds4_gpu_matmul_quant_tensor(
+                                out_row,
+                                model_raw,
+                                model_bytes,
+                                out_b_offset,
+                                out_b_type,
+                                low_dim,
+                                out_dim,
+                                low_row,
+                                1) != 0);
+            }
+        }
+        ds4_gpu_tensor_free(heads_row);
+        ds4_gpu_tensor_free(low_row);
+        ds4_gpu_tensor_free(out_row);
+    }
+    TEST_ASSERT(ds4_gpu_tensor_read(
+                    reference_low, 0, reference_low_host, low_bytes) != 0);
+    TEST_ASSERT(ds4_gpu_tensor_read(
+                    reference_out, 0, reference_out_host, out_bytes) != 0);
+
+    /* The API itself is fail-closed; the graph caller owns the row fallback. */
+    TEST_ASSERT(unsetenv(enable_env) == 0);
+    TEST_ASSERT(unsetenv(disable_env) == 0);
+    TEST_ASSERT(unsetenv(require_env) == 0);
+    TEST_ASSERT(ds4_gpu_attention_output_q4_K_batch_tensor(
+                    candidate_out,
+                    candidate_low,
+                    group_tmp,
+                    low_tmp,
+                    model_raw,
+                    model_bytes,
+                    0,
+                    out_b_offset,
+                    out_b_type,
+                    group_dim,
+                    rank,
+                    n_groups,
+                    out_dim,
+                    heads,
+                    2u) == 0);
+
+    TEST_ASSERT(setenv(enable_env, "1", 1) == 0);
+    for (uint32_t n_rows = 2u; n_rows <= max_rows; n_rows++) {
+        for (uint64_t i = 0; i < (uint64_t)alloc_rows * low_dim; i++) {
+            const uint32_t bits = 0x7fc10000u + (uint32_t)(i & 0xffffu);
+            memcpy(&candidate_low_host[i], &bits, sizeof(bits));
+        }
+        for (uint64_t i = 0; i < (uint64_t)alloc_rows * out_dim; i++) {
+            const uint32_t bits = 0x7fc20000u + (uint32_t)(i & 0xffffu);
+            memcpy(&candidate_out_host[i], &bits, sizeof(bits));
+        }
+        TEST_ASSERT(ds4_gpu_tensor_write(
+                        candidate_low, 0, candidate_low_host, low_bytes) != 0);
+        TEST_ASSERT(ds4_gpu_tensor_write(
+                        candidate_out, 0, candidate_out_host, out_bytes) != 0);
+        TEST_ASSERT(ds4_gpu_attention_output_q4_K_batch_tensor(
+                        candidate_out,
+                        candidate_low,
+                        group_tmp,
+                        low_tmp,
+                        model_raw,
+                        model_bytes,
+                        0,
+                        out_b_offset,
+                        out_b_type,
+                        group_dim,
+                        rank,
+                        n_groups,
+                        out_dim,
+                        heads,
+                        n_rows) == 1);
+        TEST_ASSERT(ds4_gpu_tensor_read(
+                        candidate_low, 0, candidate_low_host, low_bytes) != 0);
+        TEST_ASSERT(ds4_gpu_tensor_read(
+                        candidate_out, 0, candidate_out_host, out_bytes) != 0);
+
+        const size_t low_count = (size_t)n_rows * low_dim;
+        const size_t out_count = (size_t)n_rows * out_dim;
+        const test_float_compare_stats low_stats = test_compare_float_bits(
+            reference_low_host, candidate_low_host, low_count);
+        const test_float_compare_stats out_stats = test_compare_float_bits(
+            reference_out_host, candidate_out_host, out_count);
+        size_t low_oob_mismatch = 0;
+        size_t out_oob_mismatch = 0;
+        for (uint64_t i = (uint64_t)n_rows * low_dim;
+             i < (uint64_t)alloc_rows * low_dim;
+             i++) {
+            uint32_t bits = 0;
+            memcpy(&bits, &candidate_low_host[i], sizeof(bits));
+            if (bits != 0x7fc10000u + (uint32_t)(i & 0xffffu)) {
+                low_oob_mismatch++;
+            }
+        }
+        for (uint64_t i = (uint64_t)n_rows * out_dim;
+             i < (uint64_t)alloc_rows * out_dim;
+             i++) {
+            uint32_t bits = 0;
+            memcpy(&bits, &candidate_out_host[i], sizeof(bits));
+            if (bits != 0x7fc20000u + (uint32_t)(i & 0xffffu)) {
+                out_oob_mismatch++;
+            }
+        }
+        fprintf(stderr,
+                "ds4-test: Metal Q4 attention-output out_b=%s exact-%u "
+                "low=%zu/%zu out=%zu/%zu low_oob=%zu out_oob=%zu\n",
+                out_b_type == 8u ? "Q8_0" : "Q4_K",
+                n_rows,
+                low_stats.mismatch_count,
+                low_count,
+                out_stats.mismatch_count,
+                out_count,
+                low_oob_mismatch,
+                out_oob_mismatch);
+        TEST_ASSERT(low_stats.mismatch_count == 0 && low_stats.max_ulp == 0);
+        TEST_ASSERT(out_stats.mismatch_count == 0 && out_stats.max_ulp == 0);
+        TEST_ASSERT(low_oob_mismatch == 0);
+        TEST_ASSERT(out_oob_mismatch == 0);
+    }
+
+    /* REQUIRE is a fail-closed model-oracle gate and implies enable for the
+     * exact N=2..5 scope. */
+    TEST_ASSERT(unsetenv(enable_env) == 0);
+    TEST_ASSERT(setenv(require_env, "1", 1) == 0);
+    TEST_ASSERT(ds4_gpu_attention_output_q4_K_batch_tensor(
+                    candidate_out,
+                    candidate_low,
+                    group_tmp,
+                    low_tmp,
+                    model_raw,
+                    model_bytes,
+                    0,
+                    out_b_offset,
+                    out_b_type,
+                    group_dim,
+                    rank,
+                    n_groups,
+                    out_dim,
+                    heads,
+                    2u) == 1);
+
+    if (out_b_type == 12u) {
+        /* A required candidate must hard-fail rather than bypass an explicit
+         * request for the alternate Q4 schedule. */
+        TEST_ASSERT(unsetenv(disable_env) == 0);
+        TEST_ASSERT(setenv(disable_classic_env, "1", 1) == 0);
+        TEST_ASSERT(ds4_gpu_attention_output_q4_K_batch_tensor(
+                        candidate_out,
+                        candidate_low,
+                        group_tmp,
+                        low_tmp,
+                        model_raw,
+                        model_bytes,
+                        0,
+                        out_b_offset,
+                        out_b_type,
+                        group_dim,
+                        rank,
+                        n_groups,
+                        out_dim,
+                        heads,
+                        2u) == -1);
+        TEST_ASSERT(unsetenv(disable_classic_env) == 0);
+    }
+
+    /* The unconditional kill switch wins over REQUIRE and makes the
+     * model-oracle run fail instead of silently selecting the fallback. */
+    TEST_ASSERT(setenv(disable_env, "1", 1) == 0);
+    TEST_ASSERT(ds4_gpu_attention_output_q4_K_batch_tensor(
+                    candidate_out,
+                    candidate_low,
+                    group_tmp,
+                    low_tmp,
+                    model_raw,
+                    model_bytes,
+                    0,
+                    out_b_offset,
+                    out_b_type,
+                    group_dim,
+                    rank,
+                    n_groups,
+                    out_dim,
+                    heads,
+                    2u) == -1);
+
+    /* Without REQUIRE the same kill switch retains the ordinary fallback. */
+    TEST_ASSERT(unsetenv(require_env) == 0);
+    TEST_ASSERT(setenv(enable_env, "1", 1) == 0);
+    TEST_ASSERT(ds4_gpu_attention_output_q4_K_batch_tensor(
+                    candidate_out,
+                    candidate_low,
+                    group_tmp,
+                    low_tmp,
+                    model_raw,
+                    model_bytes,
+                    0,
+                    out_b_offset,
+                    out_b_type,
+                    group_dim,
+                    rank,
+                    n_groups,
+                    out_dim,
+                    heads,
+                    2u) == 0);
+
+cleanup:
+    ds4_gpu_tensor_free(low_tmp);
+    ds4_gpu_tensor_free(group_tmp);
+    ds4_gpu_tensor_free(candidate_out);
+    ds4_gpu_tensor_free(candidate_low);
+    ds4_gpu_tensor_free(reference_out);
+    ds4_gpu_tensor_free(reference_low);
+    ds4_gpu_tensor_free(heads);
+    free(candidate_out_host);
+    free(candidate_low_host);
+    free(reference_out_host);
+    free(reference_low_host);
+    free(heads_host);
+    free(model_raw);
+    test_restore_env(disable_classic_env, saved_disable_classic);
+    test_restore_env(require_env, saved_require);
+    test_restore_env(disable_env, saved_disable);
+    test_restore_env(enable_env, saved_enable);
+}
+
+static void test_metal_q4_attention_output_tiny_batch_exact(void) {
+    test_metal_q4_attention_output_tiny_batch_exact_case(8u);
+    test_metal_q4_attention_output_tiny_batch_exact_case(12u);
+}
+
+static void test_metal_dspark_device_proposer_q8(void) {
+    /* Keep this fixture small while exercising the complete public contract:
+     * six on-device draft steps, a confidence stop after a verified prefix,
+     * and stable lowest-token tie breaking in the Markov argmax. */
+    _Static_assert(sizeof(ds4_gpu_dspark_device_proposal) == 64u,
+                   "DSpark device proposal ABI must stay 64 bytes");
+    _Static_assert(DS4_GPU_DSPARK_MAX_DRAFTS == 6u,
+                   "DSpark device proposer test covers the six-token limit");
+    _Static_assert(offsetof(ds4_gpu_dspark_device_proposal, tokens) == 0u &&
+                   offsetof(ds4_gpu_dspark_device_proposal,
+                            confidence_logits) == 24u &&
+                   offsetof(ds4_gpu_dspark_device_proposal,
+                            proposal_len) == 48u &&
+                   offsetof(ds4_gpu_dspark_device_proposal,
+                            confidence_len) == 52u &&
+                   offsetof(ds4_gpu_dspark_device_proposal, status) == 56u &&
+                   offsetof(ds4_gpu_dspark_device_proposal, reserved) == 60u,
+                   "DSpark device proposal field offsets changed");
+
+    const uint32_t vocab = 64u;
+    const uint32_t rank = 32u;
+    const uint32_t hidden_dim = 32u;
+    const uint32_t max_drafts = DS4_GPU_DSPARK_MAX_DRAFTS;
+    const uint32_t first_prev_token = 17u;
+    const uint32_t tied_token_lo = 3u;
+    const uint32_t tied_token_hi = 11u;
+    const float reused_confidence = 8.0f;
+    const uint64_t page = (uint64_t)getpagesize();
+    const uint64_t markov_row_bytes = (uint64_t)(rank / 32u) * 34u;
+    const uint64_t markov_bytes = (uint64_t)vocab * markov_row_bytes;
+    const uint64_t confidence_bytes =
+        (uint64_t)((hidden_dim + rank) / 32u) * 34u;
+    const uint64_t w1_offset = 0u;
+    const uint64_t w2_offset = test_round_up_u64(markov_bytes, page);
+    const uint64_t confidence_offset =
+        test_round_up_u64(w2_offset + markov_bytes, page);
+    const uint64_t model_bytes = test_round_up_u64(
+        confidence_offset + confidence_bytes, page);
+    const uint64_t logits_bytes =
+        (uint64_t)max_drafts * vocab * sizeof(float);
+    const uint64_t hidden_bytes =
+        (uint64_t)max_drafts * hidden_dim * sizeof(float);
+
+    void *model_raw = NULL;
+    TEST_ASSERT(posix_memalign(&model_raw,
+                               (size_t)page,
+                               (size_t)model_bytes) == 0);
+    if (!model_raw) return;
+    /* Zero is a valid Q8_0 block (zero half scale and zero quants).  It makes
+     * every Markov correction and computed confidence exactly zero while the
+     * kernels still traverse all Q8 blocks. */
+    memset(model_raw, 0, (size_t)model_bytes);
+
+    ds4_gpu_tensor *logits = ds4_gpu_tensor_alloc(logits_bytes);
+    ds4_gpu_tensor *hidden = ds4_gpu_tensor_alloc(hidden_bytes);
+    ds4_gpu_tensor *result =
+        ds4_gpu_tensor_alloc(DS4_GPU_DSPARK_DEVICE_PROPOSAL_BYTES);
+    TEST_ASSERT(logits != NULL);
+    TEST_ASSERT(hidden != NULL);
+    TEST_ASSERT(result != NULL);
+    if (!logits || !hidden || !result) {
+        ds4_gpu_tensor_free(logits);
+        ds4_gpu_tensor_free(hidden);
+        ds4_gpu_tensor_free(result);
+        free(model_raw);
+        return;
+    }
+
+    float *logits_host = malloc((size_t)logits_bytes);
+    float *hidden_host = malloc((size_t)hidden_bytes);
+    TEST_ASSERT(logits_host != NULL);
+    TEST_ASSERT(hidden_host != NULL);
+    if (!logits_host || !hidden_host) {
+        free(logits_host);
+        free(hidden_host);
+        ds4_gpu_tensor_free(logits);
+        ds4_gpu_tensor_free(hidden);
+        ds4_gpu_tensor_free(result);
+        free(model_raw);
+        return;
+    }
+
+    for (uint32_t draft = 0u; draft < max_drafts; draft++) {
+        for (uint32_t token = 0u; token < vocab; token++) {
+            logits_host[(uint64_t)draft * vocab + token] =
+                -4.0f - (float)(token % 7u) * 0.03125f;
+        }
+        logits_host[(uint64_t)draft * vocab + tied_token_lo] = 5.0f;
+        logits_host[(uint64_t)draft * vocab + tied_token_hi] = 5.0f;
+        for (uint32_t i = 0u; i < hidden_dim; i++) {
+            hidden_host[(uint64_t)draft * hidden_dim + i] =
+                (float)((int)((draft * 19u + i * 13u) % 37u) - 18) / 23.0f;
+        }
+    }
+
+    TEST_ASSERT(ds4_gpu_tensor_write(
+                    logits, 0, logits_host, logits_bytes) != 0);
+    TEST_ASSERT(ds4_gpu_tensor_write(
+                    hidden, 0, hidden_host, hidden_bytes) != 0);
+    TEST_ASSERT(ds4_gpu_set_model_map(model_raw, model_bytes) != 0);
+    ds4_gpu_set_quality(false);
+
+    char *saved_enable =
+        test_save_env("DS4_METAL_DSPARK_DEVICE_PROPOSER");
+    char *saved_kill =
+        test_save_env("DS4_METAL_DSPARK_NO_DEVICE_PROPOSER");
+    setenv("DS4_METAL_DSPARK_DEVICE_PROPOSER", "1", 1);
+    unsetenv("DS4_METAL_DSPARK_NO_DEVICE_PROPOSER");
+
+    ds4_gpu_dspark_device_proposal public_result;
+    bool full_six_ok = false;
+    for (uint32_t n_drafts = 1u; n_drafts <= max_drafts; n_drafts++) {
+        memset(&public_result, 0xa5, sizeof(public_result));
+        TEST_ASSERT(ds4_gpu_tensor_write(result,
+                                          0,
+                                          &public_result,
+                                          sizeof(public_result)) != 0);
+        const int ok = ds4_gpu_dspark_markov_confidence_q8_tensor(
+                result,
+                logits,
+                hidden,
+                model_raw,
+                model_bytes,
+                w1_offset,
+                w2_offset,
+                confidence_offset,
+                first_prev_token,
+                vocab,
+                rank,
+                hidden_dim,
+                n_drafts,
+                0.25f,
+                1,
+                reused_confidence);
+        TEST_ASSERT(ok != 0);
+        if (!ok) break;
+        TEST_ASSERT(ds4_gpu_tensor_read(result,
+                                         0,
+                                         &public_result,
+                                         sizeof(public_result)) != 0);
+        TEST_ASSERT(public_result.status == 1u);
+        TEST_ASSERT(public_result.reserved == 0u);
+        TEST_ASSERT(public_result.proposal_len == n_drafts);
+        TEST_ASSERT(public_result.confidence_len == n_drafts);
+        for (uint32_t i = 0u; i < max_drafts; i++) {
+            if (i < n_drafts) {
+                TEST_ASSERT(public_result.tokens[i] ==
+                            (int32_t)tied_token_lo);
+                TEST_ASSERT(public_result.confidence_logits[i] ==
+                            (i == 0u ? reused_confidence : 0.0f));
+            } else {
+                TEST_ASSERT(public_result.tokens[i] == -1);
+                TEST_ASSERT(public_result.confidence_logits[i] == 0.0f);
+            }
+        }
+        if (n_drafts == max_drafts) full_six_ok = true;
+    }
+    TEST_ASSERT(full_six_ok);
+
+    /* Draft zero passes via the supplied confidence.  Draft one computes a
+     * zero logit (p=0.5), fails the 0.75 threshold, and therefore leaves a
+     * one-token proposal with two evaluated confidence rows. */
+    memset(&public_result, 0xa5, sizeof(public_result));
+    TEST_ASSERT(ds4_gpu_tensor_write(result,
+                                      0,
+                                      &public_result,
+                                      sizeof(public_result)) != 0);
+    const int stop_ok = ds4_gpu_dspark_markov_confidence_q8_tensor(
+            result,
+            logits,
+            hidden,
+            model_raw,
+            model_bytes,
+            w1_offset,
+            w2_offset,
+            confidence_offset,
+            first_prev_token,
+            vocab,
+            rank,
+            hidden_dim,
+            max_drafts,
+            0.75f,
+            1,
+            reused_confidence);
+    TEST_ASSERT(stop_ok != 0);
+    if (stop_ok) {
+        TEST_ASSERT(ds4_gpu_tensor_read(result,
+                                         0,
+                                         &public_result,
+                                         sizeof(public_result)) != 0);
+        TEST_ASSERT(public_result.status == 1u);
+        TEST_ASSERT(public_result.reserved == 0u);
+        TEST_ASSERT(public_result.proposal_len == 1u);
+        TEST_ASSERT(public_result.confidence_len == 2u);
+        TEST_ASSERT(public_result.tokens[0] == (int32_t)tied_token_lo);
+        TEST_ASSERT(public_result.confidence_logits[0] == reused_confidence);
+        TEST_ASSERT(public_result.confidence_logits[1] == 0.0f);
+        for (uint32_t i = 1u; i < max_drafts; i++) {
+            TEST_ASSERT(public_result.tokens[i] == -1);
+            if (i > 1u) {
+                TEST_ASSERT(public_result.confidence_logits[i] == 0.0f);
+            }
+        }
+    }
+
+    setenv("DS4_METAL_DSPARK_NO_DEVICE_PROPOSER", "1", 1);
+    TEST_ASSERT(ds4_gpu_dspark_markov_confidence_q8_tensor(
+                    result,
+                    logits,
+                    hidden,
+                    model_raw,
+                    model_bytes,
+                    w1_offset,
+                    w2_offset,
+                    confidence_offset,
+                    first_prev_token,
+                    vocab,
+                    rank,
+                    hidden_dim,
+                    1u,
+                    0.25f,
+                    1,
+                    reused_confidence) == 0);
+    test_restore_env("DS4_METAL_DSPARK_NO_DEVICE_PROPOSER", saved_kill);
+    test_restore_env("DS4_METAL_DSPARK_DEVICE_PROPOSER", saved_enable);
+
+    fprintf(stderr,
+            "ds4-test: Metal DSpark device proposer full6=%d stop_prefix=%u "
+            "confidence_rows=%u tie=%u<%u\n",
+            full_six_ok ? 1 : 0,
+            stop_ok ? public_result.proposal_len : 0u,
+            stop_ok ? public_result.confidence_len : 0u,
+            tied_token_lo,
+            tied_token_hi);
+
+    free(logits_host);
+    free(hidden_host);
+    ds4_gpu_tensor_free(logits);
+    ds4_gpu_tensor_free(hidden);
+    ds4_gpu_tensor_free(result);
+    free(model_raw);
+}
+
 static void test_metal_f16_compressor_pair_state_store_exact_case(
         uint32_t width,
         uint32_t ratio,
@@ -1303,93 +2546,756 @@ static void test_metal_f16_compressor_pair_state_store_exact_case(
                         ref_state_score, 0, ref_state_score_host,
                         state_bytes) != 0);
         TEST_ASSERT(ds4_gpu_tensor_read(
-                        fused_state_kv, 0, fused_state_kv_host,
-                        state_bytes) != 0);
+                        fused_state_kv, 0, fused_state_kv_host,
+                        state_bytes) != 0);
+        TEST_ASSERT(ds4_gpu_tensor_read(
+                        fused_state_score, 0, fused_state_score_host,
+                        state_bytes) != 0);
+        TEST_ASSERT(ds4_gpu_tensor_read(
+                        ref_comp, 0, ref_comp_host, comp_bytes) != 0);
+        TEST_ASSERT(ds4_gpu_tensor_read(
+                        fused_comp, 0, fused_comp_host, comp_bytes) != 0);
+
+        kv_stats = test_compare_float_bits(
+            ref_kv_host, fused_kv_host, width);
+        score_stats = test_compare_float_bits(
+            ref_score_host, fused_score_host, width);
+        state_kv_stats = test_compare_float_bits(
+            ref_state_kv_host, fused_state_kv_host, (size_t)state_count);
+        state_score_stats = test_compare_float_bits(
+            ref_state_score_host, fused_state_score_host,
+            (size_t)state_count);
+        comp_stats = test_compare_float_bits(
+            ref_comp_host, fused_comp_host, head_dim);
+    }
+
+    test_restore_env(pair_disable_env, saved_pair_disable);
+    test_restore_env(store_disable_env, saved_store_disable);
+    test_restore_env(decode_pack_disable_env, saved_decode_pack_disable);
+    test_restore_env(exact_reduction_disable_env,
+                     saved_exact_reduction_disable);
+    test_restore_env(
+        exact_reduction_poison_env, saved_exact_reduction_poison);
+
+    fprintf(stderr,
+            "ds4-test: compressor pair state-store exact width=%u ratio=%u "
+            "pos=%u emit=%u ape=%s decode_pack=%u exact_reduce=%u "
+            "proj=%zu/%zu state=%zu/%zu "
+            "comp=%zu max_ulp=%u/%u/%u/%u/%u\n",
+            width, ratio, pos, emit ? 1u : 0u,
+            ape_type == 1u ? "f16" : "f32",
+            test_decode_pack ? 1u : 0u,
+            test_decode_pack ? 1u : 0u,
+            kv_stats.mismatch_count, score_stats.mismatch_count,
+            state_kv_stats.mismatch_count,
+            state_score_stats.mismatch_count,
+            comp_stats.mismatch_count,
+            kv_stats.max_ulp, score_stats.max_ulp,
+            state_kv_stats.max_ulp, state_score_stats.max_ulp,
+            comp_stats.max_ulp);
+    TEST_ASSERT(kv_stats.mismatch_count == 0);
+    TEST_ASSERT(score_stats.mismatch_count == 0);
+    TEST_ASSERT(state_kv_stats.mismatch_count == 0);
+    TEST_ASSERT(state_score_stats.mismatch_count == 0);
+    TEST_ASSERT(comp_stats.mismatch_count == 0);
+
+    free(fused_comp_host);
+    free(ref_comp_host);
+    free(fused_state_score_host);
+    free(fused_state_kv_host);
+    free(ref_state_score_host);
+    free(ref_state_kv_host);
+    free(fused_score_host);
+    free(fused_kv_host);
+    free(ref_score_host);
+    free(ref_kv_host);
+    free(x_host);
+    ds4_gpu_tensor_free(fused_comp);
+    ds4_gpu_tensor_free(ref_comp);
+    ds4_gpu_tensor_free(fused_state_score);
+    ds4_gpu_tensor_free(fused_state_kv);
+    ds4_gpu_tensor_free(ref_state_score);
+    ds4_gpu_tensor_free(ref_state_kv);
+    ds4_gpu_tensor_free(fused_score);
+    ds4_gpu_tensor_free(fused_kv);
+    ds4_gpu_tensor_free(ref_score);
+    ds4_gpu_tensor_free(ref_kv);
+    ds4_gpu_tensor_free(x);
+    free(model_raw);
+}
+
+static void test_metal_f16_compressor_pair_state_store_exact(void) {
+    test_metal_f16_compressor_pair_state_store_exact_case(
+        256, 4, 8, 0, 17, false);
+    test_metal_f16_compressor_pair_state_store_exact_case(
+        256, 4, 11, 1, 23, true);
+    test_metal_f16_compressor_pair_state_store_exact_case(
+        1024, 4, 11, 1, 29, true);
+    test_metal_f16_compressor_pair_state_store_exact_case(
+        512, 128, 255, 1, 43, false);
+}
+
+static void test_metal_f16_compressor_quad_state_store_exact_case(
+        uint32_t width0,
+        uint32_t pos,
+        uint32_t ape0_type,
+        uint32_t ape1_type,
+        uint32_t seed) {
+    const uint32_t in_dim = 4096u;
+    const uint32_t ratio = 4u;
+    const uint32_t width1 = 256u;
+    const uint32_t state_rows = 2u * ratio;
+    const uint32_t widths[4] = {width0, width0, width1, width1};
+    const uint32_t ape_types[2] = {ape0_type, ape1_type};
+    const uint64_t page = (uint64_t)getpagesize();
+    const uint64_t weight_bytes[4] = {
+        (uint64_t)width0 * in_dim * sizeof(uint16_t),
+        (uint64_t)width0 * in_dim * sizeof(uint16_t),
+        (uint64_t)width1 * in_dim * sizeof(uint16_t),
+        (uint64_t)width1 * in_dim * sizeof(uint16_t),
+    };
+    uint64_t weight_offsets[4] = {0};
+    for (uint32_t i = 1; i < 4u; i++) {
+        weight_offsets[i] = test_round_up_u64(
+            weight_offsets[i - 1u] + weight_bytes[i - 1u], page);
+    }
+    const uint64_t ape0_offset = test_round_up_u64(
+        weight_offsets[3] + weight_bytes[3], page);
+    const uint64_t ape0_bytes = (uint64_t)ratio * width0 *
+        (ape0_type == 1u ? sizeof(uint16_t) : sizeof(float));
+    const uint64_t ape1_offset = test_round_up_u64(
+        ape0_offset + ape0_bytes, page);
+    const uint64_t ape1_bytes = (uint64_t)ratio * width1 *
+        (ape1_type == 1u ? sizeof(uint16_t) : sizeof(float));
+    const uint64_t ape_offsets[2] = {ape0_offset, ape1_offset};
+    const uint32_t ape_widths[2] = {width0, width1};
+    const uint64_t model_bytes = test_round_up_u64(
+        ape1_offset + ape1_bytes, page);
+    const uint64_t x_bytes = (uint64_t)in_dim * sizeof(float);
+    const uint64_t max_state_count = (uint64_t)state_rows * width0;
+
+    void *model_raw = NULL;
+    TEST_ASSERT(posix_memalign(
+                    &model_raw, (size_t)page, (size_t)model_bytes) == 0);
+    ds4_gpu_tensor *x = ds4_gpu_tensor_alloc(x_bytes);
+    ds4_gpu_tensor *ref_out[4] = {0};
+    ds4_gpu_tensor *quad_out[4] = {0};
+    ds4_gpu_tensor *ref_state[4] = {0};
+    ds4_gpu_tensor *quad_state[4] = {0};
+    bool allocated = model_raw != NULL && x != NULL;
+    for (uint32_t i = 0; i < 4u; i++) {
+        const uint64_t out_bytes =
+            (uint64_t)widths[i] * sizeof(float);
+        const uint64_t state_bytes =
+            (uint64_t)state_rows * widths[i] * sizeof(float);
+        ref_out[i] = ds4_gpu_tensor_alloc(out_bytes);
+        quad_out[i] = ds4_gpu_tensor_alloc(out_bytes);
+        ref_state[i] = ds4_gpu_tensor_alloc(state_bytes);
+        quad_state[i] = ds4_gpu_tensor_alloc(state_bytes);
+        TEST_ASSERT(ref_out[i] != NULL);
+        TEST_ASSERT(quad_out[i] != NULL);
+        TEST_ASSERT(ref_state[i] != NULL);
+        TEST_ASSERT(quad_state[i] != NULL);
+        allocated = allocated && ref_out[i] && quad_out[i] &&
+                    ref_state[i] && quad_state[i];
+    }
+
+    float *x_host = malloc((size_t)x_bytes);
+    float *ref_host = malloc((size_t)max_state_count * sizeof(float));
+    float *quad_host = malloc((size_t)max_state_count * sizeof(float));
+    TEST_ASSERT(x_host != NULL);
+    TEST_ASSERT(ref_host != NULL);
+    TEST_ASSERT(quad_host != NULL);
+    allocated = allocated && x_host && ref_host && quad_host;
+
+    const char *force_pair_env =
+        "DS4_METAL_ENABLE_COMPRESSOR_PAIR_STATE_STORE";
+    const char *disable_pair_state_env =
+        "DS4_METAL_DISABLE_M3_COMPRESSOR_PAIR_STATE_STORE";
+    const char *disable_pair_proj_env =
+        "DS4_METAL_DISABLE_COMPRESSOR_PAIR_PROJ";
+    const char *disable_store_env =
+        "DS4_METAL_DISABLE_COMPRESSOR_STORE_ONE";
+    char *saved_force_pair = test_save_env(force_pair_env);
+    char *saved_disable_pair_state = test_save_env(disable_pair_state_env);
+    char *saved_disable_pair_proj = test_save_env(disable_pair_proj_env);
+    char *saved_disable_store = test_save_env(disable_store_env);
+
+    test_float_compare_stats out_stats[4] = {{0}};
+    test_float_compare_stats state_stats[4] = {{0}};
+    if (allocated) {
+        memset(model_raw, 0, (size_t)model_bytes);
+        for (uint32_t matrix = 0; matrix < 4u; matrix++) {
+            uint16_t *weights = (uint16_t *)(
+                (uint8_t *)model_raw + weight_offsets[matrix]);
+            for (uint32_t o = 0; o < widths[matrix]; o++) {
+                for (uint32_t i = 0; i < in_dim; i++) {
+                    const int value =
+                        (int)((o * (17u + 2u * matrix) +
+                               i * (23u + 4u * matrix) +
+                               (o ^ (i >> (matrix & 3u))) *
+                                   (3u + 2u * matrix) +
+                               seed * (29u + matrix)) % 127u) - 63;
+                    weights[(uint64_t)o * in_dim + i] =
+                        test_float_to_f16(
+                            (float)value / (96.0f + 8.0f * matrix));
+                }
+            }
+        }
+
+        for (uint32_t which = 0; which < 2u; which++) {
+            const uint64_t count =
+                (uint64_t)ratio * ape_widths[which];
+            if (ape_types[which] == 1u) {
+                uint16_t *ape = (uint16_t *)(
+                    (uint8_t *)model_raw + ape_offsets[which]);
+                for (uint64_t i = 0; i < count; i++) {
+                    const int value =
+                        (int)((i * (13u + 2u * which) +
+                               (i ^ (i >> 3u)) * (7u + 2u * which) +
+                               seed * (17u + which)) % 61u) - 30;
+                    ape[i] = test_float_to_f16(
+                        (float)value / (80.0f + 8.0f * which));
+                }
+            } else {
+                float *ape = (float *)(
+                    (uint8_t *)model_raw + ape_offsets[which]);
+                for (uint64_t i = 0; i < count; i++) {
+                    const int value =
+                        (int)((i * (13u + 2u * which) +
+                               (i ^ (i >> 3u)) * (7u + 2u * which) +
+                               seed * (17u + which)) % 61u) - 30;
+                    ape[i] =
+                        (float)value / (80.0f + 8.0f * which);
+                }
+            }
+        }
+
+        for (uint32_t i = 0; i < in_dim; i++) {
+            const int value =
+                (int)((i * 29u + (i ^ (i >> 4u)) * 9u +
+                       seed * 11u) % 127u) - 63;
+            x_host[i] = (float)value / 88.0f;
+        }
+        TEST_ASSERT(ds4_gpu_tensor_write(x, 0, x_host, x_bytes) != 0);
+
+        for (uint32_t buffer = 0; buffer < 4u; buffer++) {
+            const uint64_t out_bytes =
+                (uint64_t)widths[buffer] * sizeof(float);
+            const uint64_t state_count =
+                (uint64_t)state_rows * widths[buffer];
+            const uint64_t state_bytes = state_count * sizeof(float);
+            for (uint32_t i = 0; i < widths[buffer]; i++) {
+                const uint32_t poison =
+                    0x7fc00001u + ((buffer * 1024u + i) & 0x3fffu);
+                memcpy(ref_host + i, &poison, sizeof(poison));
+            }
+            TEST_ASSERT(ds4_gpu_tensor_write(
+                            ref_out[buffer], 0, ref_host,
+                            out_bytes) != 0);
+            TEST_ASSERT(ds4_gpu_tensor_write(
+                            quad_out[buffer], 0, ref_host,
+                            out_bytes) != 0);
+
+            for (uint64_t i = 0; i < state_count; i++) {
+                const int value =
+                    (int)((i * (5u + 2u * buffer) +
+                           seed * (13u + buffer)) % 193u) - 96;
+                ref_host[i] =
+                    (float)value / (64.0f + 8.0f * buffer);
+            }
+            TEST_ASSERT(ds4_gpu_tensor_write(
+                            ref_state[buffer], 0, ref_host,
+                            state_bytes) != 0);
+            TEST_ASSERT(ds4_gpu_tensor_write(
+                            quad_state[buffer], 0, ref_host,
+                            state_bytes) != 0);
+        }
+
+        TEST_ASSERT(ds4_gpu_set_model_map(model_raw, model_bytes) != 0);
+        ds4_gpu_set_quality(false);
+        TEST_ASSERT(setenv(force_pair_env, "1", 1) == 0);
+        TEST_ASSERT(unsetenv(disable_pair_state_env) == 0);
+        TEST_ASSERT(unsetenv(disable_pair_proj_env) == 0);
+        TEST_ASSERT(unsetenv(disable_store_env) == 0);
+
+        TEST_ASSERT(ds4_gpu_matmul_f16_pair_compressor_store_tensor(
+                        ref_out[0], ref_out[1],
+                        ref_state[0], ref_state[1],
+                        model_raw, model_bytes,
+                        weight_offsets[0], weight_offsets[1],
+                        ape_offsets[0], ape_types[0],
+                        in_dim, width0, x, ratio, pos) == 1);
+        TEST_ASSERT(ds4_gpu_matmul_f16_pair_compressor_store_tensor(
+                        ref_out[2], ref_out[3],
+                        ref_state[2], ref_state[3],
+                        model_raw, model_bytes,
+                        weight_offsets[2], weight_offsets[3],
+                        ape_offsets[1], ape_types[1],
+                        in_dim, width1, x, ratio, pos) == 1);
+        TEST_ASSERT(ds4_gpu_matmul_f16_quad_compressor_store_tensor(
+                        quad_out[0], quad_out[1],
+                        quad_out[2], quad_out[3],
+                        quad_state[0], quad_state[1],
+                        quad_state[2], quad_state[3],
+                        model_raw, model_bytes,
+                        weight_offsets[0], weight_offsets[1],
+                        weight_offsets[2], weight_offsets[3],
+                        ape_offsets[0], ape_types[0],
+                        ape_offsets[1], ape_types[1],
+                        in_dim, width0, width1, x, ratio, pos) == 1);
+
+        for (uint32_t buffer = 0; buffer < 4u; buffer++) {
+            const uint64_t out_bytes =
+                (uint64_t)widths[buffer] * sizeof(float);
+            const uint64_t state_count =
+                (uint64_t)state_rows * widths[buffer];
+            const uint64_t state_bytes = state_count * sizeof(float);
+            TEST_ASSERT(ds4_gpu_tensor_read(
+                            ref_out[buffer], 0, ref_host,
+                            out_bytes) != 0);
+            TEST_ASSERT(ds4_gpu_tensor_read(
+                            quad_out[buffer], 0, quad_host,
+                            out_bytes) != 0);
+            out_stats[buffer] = test_compare_float_bits(
+                ref_host, quad_host, widths[buffer]);
+            TEST_ASSERT(ds4_gpu_tensor_read(
+                            ref_state[buffer], 0, ref_host,
+                            state_bytes) != 0);
+            TEST_ASSERT(ds4_gpu_tensor_read(
+                            quad_state[buffer], 0, quad_host,
+                            state_bytes) != 0);
+            state_stats[buffer] = test_compare_float_bits(
+                ref_host, quad_host, (size_t)state_count);
+        }
+    }
+
+    test_restore_env(force_pair_env, saved_force_pair);
+    test_restore_env(disable_pair_state_env, saved_disable_pair_state);
+    test_restore_env(disable_pair_proj_env, saved_disable_pair_proj);
+    test_restore_env(disable_store_env, saved_disable_store);
+
+    size_t out_mismatches = 0;
+    size_t state_mismatches = 0;
+    uint32_t max_out_ulp = 0;
+    uint32_t max_state_ulp = 0;
+    for (uint32_t i = 0; i < 4u; i++) {
+        out_mismatches += out_stats[i].mismatch_count;
+        state_mismatches += state_stats[i].mismatch_count;
+        if (out_stats[i].max_ulp > max_out_ulp) {
+            max_out_ulp = out_stats[i].max_ulp;
+        }
+        if (state_stats[i].max_ulp > max_state_ulp) {
+            max_state_ulp = state_stats[i].max_ulp;
+        }
+        TEST_ASSERT(out_stats[i].mismatch_count == 0);
+        TEST_ASSERT(state_stats[i].mismatch_count == 0);
+    }
+    fprintf(stderr,
+            "ds4-test: compressor quad state-store exact width=%u/256 "
+            "pos_mod4=%u ape=%s/%s outputs=%zu states=%zu "
+            "max_ulp=%u/%u\n",
+            width0, pos % ratio,
+            ape0_type == 1u ? "f16" : "f32",
+            ape1_type == 1u ? "f16" : "f32",
+            out_mismatches, state_mismatches,
+            max_out_ulp, max_state_ulp);
+
+    free(quad_host);
+    free(ref_host);
+    free(x_host);
+    for (uint32_t i = 0; i < 4u; i++) {
+        ds4_gpu_tensor_free(quad_state[i]);
+        ds4_gpu_tensor_free(ref_state[i]);
+        ds4_gpu_tensor_free(quad_out[i]);
+        ds4_gpu_tensor_free(ref_out[i]);
+    }
+    ds4_gpu_tensor_free(x);
+    free(model_raw);
+}
+
+static void test_metal_f16_compressor_quad_state_store_exact(void) {
+    test_metal_f16_compressor_quad_state_store_exact_case(
+        1024u, 8u, 1u, 0u, 59u);
+    test_metal_f16_compressor_quad_state_store_exact_case(
+        512u, 11u, 0u, 1u, 71u);
+}
+
+static void test_metal_q8_qkv_compressor_compound_exact_case(
+        uint32_t ratio,
+        uint32_t pos,
+        uint32_t ape0_type,
+        uint32_t ape1_type,
+        uint32_t seed,
+        bool     batched_commands) {
+    enum {
+        Q_A_WEIGHT = 0,
+        KV_WEIGHT,
+        COMP0_KV_WEIGHT,
+        COMP0_SCORE_WEIGHT,
+        COMP1_KV_WEIGHT,
+        COMP1_SCORE_WEIGHT,
+        COMP0_APE,
+        COMP1_APE,
+        MODEL_RANGE_COUNT,
+    };
+    const uint32_t in_dim = 4096u;
+    const uint32_t q_rank = 77u;
+    const uint32_t kv_dim = 19u;
+    const uint32_t width0 = ratio == 4u ? 1024u : 512u;
+    const uint32_t width1 = ratio == 4u ? 256u : 0u;
+    const uint32_t state_rows = ratio == 4u ? 2u * ratio : ratio;
+    const uint32_t n_comp_outputs = width1 != 0u ? 4u : 2u;
+    const uint32_t comp_widths[4] = {
+        width0, width0, width1, width1,
+    };
+    const uint64_t page = (uint64_t)getpagesize();
+    const uint64_t q8_row_bytes = (uint64_t)(in_dim / 32u) * 34u;
+    const uint64_t f16_row_bytes =
+        (uint64_t)in_dim * sizeof(uint16_t);
+    uint64_t range_bytes[MODEL_RANGE_COUNT] = {
+        (uint64_t)q_rank * q8_row_bytes,
+        (uint64_t)kv_dim * q8_row_bytes,
+        (uint64_t)width0 * f16_row_bytes,
+        (uint64_t)width0 * f16_row_bytes,
+        (uint64_t)width1 * f16_row_bytes,
+        (uint64_t)width1 * f16_row_bytes,
+        (uint64_t)ratio * width0 *
+            (ape0_type == 1u ? sizeof(uint16_t) : sizeof(float)),
+        (uint64_t)ratio * width1 *
+            (ape1_type == 1u ? sizeof(uint16_t) : sizeof(float)),
+    };
+    uint64_t range_offsets[MODEL_RANGE_COUNT] = {0};
+    uint64_t cursor = 0;
+    for (uint32_t i = 0; i < MODEL_RANGE_COUNT; i++) {
+        if (range_bytes[i] == 0u) continue;
+        range_offsets[i] = test_round_up_u64(cursor, page);
+        cursor = range_offsets[i] + range_bytes[i];
+    }
+    const uint64_t model_bytes = test_round_up_u64(cursor, page);
+    const uint64_t x_bytes = (uint64_t)in_dim * sizeof(float);
+
+    void *model_raw = NULL;
+    TEST_ASSERT(posix_memalign(
+                    &model_raw, (size_t)page, (size_t)model_bytes) == 0);
+    ds4_gpu_tensor *x = ds4_gpu_tensor_alloc(x_bytes);
+    ds4_gpu_tensor *ref_q =
+        ds4_gpu_tensor_alloc((uint64_t)q_rank * sizeof(float));
+    ds4_gpu_tensor *ref_kv =
+        ds4_gpu_tensor_alloc((uint64_t)kv_dim * sizeof(float));
+    ds4_gpu_tensor *fused_q =
+        ds4_gpu_tensor_alloc((uint64_t)q_rank * sizeof(float));
+    ds4_gpu_tensor *fused_kv =
+        ds4_gpu_tensor_alloc((uint64_t)kv_dim * sizeof(float));
+    ds4_gpu_tensor *ref_out[4] = {0};
+    ds4_gpu_tensor *fused_out[4] = {0};
+    ds4_gpu_tensor *ref_state[4] = {0};
+    ds4_gpu_tensor *fused_state[4] = {0};
+    bool allocated = model_raw && x && ref_q && ref_kv && fused_q && fused_kv;
+    for (uint32_t i = 0; i < n_comp_outputs; i++) {
+        const uint64_t out_bytes =
+            (uint64_t)comp_widths[i] * sizeof(float);
+        const uint64_t state_bytes =
+            (uint64_t)state_rows * comp_widths[i] * sizeof(float);
+        ref_out[i] = ds4_gpu_tensor_alloc(out_bytes);
+        fused_out[i] = ds4_gpu_tensor_alloc(out_bytes);
+        ref_state[i] = ds4_gpu_tensor_alloc(state_bytes);
+        fused_state[i] = ds4_gpu_tensor_alloc(state_bytes);
+        TEST_ASSERT(ref_out[i] && fused_out[i] &&
+                    ref_state[i] && fused_state[i]);
+        allocated = allocated && ref_out[i] && fused_out[i] &&
+                    ref_state[i] && fused_state[i];
+    }
+
+    uint64_t host_count = (uint64_t)state_rows * width0;
+    if (host_count < q_rank) host_count = q_rank;
+    if (host_count < kv_dim) host_count = kv_dim;
+    float *x_host = malloc((size_t)x_bytes);
+    float *ref_host = malloc((size_t)host_count * sizeof(float));
+    float *fused_host = malloc((size_t)host_count * sizeof(float));
+    TEST_ASSERT(x_host && ref_host && fused_host);
+    allocated = allocated && x_host && ref_host && fused_host;
+
+    const char *force_pair_env =
+        "DS4_METAL_ENABLE_COMPRESSOR_PAIR_STATE_STORE";
+    const char *disable_pair_state_env =
+        "DS4_METAL_DISABLE_M3_COMPRESSOR_PAIR_STATE_STORE";
+    const char *disable_pair_proj_env =
+        "DS4_METAL_DISABLE_COMPRESSOR_PAIR_PROJ";
+    const char *disable_store_env =
+        "DS4_METAL_DISABLE_COMPRESSOR_STORE_ONE";
+    const char *q8_nsg_env = "DS4_METAL_Q8_MV_NSG";
+    char *saved_force_pair = test_save_env(force_pair_env);
+    char *saved_disable_pair_state = test_save_env(disable_pair_state_env);
+    char *saved_disable_pair_proj = test_save_env(disable_pair_proj_env);
+    char *saved_disable_store = test_save_env(disable_store_env);
+    char *saved_q8_nsg = test_save_env(q8_nsg_env);
+
+    size_t q_mismatches = 0;
+    size_t kv_mismatches = 0;
+    size_t out_mismatches = 0;
+    size_t state_mismatches = 0;
+    uint32_t max_ulp = 0;
+    if (allocated) {
+        memset(model_raw, 0, (size_t)model_bytes);
+        test_fill_q8_0_weights(
+            (uint8_t *)model_raw + range_offsets[Q_A_WEIGHT],
+            in_dim, q_rank, seed + 1u);
+        test_fill_q8_0_weights(
+            (uint8_t *)model_raw + range_offsets[KV_WEIGHT],
+            in_dim, kv_dim, seed + 3u);
+
+        for (uint32_t matrix = 0; matrix < n_comp_outputs; matrix++) {
+            uint16_t *weights = (uint16_t *)(
+                (uint8_t *)model_raw +
+                range_offsets[COMP0_KV_WEIGHT + matrix]);
+            const uint32_t width = comp_widths[matrix];
+            for (uint32_t o = 0; o < width; o++) {
+                for (uint32_t i = 0; i < in_dim; i++) {
+                    const int value =
+                        (int)((o * (13u + 2u * matrix) +
+                               i * (19u + 4u * matrix) +
+                               (o ^ (i >> (matrix & 3u))) *
+                                   (5u + 2u * matrix) +
+                               seed * (23u + matrix)) % 127u) - 63;
+                    weights[(uint64_t)o * in_dim + i] =
+                        test_float_to_f16(
+                            (float)value / (88.0f + 8.0f * matrix));
+                }
+            }
+        }
+
+        const uint32_t ape_types[2] = {ape0_type, ape1_type};
+        const uint32_t ape_widths[2] = {width0, width1};
+        const uint32_t ape_ranges[2] = {COMP0_APE, COMP1_APE};
+        for (uint32_t which = 0; which < 2u; which++) {
+            if (ape_widths[which] == 0u) continue;
+            const uint64_t count =
+                (uint64_t)ratio * ape_widths[which];
+            if (ape_types[which] == 1u) {
+                uint16_t *ape = (uint16_t *)(
+                    (uint8_t *)model_raw + range_offsets[ape_ranges[which]]);
+                for (uint64_t i = 0; i < count; i++) {
+                    const int value =
+                        (int)((i * (11u + 2u * which) +
+                               (i ^ (i >> 3u)) * (7u + 2u * which) +
+                               seed * (17u + which)) % 61u) - 30;
+                    ape[i] = test_float_to_f16(
+                        (float)value / (72.0f + 8.0f * which));
+                }
+            } else {
+                float *ape = (float *)(
+                    (uint8_t *)model_raw + range_offsets[ape_ranges[which]]);
+                for (uint64_t i = 0; i < count; i++) {
+                    const int value =
+                        (int)((i * (11u + 2u * which) +
+                               (i ^ (i >> 3u)) * (7u + 2u * which) +
+                               seed * (17u + which)) % 61u) - 30;
+                    ape[i] =
+                        (float)value / (72.0f + 8.0f * which);
+                }
+            }
+        }
+
+        for (uint32_t i = 0; i < in_dim; i++) {
+            const int value =
+                (int)((i * 29u + (i ^ (i >> 4u)) * 9u +
+                       seed * 11u) % 127u) - 63;
+            x_host[i] = (float)value / 84.0f;
+        }
+        TEST_ASSERT(ds4_gpu_tensor_write(x, 0, x_host, x_bytes) != 0);
+
+        for (uint32_t buffer = 0; buffer < n_comp_outputs; buffer++) {
+            const uint32_t width = comp_widths[buffer];
+            const uint64_t out_bytes =
+                (uint64_t)width * sizeof(float);
+            const uint64_t state_count =
+                (uint64_t)state_rows * width;
+            const uint64_t state_bytes = state_count * sizeof(float);
+            for (uint32_t i = 0; i < width; i++) {
+                const uint32_t poison =
+                    0x7fc10001u + ((buffer * 2048u + i) & 0x7fffu);
+                memcpy(ref_host + i, &poison, sizeof(poison));
+            }
+            TEST_ASSERT(ds4_gpu_tensor_write(
+                            ref_out[buffer], 0, ref_host,
+                            out_bytes) != 0);
+            TEST_ASSERT(ds4_gpu_tensor_write(
+                            fused_out[buffer], 0, ref_host,
+                            out_bytes) != 0);
+            for (uint64_t i = 0; i < state_count; i++) {
+                const int value =
+                    (int)((i * (5u + 2u * buffer) +
+                           seed * (13u + buffer)) % 193u) - 96;
+                ref_host[i] =
+                    (float)value / (64.0f + 8.0f * buffer);
+            }
+            TEST_ASSERT(ds4_gpu_tensor_write(
+                            ref_state[buffer], 0, ref_host,
+                            state_bytes) != 0);
+            TEST_ASSERT(ds4_gpu_tensor_write(
+                            fused_state[buffer], 0, ref_host,
+                            state_bytes) != 0);
+        }
+
+        TEST_ASSERT(ds4_gpu_set_model_map(model_raw, model_bytes) != 0);
+        ds4_gpu_set_quality(false);
+        TEST_ASSERT(setenv(force_pair_env, "1", 1) == 0);
+        TEST_ASSERT(unsetenv(disable_pair_state_env) == 0);
+        TEST_ASSERT(unsetenv(disable_pair_proj_env) == 0);
+        TEST_ASSERT(unsetenv(disable_store_env) == 0);
+        TEST_ASSERT(unsetenv(q8_nsg_env) == 0);
+
+        TEST_ASSERT(ds4_gpu_matmul_q8_0_pair_tensor(
+                        ref_q, ref_kv,
+                        model_raw, model_bytes,
+                        range_offsets[Q_A_WEIGHT],
+                        range_offsets[KV_WEIGHT],
+                        in_dim, q_rank, kv_dim, x, 1u) == 1);
+        TEST_ASSERT(ds4_gpu_matmul_f16_pair_compressor_store_tensor(
+                        ref_out[0], ref_out[1],
+                        ref_state[0], ref_state[1],
+                        model_raw, model_bytes,
+                        range_offsets[COMP0_KV_WEIGHT],
+                        range_offsets[COMP0_SCORE_WEIGHT],
+                        range_offsets[COMP0_APE], ape0_type,
+                        in_dim, width0, x, ratio, pos) == 1);
+        if (width1 != 0u) {
+            TEST_ASSERT(ds4_gpu_matmul_f16_pair_compressor_store_tensor(
+                            ref_out[2], ref_out[3],
+                            ref_state[2], ref_state[3],
+                            model_raw, model_bytes,
+                            range_offsets[COMP1_KV_WEIGHT],
+                            range_offsets[COMP1_SCORE_WEIGHT],
+                            range_offsets[COMP1_APE], ape1_type,
+                            in_dim, width1, x, ratio, pos) == 1);
+        }
+
+        bool commands_open = false;
+        if (batched_commands) {
+            commands_open = ds4_gpu_begin_commands() != 0;
+            TEST_ASSERT(commands_open);
+        }
+        const int fused =
+            ds4_gpu_qkv_pair_quad_compressor_store_tensor(
+                fused_q, fused_kv,
+                fused_out[0], fused_out[1],
+                width1 != 0u ? fused_out[2] : NULL,
+                width1 != 0u ? fused_out[3] : NULL,
+                fused_state[0], fused_state[1],
+                width1 != 0u ? fused_state[2] : NULL,
+                width1 != 0u ? fused_state[3] : NULL,
+                model_raw, model_bytes,
+                range_offsets[Q_A_WEIGHT],
+                range_offsets[KV_WEIGHT],
+                range_offsets[COMP0_KV_WEIGHT],
+                range_offsets[COMP0_SCORE_WEIGHT],
+                range_offsets[COMP1_KV_WEIGHT],
+                range_offsets[COMP1_SCORE_WEIGHT],
+                range_offsets[COMP0_APE], ape0_type,
+                range_offsets[COMP1_APE], ape1_type,
+                in_dim, q_rank, kv_dim, width0, width1,
+                x, ratio, pos);
+        if (commands_open) {
+            TEST_ASSERT(ds4_gpu_end_commands() != 0);
+        }
+        TEST_ASSERT(fused == 1);
+
+        TEST_ASSERT(ds4_gpu_tensor_read(
+                        ref_q, 0, ref_host,
+                        (uint64_t)q_rank * sizeof(float)) != 0);
         TEST_ASSERT(ds4_gpu_tensor_read(
-                        fused_state_score, 0, fused_state_score_host,
-                        state_bytes) != 0);
+                        fused_q, 0, fused_host,
+                        (uint64_t)q_rank * sizeof(float)) != 0);
+        test_float_compare_stats stats =
+            test_compare_float_bits(ref_host, fused_host, q_rank);
+        q_mismatches = stats.mismatch_count;
+        if (stats.max_ulp > max_ulp) max_ulp = stats.max_ulp;
+
         TEST_ASSERT(ds4_gpu_tensor_read(
-                        ref_comp, 0, ref_comp_host, comp_bytes) != 0);
+                        ref_kv, 0, ref_host,
+                        (uint64_t)kv_dim * sizeof(float)) != 0);
         TEST_ASSERT(ds4_gpu_tensor_read(
-                        fused_comp, 0, fused_comp_host, comp_bytes) != 0);
+                        fused_kv, 0, fused_host,
+                        (uint64_t)kv_dim * sizeof(float)) != 0);
+        stats = test_compare_float_bits(ref_host, fused_host, kv_dim);
+        kv_mismatches = stats.mismatch_count;
+        if (stats.max_ulp > max_ulp) max_ulp = stats.max_ulp;
+
+        for (uint32_t buffer = 0; buffer < n_comp_outputs; buffer++) {
+            const uint32_t width = comp_widths[buffer];
+            const uint64_t out_bytes =
+                (uint64_t)width * sizeof(float);
+            const uint64_t state_count =
+                (uint64_t)state_rows * width;
+            const uint64_t state_bytes = state_count * sizeof(float);
+            TEST_ASSERT(ds4_gpu_tensor_read(
+                            ref_out[buffer], 0, ref_host,
+                            out_bytes) != 0);
+            TEST_ASSERT(ds4_gpu_tensor_read(
+                            fused_out[buffer], 0, fused_host,
+                            out_bytes) != 0);
+            stats = test_compare_float_bits(ref_host, fused_host, width);
+            out_mismatches += stats.mismatch_count;
+            if (stats.max_ulp > max_ulp) max_ulp = stats.max_ulp;
 
-        kv_stats = test_compare_float_bits(
-            ref_kv_host, fused_kv_host, width);
-        score_stats = test_compare_float_bits(
-            ref_score_host, fused_score_host, width);
-        state_kv_stats = test_compare_float_bits(
-            ref_state_kv_host, fused_state_kv_host, (size_t)state_count);
-        state_score_stats = test_compare_float_bits(
-            ref_state_score_host, fused_state_score_host,
-            (size_t)state_count);
-        comp_stats = test_compare_float_bits(
-            ref_comp_host, fused_comp_host, head_dim);
+            TEST_ASSERT(ds4_gpu_tensor_read(
+                            ref_state[buffer], 0, ref_host,
+                            state_bytes) != 0);
+            TEST_ASSERT(ds4_gpu_tensor_read(
+                            fused_state[buffer], 0, fused_host,
+                            state_bytes) != 0);
+            stats = test_compare_float_bits(
+                ref_host, fused_host, (size_t)state_count);
+            state_mismatches += stats.mismatch_count;
+            if (stats.max_ulp > max_ulp) max_ulp = stats.max_ulp;
+        }
     }
 
-    test_restore_env(pair_disable_env, saved_pair_disable);
-    test_restore_env(store_disable_env, saved_store_disable);
-    test_restore_env(decode_pack_disable_env, saved_decode_pack_disable);
-    test_restore_env(exact_reduction_disable_env,
-                     saved_exact_reduction_disable);
-    test_restore_env(
-        exact_reduction_poison_env, saved_exact_reduction_poison);
-
     fprintf(stderr,
-            "ds4-test: compressor pair state-store exact width=%u ratio=%u "
-            "pos=%u emit=%u ape=%s decode_pack=%u exact_reduce=%u "
-            "proj=%zu/%zu state=%zu/%zu "
-            "comp=%zu max_ulp=%u/%u/%u/%u/%u\n",
-            width, ratio, pos, emit ? 1u : 0u,
-            ape_type == 1u ? "f16" : "f32",
-            test_decode_pack ? 1u : 0u,
-            test_decode_pack ? 1u : 0u,
-            kv_stats.mismatch_count, score_stats.mismatch_count,
-            state_kv_stats.mismatch_count,
-            state_score_stats.mismatch_count,
-            comp_stats.mismatch_count,
-            kv_stats.max_ulp, score_stats.max_ulp,
-            state_kv_stats.max_ulp, state_score_stats.max_ulp,
-            comp_stats.max_ulp);
-    TEST_ASSERT(kv_stats.mismatch_count == 0);
-    TEST_ASSERT(score_stats.mismatch_count == 0);
-    TEST_ASSERT(state_kv_stats.mismatch_count == 0);
-    TEST_ASSERT(state_score_stats.mismatch_count == 0);
-    TEST_ASSERT(comp_stats.mismatch_count == 0);
+            "ds4-test: Q8 QKV/compressor compound exact ratio=%u "
+            "pos_mod=%u batch=%u q=%zu kv=%zu outputs=%zu states=%zu "
+            "max_ulp=%u\n",
+            ratio, pos % ratio, batched_commands ? 1u : 0u,
+            q_mismatches, kv_mismatches,
+            out_mismatches, state_mismatches, max_ulp);
+    TEST_ASSERT(q_mismatches == 0);
+    TEST_ASSERT(kv_mismatches == 0);
+    TEST_ASSERT(out_mismatches == 0);
+    TEST_ASSERT(state_mismatches == 0);
+    TEST_ASSERT(max_ulp == 0);
 
-    free(fused_comp_host);
-    free(ref_comp_host);
-    free(fused_state_score_host);
-    free(fused_state_kv_host);
-    free(ref_state_score_host);
-    free(ref_state_kv_host);
-    free(fused_score_host);
-    free(fused_kv_host);
-    free(ref_score_host);
-    free(ref_kv_host);
+    test_restore_env(q8_nsg_env, saved_q8_nsg);
+    test_restore_env(disable_store_env, saved_disable_store);
+    test_restore_env(disable_pair_proj_env, saved_disable_pair_proj);
+    test_restore_env(disable_pair_state_env, saved_disable_pair_state);
+    test_restore_env(force_pair_env, saved_force_pair);
+    free(fused_host);
+    free(ref_host);
     free(x_host);
-    ds4_gpu_tensor_free(fused_comp);
-    ds4_gpu_tensor_free(ref_comp);
-    ds4_gpu_tensor_free(fused_state_score);
-    ds4_gpu_tensor_free(fused_state_kv);
-    ds4_gpu_tensor_free(ref_state_score);
-    ds4_gpu_tensor_free(ref_state_kv);
-    ds4_gpu_tensor_free(fused_score);
+    for (uint32_t i = 0; i < 4u; i++) {
+        ds4_gpu_tensor_free(fused_state[i]);
+        ds4_gpu_tensor_free(ref_state[i]);
+        ds4_gpu_tensor_free(fused_out[i]);
+        ds4_gpu_tensor_free(ref_out[i]);
+    }
     ds4_gpu_tensor_free(fused_kv);
-    ds4_gpu_tensor_free(ref_score);
+    ds4_gpu_tensor_free(fused_q);
     ds4_gpu_tensor_free(ref_kv);
+    ds4_gpu_tensor_free(ref_q);
     ds4_gpu_tensor_free(x);
     free(model_raw);
 }
 
-static void test_metal_f16_compressor_pair_state_store_exact(void) {
-    test_metal_f16_compressor_pair_state_store_exact_case(
-        256, 4, 8, 0, 17, false);
-    test_metal_f16_compressor_pair_state_store_exact_case(
-        256, 4, 11, 1, 23, true);
-    test_metal_f16_compressor_pair_state_store_exact_case(
-        1024, 4, 11, 1, 29, true);
-    test_metal_f16_compressor_pair_state_store_exact_case(
-        512, 128, 255, 1, 43, false);
+static void test_metal_q8_qkv_compressor_compound_exact(void) {
+    test_metal_q8_qkv_compressor_compound_exact_case(
+        4u, 11u, 1u, 0u, 79u, false);
+    test_metal_q8_qkv_compressor_compound_exact_case(
+        4u, 11u, 1u, 0u, 79u, true);
+    test_metal_q8_qkv_compressor_compound_exact_case(
+        128u, 255u, 0u, 1u, 83u, false);
+    test_metal_q8_qkv_compressor_compound_exact_case(
+        128u, 255u, 0u, 1u, 83u, true);
 }
 
 static void test_metal_compressor_ape_add_exact_case(
@@ -2253,33 +4159,298 @@ static void test_metal_compressor_ratio4_direct_pool_exact_case(
     free(ref_comp_host);
     free(state_score_host);
     free(state_kv_host);
-    free(source_after_host);
-    free(sc_host);
-    free(kv_host);
-    ds4_gpu_tensor_free(direct_state_score);
-    ds4_gpu_tensor_free(direct_state_kv);
+    free(source_after_host);
+    free(sc_host);
+    free(kv_host);
+    ds4_gpu_tensor_free(direct_state_score);
+    ds4_gpu_tensor_free(direct_state_kv);
+    ds4_gpu_tensor_free(ref_state_score);
+    ds4_gpu_tensor_free(ref_state_kv);
+    ds4_gpu_tensor_free(direct_comp);
+    ds4_gpu_tensor_free(ref_comp);
+    ds4_gpu_tensor_free(sc);
+    ds4_gpu_tensor_free(kv);
+}
+
+static void test_metal_compressor_ratio4_direct_pool_exact(void) {
+    /* Prefill direct-pool coverage, including its legacy n_comp == 1 case. */
+    test_metal_compressor_ratio4_direct_pool_exact_case(
+        512, 0, 4, 1, false, 59);
+    test_metal_compressor_ratio4_direct_pool_exact_case(
+        512, 1, 16, 0, false, 61);
+    test_metal_compressor_ratio4_direct_pool_exact_case(
+        128, 3, 14, 1, false, 67);
+    test_metal_compressor_ratio4_direct_pool_exact_case(
+        512, 2048, 8, 1, true, 71);
+    test_metal_compressor_ratio4_direct_pool_exact_case(
+        128, 12, 12, 0, true, 73);
+    test_metal_compressor_ratio4_direct_pool_exact_case(
+        128, 8, 4, 1, true, 79);
+}
+
+static void test_metal_compressor_ratio4_exact_pool_decode_case(
+        uint32_t head_dim,
+        uint32_t ape_type,
+        uint32_t seed) {
+    const uint32_t ratio = 4u;
+    const uint32_t width = 2u * head_dim;
+    const uint32_t state_rows = 8u;
+    const uint32_t pos = 3u;
+    const uint64_t cur_count = width;
+    const uint64_t state_count = (uint64_t)state_rows * width;
+    const uint64_t cur_bytes = cur_count * sizeof(float);
+    const uint64_t state_bytes = state_count * sizeof(float);
+    const uint64_t comp_bytes = (uint64_t)head_dim * sizeof(float);
+    const uint64_t ape_elem_bytes = ape_type == 1u ? 2u : 4u;
+    const uint64_t ape_bytes = (uint64_t)ratio * width * ape_elem_bytes;
+    const uint64_t page = (uint64_t)getpagesize();
+    const uint64_t norm_offset = test_round_up_u64(ape_bytes, page);
+    const uint64_t model_bytes = test_round_up_u64(
+        norm_offset + (uint64_t)head_dim * sizeof(float), page);
+
+    ds4_gpu_tensor *kv_cur = ds4_gpu_tensor_alloc(cur_bytes);
+    ds4_gpu_tensor *sc_cur = ds4_gpu_tensor_alloc(cur_bytes);
+    ds4_gpu_tensor *ref_state_kv = ds4_gpu_tensor_alloc(state_bytes);
+    ds4_gpu_tensor *ref_state_score = ds4_gpu_tensor_alloc(state_bytes);
+    ds4_gpu_tensor *exact_state_kv = ds4_gpu_tensor_alloc(state_bytes);
+    ds4_gpu_tensor *exact_state_score = ds4_gpu_tensor_alloc(state_bytes);
+    ds4_gpu_tensor *ref_comp = ds4_gpu_tensor_alloc(comp_bytes);
+    ds4_gpu_tensor *exact_comp = ds4_gpu_tensor_alloc(comp_bytes);
+    float *kv_cur_host = malloc((size_t)cur_bytes);
+    float *sc_cur_host = malloc((size_t)cur_bytes);
+    float *state_kv_host = malloc((size_t)state_bytes);
+    float *state_score_host = malloc((size_t)state_bytes);
+    float *ref_state_kv_host = malloc((size_t)state_bytes);
+    float *ref_state_score_host = malloc((size_t)state_bytes);
+    float *exact_state_kv_host = malloc((size_t)state_bytes);
+    float *exact_state_score_host = malloc((size_t)state_bytes);
+    float *ref_comp_host = malloc((size_t)comp_bytes);
+    float *exact_comp_host = malloc((size_t)comp_bytes);
+    void *model_raw = NULL;
+    const int model_alloc_ok = posix_memalign(
+        &model_raw, (size_t)page, (size_t)model_bytes) == 0;
+
+    TEST_ASSERT(kv_cur != NULL);
+    TEST_ASSERT(sc_cur != NULL);
+    TEST_ASSERT(ref_state_kv != NULL);
+    TEST_ASSERT(ref_state_score != NULL);
+    TEST_ASSERT(exact_state_kv != NULL);
+    TEST_ASSERT(exact_state_score != NULL);
+    TEST_ASSERT(ref_comp != NULL);
+    TEST_ASSERT(exact_comp != NULL);
+    TEST_ASSERT(kv_cur_host != NULL);
+    TEST_ASSERT(sc_cur_host != NULL);
+    TEST_ASSERT(state_kv_host != NULL);
+    TEST_ASSERT(state_score_host != NULL);
+    TEST_ASSERT(ref_state_kv_host != NULL);
+    TEST_ASSERT(ref_state_score_host != NULL);
+    TEST_ASSERT(exact_state_kv_host != NULL);
+    TEST_ASSERT(exact_state_score_host != NULL);
+    TEST_ASSERT(ref_comp_host != NULL);
+    TEST_ASSERT(exact_comp_host != NULL);
+    TEST_ASSERT(model_alloc_ok);
+
+    const char *enable_env =
+        "DS4_METAL_ENABLE_COMPRESSOR_EXACT_POOL_RATIO4";
+    const char *disable_env =
+        "DS4_METAL_DISABLE_COMPRESSOR_EXACT_POOL_RATIO4";
+    const char *pre_m5_disable_env =
+        "DS4_METAL_DISABLE_PRE_M5_COMPRESSOR_EXACT_POOL_RATIO4";
+    const char *m3_disable_env =
+        "DS4_METAL_DISABLE_M3_COMPRESSOR_EXACT_POOL_RATIO4";
+    const char *m5_disable_env =
+        "DS4_METAL_DISABLE_M5_COMPRESSOR_EXACT_POOL_RATIO4";
+    const char *all_pre_m5_disable_env =
+        "DS4_METAL_DISABLE_PRE_M5_DECODE_PORTS";
+    const char *require_env =
+        "DS4_METAL_REQUIRE_COMPRESSOR_EXACT_POOL_RATIO4";
+    const char *poison_env =
+        "DS4_METAL_TEST_POISON_COMPRESSOR_EXACT_REDUCTION_SCRATCH";
+    char *saved_enable = test_save_env(enable_env);
+    char *saved_disable = test_save_env(disable_env);
+    char *saved_pre_m5_disable = test_save_env(pre_m5_disable_env);
+    char *saved_m3_disable = test_save_env(m3_disable_env);
+    char *saved_m5_disable = test_save_env(m5_disable_env);
+    char *saved_all_pre_m5_disable = test_save_env(all_pre_m5_disable_env);
+    char *saved_require = test_save_env(require_env);
+    char *saved_poison = test_save_env(poison_env);
+    test_float_compare_stats comp_stats = {0};
+    test_float_compare_stats state_kv_stats = {0};
+    test_float_compare_stats state_score_stats = {0};
+
+    const bool allocated = kv_cur && sc_cur && ref_state_kv &&
+        ref_state_score && exact_state_kv && exact_state_score && ref_comp &&
+        exact_comp && kv_cur_host && sc_cur_host && state_kv_host &&
+        state_score_host && ref_state_kv_host && ref_state_score_host &&
+        exact_state_kv_host && exact_state_score_host && ref_comp_host &&
+        exact_comp_host && model_alloc_ok;
+    if (allocated) {
+        memset(model_raw, 0, (size_t)model_bytes);
+        if (ape_type == 1u) {
+            uint16_t *ape = model_raw;
+            for (uint64_t i = 0; i < (uint64_t)ratio * width; i++) {
+                const int value =
+                    (int)((i * 17u + (i >> 3u) * 13u + seed * 11u) %
+                          101u) - 50;
+                ape[i] = test_float_to_f16((float)value / 96.0f);
+            }
+        } else {
+            float *ape = model_raw;
+            for (uint64_t i = 0; i < (uint64_t)ratio * width; i++) {
+                const int value =
+                    (int)((i * 17u + (i >> 3u) * 13u + seed * 11u) %
+                          101u) - 50;
+                ape[i] = (float)value / 96.0f;
+            }
+        }
+        float *norm =
+            (float *)((uint8_t *)model_raw + norm_offset);
+        for (uint32_t i = 0; i < head_dim; i++) {
+            norm[i] = 0.75f +
+                (float)((i * 7u + seed * 3u) % 29u) / 128.0f;
+        }
+
+        for (uint32_t i = 0; i < width; i++) {
+            const int kv_value =
+                (int)(((uint64_t)i * 29u + seed * 37u) % 211u) - 105;
+            const int score_value =
+                (int)(((uint64_t)i * 31u + seed * 19u) % 181u) - 90;
+            kv_cur_host[i] = (float)kv_value / 112.0f;
+            sc_cur_host[i] = (float)score_value / 48.0f;
+        }
+        for (uint32_t row = 0; row < state_rows; row++) {
+            for (uint32_t col = 0; col < width; col++) {
+                const uint64_t i = (uint64_t)row * width + col;
+                const int kv_value =
+                    (int)(((uint64_t)row * 43u + (uint64_t)col * 23u +
+                           seed * 41u) % 257u) - 128;
+                const int score_value =
+                    (int)(((uint64_t)row * 47u + (uint64_t)col * 17u +
+                           seed * 31u) % 193u) - 96;
+                state_kv_host[i] = (float)kv_value / 128.0f;
+                state_score_host[i] = (float)score_value / 56.0f;
+            }
+        }
+
+        TEST_ASSERT(ds4_gpu_tensor_write(
+                        kv_cur, 0, kv_cur_host, cur_bytes) != 0);
+        TEST_ASSERT(ds4_gpu_tensor_write(
+                        sc_cur, 0, sc_cur_host, cur_bytes) != 0);
+        TEST_ASSERT(ds4_gpu_tensor_write(
+                        ref_state_kv, 0, state_kv_host, state_bytes) != 0);
+        TEST_ASSERT(ds4_gpu_tensor_write(
+                        ref_state_score, 0, state_score_host,
+                        state_bytes) != 0);
+        TEST_ASSERT(ds4_gpu_tensor_write(
+                        exact_state_kv, 0, state_kv_host, state_bytes) != 0);
+        TEST_ASSERT(ds4_gpu_tensor_write(
+                        exact_state_score, 0, state_score_host,
+                        state_bytes) != 0);
+        TEST_ASSERT(ds4_gpu_set_model_map(model_raw, model_bytes) != 0);
+        ds4_gpu_set_quality(false);
+
+        /* Force-enable and force-disable together: the kill switch must win. */
+        TEST_ASSERT(setenv(enable_env, "1", 1) == 0);
+        TEST_ASSERT(setenv(disable_env, "1", 1) == 0);
+        TEST_ASSERT(unsetenv(require_env) == 0);
+        TEST_ASSERT(unsetenv(poison_env) == 0);
+        const int ref_ok = ds4_gpu_compressor_update_tensor(
+            kv_cur, sc_cur, ref_state_kv, ref_state_score, ref_comp,
+            model_raw, model_bytes, 0, ape_type, norm_offset, 0,
+            head_dim, ratio, pos, 0, 0, 0,
+            10000.0f, 1.0f, 0.0f, 1.0f, 32.0f, 1.0f, 1.0e-6f,
+            false, false, false);
+        TEST_ASSERT(ref_ok != 0);
+
+        TEST_ASSERT(unsetenv(disable_env) == 0);
+        TEST_ASSERT(unsetenv(pre_m5_disable_env) == 0);
+        TEST_ASSERT(unsetenv(m3_disable_env) == 0);
+        TEST_ASSERT(unsetenv(m5_disable_env) == 0);
+        TEST_ASSERT(unsetenv(all_pre_m5_disable_env) == 0);
+        TEST_ASSERT(setenv(require_env, "1", 1) == 0);
+        TEST_ASSERT(setenv(poison_env, "1", 1) == 0);
+        const int exact_ok = ds4_gpu_compressor_update_tensor(
+            kv_cur, sc_cur, exact_state_kv, exact_state_score, exact_comp,
+            model_raw, model_bytes, 0, ape_type, norm_offset, 0,
+            head_dim, ratio, pos, 0, 0, 0,
+            10000.0f, 1.0f, 0.0f, 1.0f, 32.0f, 1.0f, 1.0e-6f,
+            false, false, false);
+        TEST_ASSERT(exact_ok != 0);
+
+        TEST_ASSERT(ds4_gpu_tensor_read(
+                        ref_comp, 0, ref_comp_host, comp_bytes) != 0);
+        TEST_ASSERT(ds4_gpu_tensor_read(
+                        exact_comp, 0, exact_comp_host, comp_bytes) != 0);
+        TEST_ASSERT(ds4_gpu_tensor_read(
+                        ref_state_kv, 0, ref_state_kv_host,
+                        state_bytes) != 0);
+        TEST_ASSERT(ds4_gpu_tensor_read(
+                        exact_state_kv, 0, exact_state_kv_host,
+                        state_bytes) != 0);
+        TEST_ASSERT(ds4_gpu_tensor_read(
+                        ref_state_score, 0, ref_state_score_host,
+                        state_bytes) != 0);
+        TEST_ASSERT(ds4_gpu_tensor_read(
+                        exact_state_score, 0, exact_state_score_host,
+                        state_bytes) != 0);
+
+        comp_stats = test_compare_float_bits(
+            ref_comp_host, exact_comp_host, head_dim);
+        state_kv_stats = test_compare_float_bits(
+            ref_state_kv_host, exact_state_kv_host, (size_t)state_count);
+        state_score_stats = test_compare_float_bits(
+            ref_state_score_host, exact_state_score_host,
+            (size_t)state_count);
+    }
+
+    test_restore_env(enable_env, saved_enable);
+    test_restore_env(disable_env, saved_disable);
+    test_restore_env(pre_m5_disable_env, saved_pre_m5_disable);
+    test_restore_env(m3_disable_env, saved_m3_disable);
+    test_restore_env(m5_disable_env, saved_m5_disable);
+    test_restore_env(all_pre_m5_disable_env, saved_all_pre_m5_disable);
+    test_restore_env(require_env, saved_require);
+    test_restore_env(poison_env, saved_poison);
+    fprintf(stderr,
+            "ds4-test: compressor ratio4 exact decode pool head=%u ape=%s "
+            "comp=%zu/%u state_kv=%zu/%llu state_score=%zu/%llu "
+            "max_ulp=%u/%u/%u\n",
+            head_dim, ape_type == 1u ? "f16" : "f32",
+            comp_stats.mismatch_count, head_dim,
+            state_kv_stats.mismatch_count,
+            (unsigned long long)state_count,
+            state_score_stats.mismatch_count,
+            (unsigned long long)state_count,
+            comp_stats.max_ulp, state_kv_stats.max_ulp,
+            state_score_stats.max_ulp);
+    TEST_ASSERT(comp_stats.mismatch_count == 0);
+    TEST_ASSERT(state_kv_stats.mismatch_count == 0);
+    TEST_ASSERT(state_score_stats.mismatch_count == 0);
+
+    free(model_raw);
+    free(exact_comp_host);
+    free(ref_comp_host);
+    free(exact_state_score_host);
+    free(exact_state_kv_host);
+    free(ref_state_score_host);
+    free(ref_state_kv_host);
+    free(state_score_host);
+    free(state_kv_host);
+    free(sc_cur_host);
+    free(kv_cur_host);
+    ds4_gpu_tensor_free(exact_comp);
+    ds4_gpu_tensor_free(ref_comp);
+    ds4_gpu_tensor_free(exact_state_score);
+    ds4_gpu_tensor_free(exact_state_kv);
     ds4_gpu_tensor_free(ref_state_score);
     ds4_gpu_tensor_free(ref_state_kv);
-    ds4_gpu_tensor_free(direct_comp);
-    ds4_gpu_tensor_free(ref_comp);
-    ds4_gpu_tensor_free(sc);
-    ds4_gpu_tensor_free(kv);
+    ds4_gpu_tensor_free(sc_cur);
+    ds4_gpu_tensor_free(kv_cur);
 }
 
-static void test_metal_compressor_ratio4_direct_pool_exact(void) {
-    /* n_comp == 1 deliberately stays on the exact GGML reduction path. */
-    test_metal_compressor_ratio4_direct_pool_exact_case(
-        512, 0, 4, 1, false, 59);
-    test_metal_compressor_ratio4_direct_pool_exact_case(
-        512, 1, 16, 0, false, 61);
-    test_metal_compressor_ratio4_direct_pool_exact_case(
-        128, 3, 14, 1, false, 67);
-    test_metal_compressor_ratio4_direct_pool_exact_case(
-        512, 2048, 8, 1, true, 71);
-    test_metal_compressor_ratio4_direct_pool_exact_case(
-        128, 12, 12, 0, true, 73);
-    test_metal_compressor_ratio4_direct_pool_exact_case(
-        128, 8, 4, 1, true, 79);
+static void test_metal_compressor_ratio4_exact_pool_decode(void) {
+    test_metal_compressor_ratio4_exact_pool_decode_case(128u, 1u, 83u);
+    test_metal_compressor_ratio4_exact_pool_decode_case(512u, 0u, 89u);
 }
 
 static void test_metal_inplace_rope_pair_exact(void) {
@@ -3638,9 +5809,1052 @@ static void test_metal_zero_prefix_prefill_mask_cache_exact(void) {
     test_metal_zero_prefix_prefill_mask_cache_exact_kind(
         TEST_METAL_PREFILL_MASK_CACHE_RATIO128, 47);
 }
+
+static double test_metal_small_prefill_gpu_batch_ms(
+        test_metal_prefill_mask_cache_kind kind,
+        bool masked,
+        const test_metal_prefill_mask_cache_shape *shape,
+        ds4_gpu_tensor *heads,
+        const void *model_map,
+        uint64_t model_size,
+        const ds4_gpu_tensor *q,
+        const ds4_gpu_tensor *raw,
+        const ds4_gpu_tensor *comp,
+        const ds4_gpu_tensor *comp_mask,
+        uint32_t n_head,
+        uint32_t head_dim,
+        uint32_t flags,
+        uint32_t repeats,
+        uint32_t expected_nwg) {
+    ds4_gpu_test_set_flags(flags);
+    const int begin_ok = ds4_gpu_begin_commands();
+    TEST_ASSERT(begin_ok != 0);
+    if (!begin_ok) return -1.0;
+    for (uint32_t rep = 0u; rep < repeats; rep++) {
+        const int call_ok = test_metal_zero_prefix_prefill_mask_cache_call(
+            kind, heads, model_map, model_size, q, raw, comp, comp_mask,
+            shape, masked, n_head, head_dim);
+        TEST_ASSERT(call_ok != 0);
+        if (!call_ok) {
+            (void)ds4_gpu_end_commands();
+            return -1.0;
+        }
+    }
+    const int end_ok = ds4_gpu_end_commands();
+    TEST_ASSERT(end_ok != 0);
+    if (!end_ok) return -1.0;
+    const uint32_t selected_nwg =
+        ds4_gpu_test_last_flash_attn_prefill_nwg();
+    TEST_ASSERT(selected_nwg == expected_nwg);
+    if (selected_nwg != expected_nwg) return -1.0;
+    const double gpu_ms = ds4_gpu_test_last_completed_gpu_ms();
+    TEST_ASSERT(gpu_ms > 0.0);
+    return gpu_ms > 0.0 ? gpu_ms / (double)repeats : -1.0;
+}
+
+static void test_metal_small_prefill_direct_exact(void) {
+    const uint32_t head_dim = 512u;
+    const uint32_t n_head = 2u;
+    const uint32_t max_tokens = 19u;
+    const uint32_t max_comp = 14u;
+    const uint64_t guard_bytes = 256u;
+    const uint64_t raw_count = (uint64_t)max_tokens * head_dim;
+    const uint64_t comp_count = (uint64_t)max_comp * head_dim;
+    const uint64_t q_count =
+        (uint64_t)max_tokens * n_head * head_dim;
+    const uint64_t mask_count = (uint64_t)max_tokens * max_comp;
+    const uint64_t raw_bytes = raw_count * sizeof(float);
+    const uint64_t comp_bytes = comp_count * sizeof(uint16_t);
+    const uint64_t q_bytes = q_count * sizeof(float);
+    const uint64_t mask_bytes = mask_count * sizeof(float);
+    const uint64_t heads_base_bytes = guard_bytes + q_bytes + guard_bytes;
+    const uint64_t page = (uint64_t)getpagesize();
+    const char *disable_env = "DS4_METAL_DISABLE_SMALL_PREFILL_DIRECT";
+    char *saved_disable = test_save_env(disable_env);
+    const char *stage_profile_env = "DS4_METAL_FLASH_ATTN_STAGE_PROFILE";
+    char *saved_stage_profile = test_save_env(stage_profile_env);
+
+    typedef struct {
+        const char *name;
+        test_metal_prefill_mask_cache_kind kind;
+        bool masked;
+        test_metal_prefill_mask_cache_shape shape;
+    } test_metal_small_prefill_case;
+    static const test_metal_small_prefill_case cases[] = {
+        {"raw-1", TEST_METAL_PREFILL_MASK_CACHE_RAW, false,
+         {1u, 0u, 0u, 0u}},
+        {"raw-7", TEST_METAL_PREFILL_MASK_CACHE_RAW, false,
+         {7u, 0u, 5u, 0u}},
+        {"raw-19", TEST_METAL_PREFILL_MASK_CACHE_RAW, false,
+         {19u, 0u, 11u, 0u}},
+        {"static-12", TEST_METAL_PREFILL_MASK_CACHE_RATIO4, false,
+         {7u, 5u, 5u, 4u}},
+        {"static-32", TEST_METAL_PREFILL_MASK_CACHE_RATIO4, false,
+         {19u, 13u, 11u, 4u}},
+        {"masked-12", TEST_METAL_PREFILL_MASK_CACHE_RATIO4, true,
+         {7u, 5u, 5u, 4u}},
+        {"masked-32", TEST_METAL_PREFILL_MASK_CACHE_RATIO4, true,
+         {19u, 13u, 11u, 4u}},
+        {"static-33-fallback", TEST_METAL_PREFILL_MASK_CACHE_RATIO4, false,
+         {19u, 14u, 11u, 4u}},
+    };
+    enum {
+        TEST_SMALL_PREFILL_DIRECT_COLD = 0,
+        TEST_SMALL_PREFILL_PSO_FALLBACK,
+        TEST_SMALL_PREFILL_ENV_ROLLBACK,
+        TEST_SMALL_PREFILL_DIRECT_REPLAY,
+        TEST_SMALL_PREFILL_ARM_COUNT,
+    };
+
+    ds4_gpu_tensor *raw = ds4_gpu_tensor_alloc(raw_bytes);
+    ds4_gpu_tensor *comp = ds4_gpu_tensor_alloc(comp_bytes);
+    ds4_gpu_tensor *q = ds4_gpu_tensor_alloc(q_bytes);
+    ds4_gpu_tensor *comp_mask = ds4_gpu_tensor_alloc(mask_bytes);
+    ds4_gpu_tensor *heads_base = ds4_gpu_tensor_alloc(heads_base_bytes);
+    float *raw_host = malloc((size_t)raw_bytes);
+    uint16_t *comp_host = malloc((size_t)comp_bytes);
+    float *q_host = malloc((size_t)q_bytes);
+    float *mask_host = malloc((size_t)mask_bytes);
+    uint8_t *initial = malloc((size_t)heads_base_bytes);
+    uint8_t *snapshots = malloc(
+        (size_t)(TEST_SMALL_PREFILL_ARM_COUNT * heads_base_bytes));
+    /* Metal retains a no-copy model view after this oracle returns.  Static
+     * page-aligned storage keeps that backing valid until backend cleanup. */
+    static uint8_t model_storage[16384]
+        __attribute__((aligned(16384)));
+    void *model_raw = model_storage;
+    const bool model_storage_ok = page <= sizeof(model_storage);
+    TEST_ASSERT(model_storage_ok);
+    TEST_ASSERT(raw != NULL);
+    TEST_ASSERT(comp != NULL);
+    TEST_ASSERT(q != NULL);
+    TEST_ASSERT(comp_mask != NULL);
+    TEST_ASSERT(heads_base != NULL);
+    TEST_ASSERT(raw_host != NULL);
+    TEST_ASSERT(comp_host != NULL);
+    TEST_ASSERT(q_host != NULL);
+    TEST_ASSERT(mask_host != NULL);
+    TEST_ASSERT(initial != NULL);
+    TEST_ASSERT(snapshots != NULL);
+
+    size_t direct_mismatches = 0u;
+    size_t replay_mismatches = 0u;
+    size_t fallback_mismatches = 0u;
+    size_t alias_mismatches = 0u;
+    size_t alias_guard_mismatches = 0u;
+    size_t guard_mismatches = 0u;
+    size_t nonfinite = 0u;
+    size_t total_words = 0u;
+    size_t completed_cases = 0u;
+    const bool allocated = raw && comp && q && comp_mask && heads_base &&
+        raw_host && comp_host && q_host && mask_host && initial &&
+        snapshots && model_storage_ok;
+    if (allocated) {
+        memset(model_raw, 0, (size_t)page);
+        ((float *)model_raw)[0] = -0.375f;
+        ((float *)model_raw)[1] = 0.21875f;
+        for (uint64_t i = 0; i < raw_count; i++) {
+            const int value = (int)((i * 17u +
+                (i ^ (i >> 5u)) * 11u + 23u) % 211u) - 105;
+            raw_host[i] = (float)value / 192.0f;
+        }
+        for (uint64_t i = 0; i < comp_count; i++) {
+            const int value = (int)((i * 23u +
+                (i ^ (i >> 4u)) * 7u + 29u) % 193u) - 96;
+            comp_host[i] = test_float_to_f16(
+                0.125f + (float)value / 224.0f);
+        }
+        for (uint64_t i = 0; i < q_count; i++) {
+            const int value = (int)((i * 31u +
+                (i ^ (i >> 3u)) * 5u + 37u) % 227u) - 113;
+            q_host[i] = (float)value / 208.0f;
+        }
+        for (uint64_t i = 0; i < mask_count; i++) {
+            mask_host[i] = (i % 5u) == 0u
+                ? -65504.0f : -(float)((i % 13u) + 1u) / 16.0f;
+        }
+        for (uint64_t i = 0; i < heads_base_bytes; i++) {
+            initial[i] = (uint8_t)(0xa5u ^ (uint8_t)(i * 37u));
+        }
+        const uint32_t poison = 0x7fc12345u;
+        for (uint64_t i = 0; i < q_count; i++) {
+            memcpy(initial + guard_bytes + i * sizeof(poison),
+                   &poison, sizeof(poison));
+        }
+
+        TEST_ASSERT(ds4_gpu_tensor_write(
+            raw, 0u, raw_host, raw_bytes) != 0);
+        TEST_ASSERT(ds4_gpu_tensor_write(
+            comp, 0u, comp_host, comp_bytes) != 0);
+        TEST_ASSERT(ds4_gpu_tensor_write(
+            q, 0u, q_host, q_bytes) != 0);
+        TEST_ASSERT(ds4_gpu_tensor_write(
+            comp_mask, 0u, mask_host, mask_bytes) != 0);
+        TEST_ASSERT(ds4_gpu_set_model_map(model_raw, page) != 0);
+        ds4_gpu_set_quality(false);
+        TEST_ASSERT(unsetenv(disable_env) == 0);
+        TEST_ASSERT(unsetenv(stage_profile_env) == 0);
+
+        for (size_t case_i = 0;
+             case_i < sizeof(cases) / sizeof(cases[0]); case_i++) {
+            const test_metal_small_prefill_case *c = &cases[case_i];
+            const uint64_t output_count =
+                (uint64_t)c->shape.n_tokens * n_head * head_dim;
+            const uint64_t output_bytes = output_count * sizeof(float);
+            ds4_gpu_tensor *heads = ds4_gpu_tensor_view(
+                heads_base, guard_bytes, output_bytes);
+            TEST_ASSERT(heads != NULL);
+            bool case_ok = heads != NULL;
+            for (uint32_t arm = 0u;
+                 arm < TEST_SMALL_PREFILL_ARM_COUNT && case_ok; arm++) {
+                const bool cold_scratch_case =
+                    case_i == 0u || case_i == 3u;
+                if (cold_scratch_case &&
+                    (arm == TEST_SMALL_PREFILL_DIRECT_COLD ||
+                     arm == TEST_SMALL_PREFILL_PSO_FALLBACK)) {
+                    TEST_ASSERT(ds4_gpu_test_reset_flash_attn_tmp() != 0);
+                    TEST_ASSERT(ds4_gpu_test_flash_attn_tmp_bytes() == 0u);
+                }
+                TEST_ASSERT(ds4_gpu_tensor_write(
+                    heads_base, 0u, initial, heads_base_bytes) != 0);
+                const bool env_rollback =
+                    arm == TEST_SMALL_PREFILL_ENV_ROLLBACK;
+                const int env_ok = env_rollback
+                    ? setenv(disable_env, "0", 1)
+                    : unsetenv(disable_env);
+                TEST_ASSERT(env_ok == 0);
+                ds4_gpu_test_set_flags(
+                    arm == TEST_SMALL_PREFILL_PSO_FALLBACK
+                        ? DS4_GPU_TEST_FLASH_ATTN_SMALL_PREFILL_NWG1_FAILURE
+                        : 0u);
+                const int call_ok =
+                    test_metal_zero_prefix_prefill_mask_cache_call(
+                        c->kind, heads, model_raw, page, q, raw, comp,
+                        comp_mask, &c->shape, c->masked,
+                        n_head, head_dim);
+                TEST_ASSERT(call_ok != 0);
+                const uint32_t n_keys =
+                    c->shape.n_tokens + c->shape.n_comp;
+                const bool expect_direct =
+                    (arm == TEST_SMALL_PREFILL_DIRECT_COLD ||
+                     arm == TEST_SMALL_PREFILL_DIRECT_REPLAY) &&
+                    n_keys <= 32u;
+                const uint32_t expected_nwg = expect_direct ? 1u : 32u;
+                const uint32_t selected_nwg =
+                    ds4_gpu_test_last_flash_attn_prefill_nwg();
+                TEST_ASSERT(selected_nwg == expected_nwg);
+                if (cold_scratch_case &&
+                    arm == TEST_SMALL_PREFILL_DIRECT_COLD) {
+                    TEST_ASSERT(
+                        ds4_gpu_test_flash_attn_tmp_bytes() == 0u);
+                }
+                if (cold_scratch_case &&
+                    arm == TEST_SMALL_PREFILL_PSO_FALLBACK) {
+                    TEST_ASSERT(
+                        ds4_gpu_test_flash_attn_tmp_bytes() != 0u);
+                }
+                const int read_ok = call_ok && ds4_gpu_tensor_read(
+                    heads_base, 0u,
+                    snapshots + (uint64_t)arm * heads_base_bytes,
+                    heads_base_bytes) != 0;
+                TEST_ASSERT(read_ok != 0);
+                case_ok = read_ok != 0 && selected_nwg == expected_nwg;
+            }
+            ds4_gpu_test_set_flags(0u);
+            ds4_gpu_tensor_free(heads);
+            if (!case_ok) continue;
+
+            const uint8_t *direct = snapshots +
+                TEST_SMALL_PREFILL_DIRECT_COLD * heads_base_bytes;
+            const uint8_t *baseline = snapshots +
+                TEST_SMALL_PREFILL_ENV_ROLLBACK * heads_base_bytes;
+            const uint8_t *replay = snapshots +
+                TEST_SMALL_PREFILL_DIRECT_REPLAY * heads_base_bytes;
+            const uint8_t *fallback = snapshots +
+                TEST_SMALL_PREFILL_PSO_FALLBACK * heads_base_bytes;
+            const test_float_compare_stats direct_stats =
+                test_compare_float_bits(
+                    (const float *)(baseline + guard_bytes),
+                    (const float *)(direct + guard_bytes),
+                    (size_t)output_count);
+            const test_float_compare_stats replay_stats =
+                test_compare_float_bits(
+                    (const float *)(baseline + guard_bytes),
+                    (const float *)(replay + guard_bytes),
+                    (size_t)output_count);
+            const test_float_compare_stats fallback_stats =
+                test_compare_float_bits(
+                    (const float *)(baseline + guard_bytes),
+                    (const float *)(fallback + guard_bytes),
+                    (size_t)output_count);
+            direct_mismatches += direct_stats.mismatch_count;
+            replay_mismatches += replay_stats.mismatch_count;
+            fallback_mismatches += fallback_stats.mismatch_count;
+            total_words += (size_t)output_count;
+            completed_cases++;
+
+            for (uint32_t arm = 0u;
+                 arm < TEST_SMALL_PREFILL_ARM_COUNT; arm++) {
+                const uint8_t *snapshot =
+                    snapshots + (uint64_t)arm * heads_base_bytes;
+                for (uint64_t i = 0; i < heads_base_bytes; i++) {
+                    const bool in_output = i >= guard_bytes &&
+                        i < guard_bytes + output_bytes;
+                    if (!in_output && snapshot[i] != initial[i]) {
+                        guard_mismatches++;
+                    }
+                }
+                const float *values =
+                    (const float *)(snapshot + guard_bytes);
+                for (uint64_t i = 0; i < output_count; i++) {
+                    if (!isfinite(values[i])) nonfinite++;
+                }
+            }
+            fprintf(stderr,
+                    "ds4-test: small-prefill direct %s keys=%u "
+                    "direct=%zu/%llu replay=%zu/%llu fallback=%zu/%llu\n",
+                    c->name,
+                    c->shape.n_tokens + c->shape.n_comp,
+                    direct_stats.mismatch_count,
+                    (unsigned long long)output_count,
+                    replay_stats.mismatch_count,
+                    (unsigned long long)output_count,
+                    fallback_stats.mismatch_count,
+                    (unsigned long long)output_count);
+        }
+
+        /* The legacy split path permits heads to alias q because its first
+         * dispatch finishes reading q before the reducer writes heads.  The
+         * direct kernel must preserve that API behavior by falling back. */
+        {
+            static const size_t alias_case_indices[] = {1u, 3u, 5u};
+            for (size_t alias_i = 0u;
+                 alias_i < sizeof(alias_case_indices) /
+                     sizeof(alias_case_indices[0]);
+                 alias_i++) {
+                const test_metal_small_prefill_case *c =
+                    &cases[alias_case_indices[alias_i]];
+                const uint64_t output_count =
+                    (uint64_t)c->shape.n_tokens * n_head * head_dim;
+                const uint64_t output_bytes = output_count * sizeof(float);
+                ds4_gpu_tensor *heads_ref = ds4_gpu_tensor_view(
+                    heads_base, guard_bytes, output_bytes);
+                ds4_gpu_tensor *q_alias = ds4_gpu_tensor_view(
+                    heads_base, guard_bytes, q_bytes);
+                ds4_gpu_tensor *heads_alias = ds4_gpu_tensor_view(
+                    heads_base, guard_bytes, output_bytes);
+                TEST_ASSERT(heads_ref != NULL);
+                TEST_ASSERT(q_alias != NULL);
+                TEST_ASSERT(heads_alias != NULL);
+                if (heads_ref && q_alias && heads_alias) {
+                    TEST_ASSERT(ds4_gpu_tensor_write(
+                        q, 0u, q_host, q_bytes) != 0);
+                    TEST_ASSERT(ds4_gpu_tensor_write(
+                        heads_base, 0u, initial, heads_base_bytes) != 0);
+                    TEST_ASSERT(setenv(disable_env, "0", 1) == 0);
+                    ds4_gpu_test_set_flags(0u);
+                    const int ref_ok =
+                        test_metal_zero_prefix_prefill_mask_cache_call(
+                            c->kind, heads_ref, model_raw, page, q, raw,
+                            comp, comp_mask, &c->shape, c->masked,
+                            n_head, head_dim);
+                    TEST_ASSERT(ref_ok != 0);
+                    TEST_ASSERT(
+                        ds4_gpu_test_last_flash_attn_prefill_nwg() == 32u);
+                    const int ref_read_ok = ref_ok && ds4_gpu_tensor_read(
+                        heads_base, 0u, snapshots, heads_base_bytes) != 0;
+                    TEST_ASSERT(ref_read_ok != 0);
+
+                    TEST_ASSERT(ds4_gpu_tensor_write(
+                        heads_base, 0u, initial, heads_base_bytes) != 0);
+                    TEST_ASSERT(ds4_gpu_tensor_write(
+                        q_alias, 0u, q_host, q_bytes) != 0);
+                    const int before_read_ok = ds4_gpu_tensor_read(
+                        heads_base, 0u, snapshots + heads_base_bytes,
+                        heads_base_bytes) != 0;
+                    TEST_ASSERT(before_read_ok != 0);
+                    TEST_ASSERT(unsetenv(disable_env) == 0);
+                    ds4_gpu_test_set_flags(0u);
+                    const int alias_ok =
+                        test_metal_zero_prefix_prefill_mask_cache_call(
+                            c->kind, heads_alias, model_raw, page, q_alias,
+                            raw, comp, comp_mask, &c->shape, c->masked,
+                            n_head, head_dim);
+                    TEST_ASSERT(alias_ok != 0);
+                    const uint32_t alias_nwg =
+                        ds4_gpu_test_last_flash_attn_prefill_nwg();
+                    TEST_ASSERT(alias_nwg == 32u);
+                    const int alias_read_ok = alias_ok &&
+                        ds4_gpu_tensor_read(
+                            heads_base, 0u,
+                            snapshots + 2u * heads_base_bytes,
+                            heads_base_bytes) != 0;
+                    TEST_ASSERT(alias_read_ok != 0);
+                    if (ref_read_ok && before_read_ok && alias_read_ok) {
+                        const test_float_compare_stats alias_stats =
+                            test_compare_float_bits(
+                                (const float *)(snapshots + guard_bytes),
+                                (const float *)(snapshots +
+                                    2u * heads_base_bytes + guard_bytes),
+                                (size_t)output_count);
+                        alias_mismatches += alias_stats.mismatch_count;
+                        size_t case_guard_mismatches = 0u;
+                        const uint8_t *before =
+                            snapshots + heads_base_bytes;
+                        const uint8_t *after =
+                            snapshots + 2u * heads_base_bytes;
+                        for (uint64_t i = 0u;
+                             i < heads_base_bytes; i++) {
+                            const bool in_output = i >= guard_bytes &&
+                                i < guard_bytes + output_bytes;
+                            if (!in_output && before[i] != after[i]) {
+                                case_guard_mismatches++;
+                            }
+                        }
+                        alias_guard_mismatches += case_guard_mismatches;
+                        fprintf(stderr,
+                                "ds4-test: small-prefill q/heads alias "
+                                "%s nwg=%u mismatches=%zu/%llu guards=%zu\n",
+                                c->name, alias_nwg,
+                                alias_stats.mismatch_count,
+                                (unsigned long long)output_count,
+                                case_guard_mismatches);
+                    }
+                }
+                ds4_gpu_test_set_flags(0u);
+                TEST_ASSERT(unsetenv(disable_env) == 0);
+                ds4_gpu_tensor_free(heads_alias);
+                ds4_gpu_tensor_free(q_alias);
+                ds4_gpu_tensor_free(heads_ref);
+            }
+        }
+
+        if (test_env_bool("DS4_TEST_METAL_SMALL_PREFILL_TIMING")) {
+            /* GPUStartTime/GPUEndTime excludes CPU mask construction,
+             * encoding, submit, and wait time.  Both arms run the same
+             * copy/pad kernels; their only topology difference is NWG=32 +
+             * reduce versus direct NWG=1 output.  Alternate pair order to
+             * limit thermal/order bias.  Keep this opt-in so the exactness
+             * oracle remains suitable for routine test runs. */
+            static const size_t timing_case_indices[] = {2u, 4u};
+            const uint32_t timing_repeats = 16u;
+            const uint32_t timing_samples = 4u;
+            for (size_t timing_i = 0u;
+                 timing_i < sizeof(timing_case_indices) /
+                     sizeof(timing_case_indices[0]);
+                 timing_i++) {
+                const test_metal_small_prefill_case *c =
+                    &cases[timing_case_indices[timing_i]];
+                const uint64_t output_bytes =
+                    (uint64_t)c->shape.n_tokens * n_head * head_dim *
+                    sizeof(float);
+                ds4_gpu_tensor *heads = ds4_gpu_tensor_view(
+                    heads_base, guard_bytes, output_bytes);
+                TEST_ASSERT(heads != NULL);
+                if (!heads) continue;
+
+                (void)test_metal_small_prefill_gpu_batch_ms(
+                    c->kind, c->masked, &c->shape, heads, model_raw, page,
+                    q, raw, comp, comp_mask, n_head, head_dim,
+                    DS4_GPU_TEST_FLASH_ATTN_SMALL_PREFILL_NWG32, 4u, 32u);
+                (void)test_metal_small_prefill_gpu_batch_ms(
+                    c->kind, c->masked, &c->shape, heads, model_raw, page,
+                    q, raw, comp, comp_mask, n_head, head_dim,
+                    0u, 4u, 1u);
+
+                double baseline_sum = 0.0;
+                double direct_sum = 0.0;
+                double log_speedup_sum = 0.0;
+                uint32_t valid_samples = 0u;
+                for (uint32_t sample = 0u;
+                     sample < timing_samples; sample++) {
+                    double pair_ms[2] = {-1.0, -1.0};
+                    for (uint32_t order_i = 0u; order_i < 2u; order_i++) {
+                        const uint32_t arm = (sample & 1u) != 0u
+                            ? 1u - order_i : order_i;
+                        pair_ms[arm] =
+                            test_metal_small_prefill_gpu_batch_ms(
+                                c->kind, c->masked, &c->shape,
+                                heads, model_raw, page, q, raw, comp,
+                                comp_mask, n_head, head_dim,
+                                arm == 0u
+                                    ? DS4_GPU_TEST_FLASH_ATTN_SMALL_PREFILL_NWG32
+                                    : 0u,
+                                timing_repeats,
+                                arm == 0u ? 32u : 1u);
+                    }
+                    if (pair_ms[0] > 0.0 && pair_ms[1] > 0.0) {
+                        baseline_sum += pair_ms[0];
+                        direct_sum += pair_ms[1];
+                        log_speedup_sum += log(pair_ms[0] / pair_ms[1]);
+                        valid_samples++;
+                    }
+                }
+                TEST_ASSERT(valid_samples == timing_samples);
+                if (valid_samples != 0u) {
+                    const double baseline_ms =
+                        baseline_sum / (double)valid_samples;
+                    const double direct_ms =
+                        direct_sum / (double)valid_samples;
+                    const double speedup =
+                        exp(log_speedup_sum / (double)valid_samples);
+                    fprintf(stderr,
+                            "ds4-test: small-prefill GPU-only kernel-chain %s "
+                            "legacy=%.6f ms direct=%.6f ms speedup=%.3fx "
+                            "throughput=%.1f%%\n",
+                            c->name, baseline_ms, direct_ms, speedup,
+                            (speedup - 1.0) * 100.0);
+                }
+                ds4_gpu_tensor_free(heads);
+            }
+        }
+    }
+
+    ds4_gpu_test_set_flags(0u);
+    test_restore_env(stage_profile_env, saved_stage_profile);
+    test_restore_env(disable_env, saved_disable);
+    fprintf(stderr,
+            "ds4-test: small-prefill direct exact cases=%zu/%zu "
+            "words=%zu direct=%zu replay=%zu fallback=%zu alias=%zu "
+            "guards=%zu alias_guards=%zu nonfinite=%zu\n",
+            completed_cases, sizeof(cases) / sizeof(cases[0]), total_words,
+            direct_mismatches, replay_mismatches, fallback_mismatches,
+            alias_mismatches, guard_mismatches, alias_guard_mismatches,
+            nonfinite);
+    TEST_ASSERT(completed_cases == sizeof(cases) / sizeof(cases[0]));
+    TEST_ASSERT(direct_mismatches == 0u);
+    TEST_ASSERT(replay_mismatches == 0u);
+    TEST_ASSERT(fallback_mismatches == 0u);
+    TEST_ASSERT(alias_mismatches == 0u);
+    TEST_ASSERT(guard_mismatches == 0u);
+    TEST_ASSERT(alias_guard_mismatches == 0u);
+    TEST_ASSERT(nonfinite == 0u);
+
+    free(snapshots);
+    free(initial);
+    free(mask_host);
+    free(q_host);
+    free(comp_host);
+    free(raw_host);
+    ds4_gpu_tensor_free(heads_base);
+    ds4_gpu_tensor_free(comp_mask);
+    ds4_gpu_tensor_free(q);
+    ds4_gpu_tensor_free(comp);
+    ds4_gpu_tensor_free(raw);
+}
 #endif
 
 #if defined(__APPLE__)
+typedef enum {
+    TEST_METAL_ATTN_OUT_HC_Q8,
+    TEST_METAL_ATTN_OUT_HC_Q4,
+} test_metal_attn_out_hc_kind;
+
+static uint32_t test_metal_attn_out_hc_poison_bits(
+        uint32_t tag,
+        uint64_t index) {
+    return 0x7fc00000u |
+        ((tag & 0x1fu) << 16u) |
+        (uint32_t)((index + 1u) & 0xffffu);
+}
+
+static void test_metal_attn_out_hc_fill_poison(
+        float    *dst,
+        uint64_t  count,
+        uint32_t  tag) {
+    for (uint64_t i = 0; i < count; i++) {
+        const uint32_t bits = test_metal_attn_out_hc_poison_bits(tag, i);
+        memcpy(dst + i, &bits, sizeof(bits));
+    }
+}
+
+static size_t test_metal_attn_out_hc_canary_mismatches(
+        const float *values,
+        uint64_t     begin,
+        uint64_t     end,
+        uint32_t     tag) {
+    size_t mismatches = 0u;
+    for (uint64_t i = begin; i < end; i++) {
+        uint32_t bits = 0u;
+        memcpy(&bits, values + i, sizeof(bits));
+        if (bits != test_metal_attn_out_hc_poison_bits(tag, i)) {
+            mismatches++;
+        }
+    }
+    return mismatches;
+}
+
+static size_t test_metal_attn_out_hc_poisoned_values(
+        const float *values,
+        uint64_t     count,
+        uint32_t     tag) {
+    size_t poisoned = 0u;
+    for (uint64_t i = 0; i < count; i++) {
+        uint32_t bits = 0u;
+        memcpy(&bits, values + i, sizeof(bits));
+        if (bits == test_metal_attn_out_hc_poison_bits(tag, i)) {
+            poisoned++;
+        }
+    }
+    return poisoned;
+}
+
+static int test_metal_batch_attn_out_hc_fused_call(
+        test_metal_attn_out_hc_kind kind,
+        ds4_gpu_tensor       *out,
+        ds4_gpu_tensor       *out_hc,
+        const ds4_gpu_tensor *residual_hc,
+        const ds4_gpu_tensor *split,
+        ds4_gpu_tensor       *low,
+        ds4_gpu_tensor       *group_tmp,
+        ds4_gpu_tensor       *low_tmp,
+        const void           *model_map,
+        uint64_t              model_size,
+        uint64_t              out_b_offset,
+        const ds4_gpu_tensor *heads,
+        uint32_t              n_tokens) {
+    const uint32_t group_dim = 4096u;
+    const uint32_t rank = 1024u;
+    const uint32_t n_groups = 8u;
+    const uint32_t out_dim = 4096u;
+    const uint32_t n_hc = 4u;
+    if (kind == TEST_METAL_ATTN_OUT_HC_Q4) {
+        return ds4_gpu_attention_output_q4_K_batch_hc_tensor(
+            out, out_hc, residual_hc, split, low, group_tmp, low_tmp,
+            model_map, model_size, 0u, out_b_offset, 12u,
+            group_dim, rank, n_groups, out_dim, heads, n_tokens, n_hc);
+    }
+    return ds4_gpu_attention_output_q8_batch_hc_tensor(
+        out, out_hc, residual_hc, split, low, group_tmp, low_tmp,
+        model_map, model_size, 0u, out_b_offset,
+        group_dim, rank, n_groups, out_dim, heads, n_tokens, n_hc);
+}
+
+static void test_metal_batch_attn_out_hc_fusion_exact_case(
+        test_metal_attn_out_hc_kind kind) {
+    const bool q4 = kind == TEST_METAL_ATTN_OUT_HC_Q4;
+    const uint32_t n_tokens = 32u;
+    const uint32_t alloc_tokens = n_tokens + 1u;
+    const uint32_t group_dim = 4096u;
+    const uint32_t rank = 1024u;
+    const uint32_t n_groups = 8u;
+    const uint32_t low_dim = n_groups * rank;
+    const uint32_t out_dim = 4096u;
+    const uint32_t n_hc = 4u;
+    const uint64_t mix_hc = 2ull * n_hc + (uint64_t)n_hc * n_hc;
+    const uint64_t page = (uint64_t)getpagesize();
+    const uint64_t row_a_bytes = q4
+        ? (uint64_t)(group_dim / 256u) * 144u
+        : (uint64_t)(group_dim / 32u) * 34u;
+    const uint64_t row_b_bytes = q4
+        ? (uint64_t)(low_dim / 256u) * 144u
+        : (uint64_t)(low_dim / 32u) * 34u;
+    const uint64_t out_a_bytes = (uint64_t)low_dim * row_a_bytes;
+    const uint64_t out_b_offset = test_round_up_u64(out_a_bytes, page);
+    const uint64_t out_b_bytes = (uint64_t)out_dim * row_b_bytes;
+    const uint64_t model_bytes =
+        test_round_up_u64(out_b_offset + out_b_bytes, page);
+    const uint64_t heads_count =
+        (uint64_t)alloc_tokens * n_groups * group_dim;
+    const uint64_t low_count = (uint64_t)alloc_tokens * low_dim;
+    const uint64_t out_count = (uint64_t)alloc_tokens * out_dim;
+    const uint64_t hc_count =
+        (uint64_t)alloc_tokens * n_hc * out_dim;
+    const uint64_t split_count = (uint64_t)alloc_tokens * mix_hc;
+    const uint64_t active_heads_count =
+        (uint64_t)n_tokens * n_groups * group_dim;
+    const uint64_t active_low_count = (uint64_t)n_tokens * low_dim;
+    const uint64_t active_out_count = (uint64_t)n_tokens * out_dim;
+    const uint64_t active_hc_count =
+        (uint64_t)n_tokens * n_hc * out_dim;
+    const uint64_t active_split_count = (uint64_t)n_tokens * mix_hc;
+    const uint64_t heads_bytes = heads_count * sizeof(float);
+    const uint64_t low_bytes = low_count * sizeof(float);
+    const uint64_t out_bytes = out_count * sizeof(float);
+    const uint64_t hc_bytes = hc_count * sizeof(float);
+    const uint64_t split_bytes = split_count * sizeof(float);
+    const uint64_t active_heads_bytes = active_heads_count * sizeof(float);
+    const uint64_t active_low_bytes = active_low_count * sizeof(float);
+    const uint64_t active_out_bytes = active_out_count * sizeof(float);
+    const uint64_t active_hc_bytes = active_hc_count * sizeof(float);
+    const uint64_t active_split_bytes = active_split_count * sizeof(float);
+    const uint64_t scratch_bytes =
+        (uint64_t)alloc_tokens * low_dim * sizeof(uint16_t);
+    enum {
+        ref_low_tag = 1u,
+        fused_low_tag = 2u,
+        ref_out_tag = 3u,
+        fused_out_tag = 4u,
+        ref_hc_tag = 5u,
+        fused_hc_tag = 6u,
+    };
+
+    const char *disable_fusion_env =
+        "DS4_METAL_DISABLE_PRE_M5_BATCH_ATTN_OUT_HC_FUSION";
+    const char *require_q8_env =
+        "DS4_METAL_REQUIRE_PRE_M5_BATCH_ATTN_OUT_HC_FUSION";
+    const char *require_q4_env =
+        "DS4_METAL_REQUIRE_Q4_BATCH_ATTN_OUT_HC_FUSION";
+    const char *require_q4_rhs_env =
+        "DS4_METAL_REQUIRE_Q4_ATTN_OUT_B_F16_RHS";
+    const char *disable_q4_rhs_env =
+        "DS4_METAL_DISABLE_Q4_ATTN_OUT_B_F16_RHS";
+    const char *stage_profile_env = "DS4_METAL_ATTN_OUT_STAGE_PROFILE";
+    const char *q8_profile_env = "DS4_METAL_Q8_PREFILL_PROFILE";
+    char *saved_disable_fusion = test_save_env(disable_fusion_env);
+    char *saved_require_q8 = test_save_env(require_q8_env);
+    char *saved_require_q4 = test_save_env(require_q4_env);
+    char *saved_require_q4_rhs = test_save_env(require_q4_rhs_env);
+    char *saved_disable_q4_rhs = test_save_env(disable_q4_rhs_env);
+    char *saved_stage_profile = test_save_env(stage_profile_env);
+    char *saved_q8_profile = test_save_env(q8_profile_env);
+
+    void *model_raw = NULL;
+    float *heads_host = NULL;
+    float *residual_host = NULL;
+    float *split_host = NULL;
+    float *ref_low_host = NULL;
+    float *fused_low_host = NULL;
+    float *ref_out_host = NULL;
+    float *fused_out_host = NULL;
+    float *ref_hc_host = NULL;
+    float *fused_hc_host = NULL;
+    ds4_gpu_tensor *heads_base = NULL;
+    ds4_gpu_tensor *residual_base = NULL;
+    ds4_gpu_tensor *split_base = NULL;
+    ds4_gpu_tensor *ref_low_base = NULL;
+    ds4_gpu_tensor *fused_low_base = NULL;
+    ds4_gpu_tensor *ref_out_base = NULL;
+    ds4_gpu_tensor *fused_out_base = NULL;
+    ds4_gpu_tensor *ref_hc_base = NULL;
+    ds4_gpu_tensor *fused_hc_base = NULL;
+    ds4_gpu_tensor *group_tmp = NULL;
+    ds4_gpu_tensor *low_tmp = NULL;
+    ds4_gpu_tensor *heads = NULL;
+    ds4_gpu_tensor *residual = NULL;
+    ds4_gpu_tensor *split = NULL;
+    ds4_gpu_tensor *ref_low = NULL;
+    ds4_gpu_tensor *fused_low = NULL;
+    ds4_gpu_tensor *ref_out = NULL;
+    ds4_gpu_tensor *fused_out = NULL;
+    ds4_gpu_tensor *ref_hc = NULL;
+    ds4_gpu_tensor *fused_hc = NULL;
+
+    TEST_ASSERT(posix_memalign(
+                    &model_raw, (size_t)page, (size_t)model_bytes) == 0);
+    if (model_raw) {
+        memset(model_raw, 0, (size_t)model_bytes);
+        if (q4) {
+            test_fill_q4_K_weights(
+                model_raw, group_dim, low_dim, 211u);
+            test_fill_q4_K_weights(
+                (uint8_t *)model_raw + out_b_offset,
+                low_dim, out_dim, 307u);
+        } else {
+            test_fill_q8_0_weights(
+                model_raw, group_dim, low_dim, 211u);
+            test_fill_q8_0_weights(
+                (uint8_t *)model_raw + out_b_offset,
+                low_dim, out_dim, 307u);
+        }
+    }
+
+    heads_host = malloc((size_t)heads_bytes);
+    residual_host = malloc((size_t)hc_bytes);
+    split_host = malloc((size_t)split_bytes);
+    ref_low_host = malloc((size_t)low_bytes);
+    fused_low_host = malloc((size_t)low_bytes);
+    ref_out_host = malloc((size_t)out_bytes);
+    fused_out_host = malloc((size_t)out_bytes);
+    ref_hc_host = malloc((size_t)hc_bytes);
+    fused_hc_host = malloc((size_t)hc_bytes);
+    heads_base = ds4_gpu_tensor_alloc(heads_bytes);
+    residual_base = ds4_gpu_tensor_alloc(hc_bytes);
+    split_base = ds4_gpu_tensor_alloc(split_bytes);
+    ref_low_base = ds4_gpu_tensor_alloc(low_bytes);
+    fused_low_base = ds4_gpu_tensor_alloc(low_bytes);
+    ref_out_base = ds4_gpu_tensor_alloc(out_bytes);
+    fused_out_base = ds4_gpu_tensor_alloc(out_bytes);
+    ref_hc_base = ds4_gpu_tensor_alloc(hc_bytes);
+    fused_hc_base = ds4_gpu_tensor_alloc(hc_bytes);
+    group_tmp = ds4_gpu_tensor_alloc(scratch_bytes);
+    low_tmp = ds4_gpu_tensor_alloc(sizeof(float));
+    heads = heads_base
+        ? ds4_gpu_tensor_view(heads_base, 0u, active_heads_bytes) : NULL;
+    residual = residual_base
+        ? ds4_gpu_tensor_view(residual_base, 0u, active_hc_bytes) : NULL;
+    split = split_base
+        ? ds4_gpu_tensor_view(split_base, 0u, active_split_bytes) : NULL;
+    ref_low = ref_low_base
+        ? ds4_gpu_tensor_view(ref_low_base, 0u, active_low_bytes) : NULL;
+    fused_low = fused_low_base
+        ? ds4_gpu_tensor_view(fused_low_base, 0u, active_low_bytes) : NULL;
+    ref_out = ref_out_base
+        ? ds4_gpu_tensor_view(ref_out_base, 0u, active_out_bytes) : NULL;
+    fused_out = fused_out_base
+        ? ds4_gpu_tensor_view(fused_out_base, 0u, active_out_bytes) : NULL;
+    ref_hc = ref_hc_base
+        ? ds4_gpu_tensor_view(ref_hc_base, 0u, active_hc_bytes) : NULL;
+    fused_hc = fused_hc_base
+        ? ds4_gpu_tensor_view(fused_hc_base, 0u, active_hc_bytes) : NULL;
+
+    const bool allocated = model_raw && heads_host && residual_host &&
+        split_host && ref_low_host && fused_low_host && ref_out_host &&
+        fused_out_host && ref_hc_host && fused_hc_host && heads_base &&
+        residual_base && split_base && ref_low_base && fused_low_base &&
+        ref_out_base && fused_out_base && ref_hc_base && fused_hc_base &&
+        group_tmp && low_tmp && heads && residual && split && ref_low &&
+        fused_low && ref_out && fused_out && ref_hc && fused_hc;
+    TEST_ASSERT(allocated);
+
+    test_float_compare_stats low_stats = {0};
+    test_float_compare_stats hc_stats = {0};
+    size_t reject_writes = 0u;
+    size_t active_poisoned = 0u;
+    size_t guard_mismatches = 0u;
+    size_t fused_out_writes = 0u;
+    if (allocated) {
+        for (uint64_t i = 0; i < heads_count; i++) {
+            const int value =
+                (int)((i * 17u + (i ^ (i >> 7u)) * 3u +
+                       (q4 ? 19u : 7u)) % 127u) - 63;
+            heads_host[i] = (float)value / 96.0f;
+        }
+        for (uint64_t i = 0; i < hc_count; i++) {
+            const int value =
+                (int)((i * 19u + (i ^ (i >> 5u)) * 7u +
+                       (q4 ? 23u : 11u)) % 149u) - 74;
+            residual_host[i] = (float)value / 80.0f;
+        }
+        for (uint32_t t = 0; t < alloc_tokens; t++) {
+            float *row = split_host + (uint64_t)t * mix_hc;
+            for (uint32_t h = 0; h < n_hc; h++) {
+                row[h] = 0.0f;
+                row[n_hc + h] =
+                    0.55f + (float)((t + h * 3u) % 11u) / 32.0f;
+            }
+            for (uint32_t dst_hc = 0; dst_hc < n_hc; dst_hc++) {
+                for (uint32_t src_hc = 0; src_hc < n_hc; src_hc++) {
+                    const int value =
+                        (int)((t * 5u + dst_hc * 7u + src_hc * 11u +
+                               (q4 ? 3u : 0u)) % 17u) - 8;
+                    row[2u * n_hc + dst_hc * n_hc + src_hc] =
+                        (float)value / 24.0f;
+                }
+            }
+        }
+        test_metal_attn_out_hc_fill_poison(
+            ref_low_host, low_count, ref_low_tag);
+        test_metal_attn_out_hc_fill_poison(
+            fused_low_host, low_count, fused_low_tag);
+        test_metal_attn_out_hc_fill_poison(
+            ref_out_host, out_count, ref_out_tag);
+        test_metal_attn_out_hc_fill_poison(
+            fused_out_host, out_count, fused_out_tag);
+        test_metal_attn_out_hc_fill_poison(
+            ref_hc_host, hc_count, ref_hc_tag);
+        test_metal_attn_out_hc_fill_poison(
+            fused_hc_host, hc_count, fused_hc_tag);
+
+        TEST_ASSERT(ds4_gpu_tensor_write(
+                        heads_base, 0u, heads_host, heads_bytes) != 0);
+        TEST_ASSERT(ds4_gpu_tensor_write(
+                        residual_base, 0u, residual_host, hc_bytes) != 0);
+        TEST_ASSERT(ds4_gpu_tensor_write(
+                        split_base, 0u, split_host, split_bytes) != 0);
+        TEST_ASSERT(ds4_gpu_tensor_write(
+                        ref_low_base, 0u, ref_low_host, low_bytes) != 0);
+        TEST_ASSERT(ds4_gpu_tensor_write(
+                        fused_low_base, 0u, fused_low_host, low_bytes) != 0);
+        TEST_ASSERT(ds4_gpu_tensor_write(
+                        ref_out_base, 0u, ref_out_host, out_bytes) != 0);
+        TEST_ASSERT(ds4_gpu_tensor_write(
+                        fused_out_base, 0u, fused_out_host, out_bytes) != 0);
+        TEST_ASSERT(ds4_gpu_tensor_write(
+                        ref_hc_base, 0u, ref_hc_host, hc_bytes) != 0);
+        TEST_ASSERT(ds4_gpu_tensor_write(
+                        fused_hc_base, 0u, fused_hc_host, hc_bytes) != 0);
+        TEST_ASSERT(ds4_gpu_set_model_map(model_raw, model_bytes) != 0);
+
+        TEST_ASSERT(unsetenv(disable_fusion_env) == 0);
+        TEST_ASSERT(unsetenv(require_q8_env) == 0);
+        TEST_ASSERT(unsetenv(require_q4_env) == 0);
+        TEST_ASSERT(unsetenv(require_q4_rhs_env) == 0);
+        TEST_ASSERT(unsetenv(disable_q4_rhs_env) == 0);
+        TEST_ASSERT(unsetenv(stage_profile_env) == 0);
+        TEST_ASSERT(unsetenv(q8_profile_env) == 0);
+        TEST_ASSERT(setenv(
+                        q4 ? require_q4_env : require_q8_env,
+                        "1", 1) == 0);
+        if (q4) {
+            TEST_ASSERT(setenv(require_q4_rhs_env, "1", 1) == 0);
+        }
+        ds4_gpu_set_quality(false);
+        ds4_gpu_test_set_flags(q4
+            ? DS4_GPU_TEST_BATCH_ATTN_OUT_Q4_HC_FUSION
+            : DS4_GPU_TEST_BATCH_ATTN_OUT_Q8_HC_FUSION);
+
+        /* The force hook removes only the production minimum.  Neighbors of
+         * the 32-row tile must still fail closed and leave every destination
+         * byte untouched. */
+        TEST_ASSERT(test_metal_batch_attn_out_hc_fused_call(
+                        kind, fused_out_base, fused_hc_base, residual_base,
+                        split_base, fused_low_base, group_tmp, low_tmp,
+                        model_raw, model_bytes, out_b_offset, heads_base,
+                        n_tokens - 1u) == -1);
+        TEST_ASSERT(test_metal_batch_attn_out_hc_fused_call(
+                        kind, fused_out_base, fused_hc_base, residual_base,
+                        split_base, fused_low_base, group_tmp, low_tmp,
+                        model_raw, model_bytes, out_b_offset, heads_base,
+                        n_tokens + 1u) == -1);
+        TEST_ASSERT(ds4_gpu_tensor_read(
+                        fused_low_base, 0u, fused_low_host, low_bytes) != 0);
+        TEST_ASSERT(ds4_gpu_tensor_read(
+                        fused_out_base, 0u, fused_out_host, out_bytes) != 0);
+        TEST_ASSERT(ds4_gpu_tensor_read(
+                        fused_hc_base, 0u, fused_hc_host, hc_bytes) != 0);
+        reject_writes += test_metal_attn_out_hc_canary_mismatches(
+            fused_low_host, 0u, low_count, fused_low_tag);
+        reject_writes += test_metal_attn_out_hc_canary_mismatches(
+            fused_out_host, 0u, out_count, fused_out_tag);
+        reject_writes += test_metal_attn_out_hc_canary_mismatches(
+            fused_hc_host, 0u, hc_count, fused_hc_tag);
+
+        int reference_ok = 0;
+        if (q4) {
+            reference_ok = ds4_gpu_attention_output_q4_K_batch_tensor(
+                ref_out, ref_low, group_tmp, low_tmp,
+                model_raw, model_bytes, 0u, out_b_offset, 12u,
+                group_dim, rank, n_groups, out_dim, heads, n_tokens);
+        } else {
+            reference_ok = ds4_gpu_attention_output_q8_batch_tensor(
+                ref_out, ref_low, group_tmp, low_tmp,
+                model_raw, model_bytes, 0u, out_b_offset,
+                group_dim, rank, n_groups, out_dim, heads, n_tokens);
+        }
+        TEST_ASSERT(reference_ok == 1);
+        TEST_ASSERT(ds4_gpu_hc_expand_split_tensor(
+                        ref_hc, ref_out, residual, split,
+                        out_dim, n_hc) != 0);
+        TEST_ASSERT(test_metal_batch_attn_out_hc_fused_call(
+                        kind, fused_out, fused_hc, residual, split,
+                        fused_low, group_tmp, low_tmp,
+                        model_raw, model_bytes, out_b_offset, heads,
+                        n_tokens) == 1);
+
+        TEST_ASSERT(ds4_gpu_tensor_read(
+                        ref_low_base, 0u, ref_low_host, low_bytes) != 0);
+        TEST_ASSERT(ds4_gpu_tensor_read(
+                        fused_low_base, 0u, fused_low_host, low_bytes) != 0);
+        TEST_ASSERT(ds4_gpu_tensor_read(
+                        ref_out_base, 0u, ref_out_host, out_bytes) != 0);
+        TEST_ASSERT(ds4_gpu_tensor_read(
+                        fused_out_base, 0u, fused_out_host, out_bytes) != 0);
+        TEST_ASSERT(ds4_gpu_tensor_read(
+                        ref_hc_base, 0u, ref_hc_host, hc_bytes) != 0);
+        TEST_ASSERT(ds4_gpu_tensor_read(
+                        fused_hc_base, 0u, fused_hc_host, hc_bytes) != 0);
+
+        low_stats = test_compare_float_bits(
+            ref_low_host, fused_low_host, (size_t)active_low_count);
+        hc_stats = test_compare_float_bits(
+            ref_hc_host, fused_hc_host, (size_t)active_hc_count);
+        active_poisoned += test_metal_attn_out_hc_poisoned_values(
+            ref_low_host, active_low_count, ref_low_tag);
+        active_poisoned += test_metal_attn_out_hc_poisoned_values(
+            fused_low_host, active_low_count, fused_low_tag);
+        active_poisoned += test_metal_attn_out_hc_poisoned_values(
+            ref_out_host, active_out_count, ref_out_tag);
+        active_poisoned += test_metal_attn_out_hc_poisoned_values(
+            ref_hc_host, active_hc_count, ref_hc_tag);
+        active_poisoned += test_metal_attn_out_hc_poisoned_values(
+            fused_hc_host, active_hc_count, fused_hc_tag);
+        fused_out_writes = test_metal_attn_out_hc_canary_mismatches(
+            fused_out_host, 0u, out_count, fused_out_tag);
+        guard_mismatches += test_metal_attn_out_hc_canary_mismatches(
+            ref_low_host, active_low_count, low_count, ref_low_tag);
+        guard_mismatches += test_metal_attn_out_hc_canary_mismatches(
+            fused_low_host, active_low_count, low_count, fused_low_tag);
+        guard_mismatches += test_metal_attn_out_hc_canary_mismatches(
+            ref_out_host, active_out_count, out_count, ref_out_tag);
+        guard_mismatches += test_metal_attn_out_hc_canary_mismatches(
+            ref_hc_host, active_hc_count, hc_count, ref_hc_tag);
+        guard_mismatches += test_metal_attn_out_hc_canary_mismatches(
+            fused_hc_host, active_hc_count, hc_count, fused_hc_tag);
+    }
+
+    fprintf(stderr,
+            "ds4-test: Metal %s batch attention-output HC exact N=%u "
+            "low=%zu/%llu hc=%zu/%llu max_ulp=%u/%u "
+            "active_poison=%zu guard=%zu fused_out_writes=%zu "
+            "reject_writes=%zu\n",
+            q4 ? "Q4_K/F16-RHS" : "Q8_0",
+            n_tokens,
+            low_stats.mismatch_count,
+            (unsigned long long)active_low_count,
+            hc_stats.mismatch_count,
+            (unsigned long long)active_hc_count,
+            low_stats.max_ulp,
+            hc_stats.max_ulp,
+            active_poisoned,
+            guard_mismatches,
+            fused_out_writes,
+            reject_writes);
+    TEST_ASSERT(low_stats.mismatch_count == 0u && low_stats.max_ulp == 0u);
+    TEST_ASSERT(hc_stats.mismatch_count == 0u && hc_stats.max_ulp == 0u);
+    TEST_ASSERT(active_poisoned == 0u);
+    TEST_ASSERT(guard_mismatches == 0u);
+    TEST_ASSERT(fused_out_writes == 0u);
+    TEST_ASSERT(reject_writes == 0u);
+
+    ds4_gpu_test_set_flags(0u);
+    ds4_gpu_set_quality(false);
+    test_restore_env(q8_profile_env, saved_q8_profile);
+    test_restore_env(stage_profile_env, saved_stage_profile);
+    test_restore_env(disable_q4_rhs_env, saved_disable_q4_rhs);
+    test_restore_env(require_q4_rhs_env, saved_require_q4_rhs);
+    test_restore_env(require_q4_env, saved_require_q4);
+    test_restore_env(require_q8_env, saved_require_q8);
+    test_restore_env(disable_fusion_env, saved_disable_fusion);
+    ds4_gpu_tensor_free(fused_hc);
+    ds4_gpu_tensor_free(ref_hc);
+    ds4_gpu_tensor_free(fused_out);
+    ds4_gpu_tensor_free(ref_out);
+    ds4_gpu_tensor_free(fused_low);
+    ds4_gpu_tensor_free(ref_low);
+    ds4_gpu_tensor_free(split);
+    ds4_gpu_tensor_free(residual);
+    ds4_gpu_tensor_free(heads);
+    ds4_gpu_tensor_free(low_tmp);
+    ds4_gpu_tensor_free(group_tmp);
+    ds4_gpu_tensor_free(fused_hc_base);
+    ds4_gpu_tensor_free(ref_hc_base);
+    ds4_gpu_tensor_free(fused_out_base);
+    ds4_gpu_tensor_free(ref_out_base);
+    ds4_gpu_tensor_free(fused_low_base);
+    ds4_gpu_tensor_free(ref_low_base);
+    ds4_gpu_tensor_free(split_base);
+    ds4_gpu_tensor_free(residual_base);
+    ds4_gpu_tensor_free(heads_base);
+    free(fused_hc_host);
+    free(ref_hc_host);
+    free(fused_out_host);
+    free(ref_out_host);
+    free(fused_low_host);
+    free(ref_low_host);
+    free(split_host);
+    free(residual_host);
+    free(heads_host);
+    free(model_raw);
+}
+
+static void test_metal_batch_attn_out_hc_fusion_exact(void) {
+    test_metal_batch_attn_out_hc_fusion_exact_case(
+        TEST_METAL_ATTN_OUT_HC_Q8);
+    test_metal_batch_attn_out_hc_fusion_exact_case(
+        TEST_METAL_ATTN_OUT_HC_Q4);
+}
+
 static void test_metal_hc_split_weighted_sum_norm_batch_exact(void) {
     /* Compare the batched HC+RMSNorm fusion against the exact two-dispatch
      * sequence used by the reference path at DS4's production dimensions. */
@@ -3836,6 +7050,194 @@ static void test_metal_hc_split_weighted_sum_norm_batch_exact(void) {
     free(model_raw);
 }
 
+static void test_metal_hc_producer_pre_norm_compound_exact(void) {
+    const uint32_t n = 16384u;
+    const uint32_t mix_dim = 24u;
+    const uint32_t n_embd = 4096u;
+    const uint32_t n_hc = 4u;
+    const uint32_t sinkhorn_iters = 20u;
+    const float eps = 1.0e-6f;
+    const uint64_t page = (uint64_t)getpagesize();
+    const uint64_t weight_offset = page;
+    const uint64_t weight_bytes =
+        (uint64_t)n * mix_dim * sizeof(uint16_t);
+    const uint64_t scale_offset =
+        test_round_up_u64(weight_offset + weight_bytes, page);
+    const uint64_t base_offset =
+        test_round_up_u64(scale_offset + 3u*sizeof(float), page);
+    const uint64_t norm_offset =
+        test_round_up_u64(base_offset + mix_dim*sizeof(float), page);
+    const uint64_t model_bytes =
+        test_round_up_u64(norm_offset + n_embd*sizeof(float), page);
+    const uint64_t residual_bytes = (uint64_t)n*sizeof(float);
+    const uint64_t mix_bytes = (uint64_t)mix_dim*sizeof(float);
+    const uint64_t out_bytes = (uint64_t)n_embd*sizeof(float);
+
+    const char *force_env =
+        "DS4_METAL_ENABLE_HC_PRODUCER_PRE_NORM_FUSE";
+    const char *disable_env =
+        "DS4_METAL_DISABLE_HC_PRODUCER_PRE_NORM_FUSE";
+    const char *disable_pre_env =
+        "DS4_METAL_DISABLE_PRE_M5_HC_PRODUCER_PRE_NORM_FUSE";
+    const char *disable_m5_env =
+        "DS4_METAL_DISABLE_M5_HC_PRODUCER_PRE_NORM_FUSE";
+    const char *disable_ports_env =
+        "DS4_METAL_DISABLE_PRE_M5_DECODE_PORTS";
+    char *saved_force = test_save_env(force_env);
+    char *saved_disable = test_save_env(disable_env);
+    char *saved_disable_pre = test_save_env(disable_pre_env);
+    char *saved_disable_m5 = test_save_env(disable_m5_env);
+    char *saved_disable_ports = test_save_env(disable_ports_env);
+
+    void *model_raw = NULL;
+    float *residual_host = NULL;
+    float *ref_host = NULL;
+    float *fused_host = NULL;
+    ds4_gpu_tensor *residual = NULL;
+    ds4_gpu_tensor *ref_mix = NULL;
+    ds4_gpu_tensor *fused_mix = NULL;
+    ds4_gpu_tensor *ref_split = NULL;
+    ds4_gpu_tensor *fused_split = NULL;
+    ds4_gpu_tensor *ref_out = NULL;
+    ds4_gpu_tensor *fused_out = NULL;
+    ds4_gpu_tensor *ref_norm = NULL;
+    ds4_gpu_tensor *fused_norm = NULL;
+
+    TEST_ASSERT(setenv(force_env, "1", 1) == 0);
+    TEST_ASSERT(unsetenv(disable_env) == 0);
+    TEST_ASSERT(unsetenv(disable_pre_env) == 0);
+    TEST_ASSERT(unsetenv(disable_m5_env) == 0);
+    TEST_ASSERT(unsetenv(disable_ports_env) == 0);
+    TEST_ASSERT(posix_memalign(
+                    &model_raw, (size_t)page, (size_t)model_bytes) == 0);
+    if (!model_raw) goto cleanup;
+    memset(model_raw, 0, (size_t)model_bytes);
+
+    uint16_t *weight =
+        (uint16_t *)((uint8_t *)model_raw + weight_offset);
+    float *hc_scale = (float *)((uint8_t *)model_raw + scale_offset);
+    float *hc_base = (float *)((uint8_t *)model_raw + base_offset);
+    float *norm_weight = (float *)((uint8_t *)model_raw + norm_offset);
+    for (uint32_t o = 0; o < mix_dim; o++) {
+        for (uint32_t i = 0; i < n; i++) {
+            const uint32_t key = i*37u + o*1009u + ((i >> 3u) ^ (o*19u));
+            const int value = (int)(key % 127u) - 63;
+            weight[(uint64_t)o*n + i] =
+                test_float_to_f16((float)value/256.0f);
+        }
+    }
+    hc_scale[0] = 0.625f;
+    hc_scale[1] = -0.75f;
+    hc_scale[2] = 0.4375f;
+    for (uint32_t i = 0; i < mix_dim; i++) {
+        const int value = (int)((i*17u + 5u) % 29u) - 14;
+        hc_base[i] = (float)value/32.0f;
+    }
+    for (uint32_t i = 0; i < n_embd; i++) {
+        norm_weight[i] = 0.5f + (float)((i*13u + 7u) % 31u)/32.0f;
+    }
+
+    residual_host = malloc((size_t)residual_bytes);
+    ref_host = malloc((size_t)out_bytes);
+    fused_host = malloc((size_t)out_bytes);
+    TEST_ASSERT(residual_host != NULL);
+    TEST_ASSERT(ref_host != NULL);
+    TEST_ASSERT(fused_host != NULL);
+    if (!residual_host || !ref_host || !fused_host) goto cleanup;
+    for (uint32_t i = 0; i < n; i++) {
+        const uint32_t key = i*131u + ((i >> 4u) ^ (i*7u));
+        const int value = (int)(key % 4093u) - 2046;
+        residual_host[i] = (float)value/1024.0f;
+    }
+
+    residual = ds4_gpu_tensor_alloc(residual_bytes);
+    ref_mix = ds4_gpu_tensor_alloc(mix_bytes);
+    fused_mix = ds4_gpu_tensor_alloc(mix_bytes);
+    ref_split = ds4_gpu_tensor_alloc(mix_bytes);
+    fused_split = ds4_gpu_tensor_alloc(mix_bytes);
+    ref_out = ds4_gpu_tensor_alloc(out_bytes);
+    fused_out = ds4_gpu_tensor_alloc(out_bytes);
+    ref_norm = ds4_gpu_tensor_alloc(out_bytes);
+    fused_norm = ds4_gpu_tensor_alloc(out_bytes);
+    TEST_ASSERT(residual && ref_mix && fused_mix && ref_split && fused_split &&
+                ref_out && fused_out && ref_norm && fused_norm);
+    if (!residual || !ref_mix || !fused_mix || !ref_split || !fused_split ||
+        !ref_out || !fused_out || !ref_norm || !fused_norm) {
+        goto cleanup;
+    }
+
+    TEST_ASSERT(ds4_gpu_tensor_write(
+                    residual, 0, residual_host, residual_bytes) != 0);
+    TEST_ASSERT(ds4_gpu_set_model_map(model_raw, model_bytes) != 0);
+    TEST_ASSERT(ds4_gpu_hc_rms_norm_mix_f16_tensor(
+                    ref_mix, residual, model_raw, model_bytes,
+                    weight_offset, n, mix_dim, eps) != 0);
+    TEST_ASSERT(ds4_gpu_hc_split_weighted_sum_norm_tensor(
+                    ref_out, ref_norm, ref_split, ref_mix, residual,
+                    model_raw, model_bytes, scale_offset, base_offset,
+                    norm_offset, n_embd, n_hc, sinkhorn_iters,
+                    eps, eps) != 0);
+    TEST_ASSERT(ds4_gpu_hc_rms_norm_mix_split_norm_f16_available() != 0);
+    TEST_ASSERT(ds4_gpu_hc_rms_norm_mix_split_norm_f16_tensor(
+                    fused_mix, fused_out, fused_norm, fused_split, residual,
+                    model_raw, model_bytes, weight_offset, scale_offset,
+                    base_offset, norm_offset, n, mix_dim, n_embd, n_hc,
+                    sinkhorn_iters, eps, eps, eps) > 0);
+
+    test_float_compare_stats mix_stats = {0};
+    test_float_compare_stats split_stats = {0};
+    test_float_compare_stats out_stats = {0};
+    test_float_compare_stats norm_stats = {0};
+#define TEST_HC_COMPOUND_COMPARE(ref_, fused_, bytes_, count_, stats_) do { \
+        TEST_ASSERT(ds4_gpu_tensor_read((ref_), 0, ref_host, (bytes_)) != 0); \
+        TEST_ASSERT(ds4_gpu_tensor_read((fused_), 0, fused_host, (bytes_)) != 0); \
+        (stats_) = test_compare_float_bits(ref_host, fused_host, (count_)); \
+    } while (0)
+    TEST_HC_COMPOUND_COMPARE(
+        ref_mix, fused_mix, mix_bytes, mix_dim, mix_stats);
+    TEST_HC_COMPOUND_COMPARE(
+        ref_split, fused_split, mix_bytes, mix_dim, split_stats);
+    TEST_HC_COMPOUND_COMPARE(
+        ref_out, fused_out, out_bytes, n_embd, out_stats);
+    TEST_HC_COMPOUND_COMPARE(
+        ref_norm, fused_norm, out_bytes, n_embd, norm_stats);
+#undef TEST_HC_COMPOUND_COMPARE
+
+    fprintf(stderr,
+            "ds4-test: HC producer/pre-norm compound exact "
+            "mix=%zu/%u split=%zu/%u collapse=%zu/%u norm=%zu/%u\n",
+            mix_stats.mismatch_count, mix_dim,
+            split_stats.mismatch_count, mix_dim,
+            out_stats.mismatch_count, n_embd,
+            norm_stats.mismatch_count, n_embd);
+    TEST_ASSERT(mix_stats.mismatch_count == 0 && mix_stats.max_ulp == 0);
+    TEST_ASSERT(split_stats.mismatch_count == 0 && split_stats.max_ulp == 0);
+    TEST_ASSERT(out_stats.mismatch_count == 0 && out_stats.max_ulp == 0);
+    TEST_ASSERT(norm_stats.mismatch_count == 0 && norm_stats.max_ulp == 0);
+    TEST_ASSERT(setenv(disable_env, "1", 1) == 0);
+    TEST_ASSERT(ds4_gpu_hc_rms_norm_mix_split_norm_f16_available() == 0);
+
+cleanup:
+    ds4_gpu_tensor_free(fused_norm);
+    ds4_gpu_tensor_free(ref_norm);
+    ds4_gpu_tensor_free(fused_out);
+    ds4_gpu_tensor_free(ref_out);
+    ds4_gpu_tensor_free(fused_split);
+    ds4_gpu_tensor_free(ref_split);
+    ds4_gpu_tensor_free(fused_mix);
+    ds4_gpu_tensor_free(ref_mix);
+    ds4_gpu_tensor_free(residual);
+    free(fused_host);
+    free(ref_host);
+    free(residual_host);
+    free(model_raw);
+    test_restore_env(disable_ports_env, saved_disable_ports);
+    test_restore_env(disable_m5_env, saved_disable_m5);
+    test_restore_env(disable_pre_env, saved_disable_pre);
+    test_restore_env(disable_env, saved_disable);
+    test_restore_env(force_env, saved_force);
+}
+
 static void test_metal_output_hc_weights4_exact(void) {
     const uint32_t n_hc = 4;
     const float eps = 1.0e-6f;
@@ -4722,6 +8124,14 @@ static void test_metal_router_weights_batch_exact(void) {
 #endif
 
 static void test_metal_kernel_group(void) {
+#if defined(__APPLE__)
+    if (test_env_bool("DS4_TEST_METAL_RESIDENT_ORACLES_ONLY")) {
+        test_metal_flush_commands_progress_exact();
+        test_metal_small_prefill_direct_exact();
+        test_metal_batch_attn_out_hc_fusion_exact();
+        return;
+    }
+#endif
     test_metal_f16_matvec_fast_nr0_4();
     test_metal_f16_prefill_matmul();
     test_metal_q8_0_prefill_matmul();
@@ -4730,18 +8140,29 @@ static void test_metal_kernel_group(void) {
     test_dspark_cache_window_crop();
     test_metal_q8_0_decode_pair_exact();
 #if defined(__APPLE__)
+    test_metal_flush_commands_progress_exact();
+    test_metal_q8_0_decode_rows_exact();
+    test_metal_q8_attention_output_static_batch_exact();
+    test_metal_q4_attention_output_tiny_batch_exact();
+    test_metal_dspark_device_proposer_q8();
     test_metal_f16_compressor_pair_state_store_exact();
+    test_metal_f16_compressor_quad_state_store_exact();
+    test_metal_q8_qkv_compressor_compound_exact();
     test_metal_compressor_ape_add_exact();
     test_metal_compressor_ratio4_pack_exact();
     test_metal_compressor_ratio4_replay_pack_exact();
     test_metal_compressor_ratio4_direct_pool_exact();
+    test_metal_compressor_ratio4_exact_pool_decode();
     test_metal_inplace_rope_pair_exact();
     test_metal_contiguous_f32_f16_roundtrip_exact();
     test_metal_gathered_kv_stage_exact();
     test_metal_contiguous_compressed_f16_attention_exact();
     test_metal_persistent_zero_attention_mask_exact();
     test_metal_zero_prefix_prefill_mask_cache_exact();
+    test_metal_small_prefill_direct_exact();
+    test_metal_batch_attn_out_hc_fusion_exact();
     test_metal_hc_split_weighted_sum_norm_batch_exact();
+    test_metal_hc_producer_pre_norm_compound_exact();
     test_metal_output_hc_weights4_exact();
     test_metal_hc_rms_scale_project_f16_exact();
     test_metal_router_simd_finalize_exact();
@@ -6731,9 +10152,6 @@ static void test_dspark_verify_depth(void) {
         return;
     }
 
-    char *saved_scheduler = test_save_env("DS4_DSPARK_SCHEDULER");
-    setenv("DS4_DSPARK_SCHEDULER", "0", 1);
-
     ds4_engine *engine = test_open_dspark_engine(support);
     ds4_tokens prompt = {0};
     int *spec = NULL;
@@ -6776,7 +10194,6 @@ static void test_dspark_verify_depth(void) {
     free(spec);
     ds4_tokens_free(&prompt);
     ds4_engine_close(engine);
-    test_restore_env("DS4_DSPARK_SCHEDULER", saved_scheduler);
 }
 #endif
 
@@ -6832,7 +10249,7 @@ static void test_print_help(const char *prog) {
     puts("\nEnvironment:");
     puts("  DS4_TEST_MODEL=FILE        Model path. Default: ds4flash.gguf");
     puts("  DS4_TEST_BACKEND=cpu       Run model tests on CPU instead of Metal/CUDA.");
-    puts("  DS4_TEST_SSD_STREAMING=1   Run model tests through Metal SSD streaming.");
+    puts("  DS4_TEST_SSD_STREAMING=1   Run model tests through backend SSD streaming.");
     puts("  DS4_TEST_SSD_STREAMING_CACHE_GB=N  Streaming routed expert cache in GiB.");
     puts("  DS4_TEST_SSD_STREAMING_CACHE_EXPERTS=N  Streaming routed expert cache count.");
     puts("  DS4_TEST_SSD_STREAMING_COLD=1  Skip streaming hot expert preload.");
@@ -6844,6 +10261,8 @@ static void test_print_help(const char *prog) {
     puts("  DS4_TEST_VECTOR_FILE=FILE  Official fixture. Default: flash-0731/official.vec.");
     puts("  DS4_TEST_LOCAL_GOLDEN_FILE=FILE  Local fixture. Default: flash-0731/local-golden.vec.");
     puts("  DS4_TEST_MPP_EQ_CASE=NAME  Run only Tensor equivalence cases whose id contains NAME.");
+    puts("  DS4_TEST_METAL_RESIDENT_ORACLES_ONLY=1  Restrict --metal-kernels to resident optimization oracles.");
+    puts("  DS4_TEST_METAL_SMALL_PREFILL_TIMING=1   Include the small-prefill GPU-only microbenchmark.");
     puts("  DS4_TEST_MTP=FILE         Legacy MTP support GGUF for --mtp-verify-depth.");
     puts("  DS4_TEST_DSPARK=FILE      DSpark support GGUF for --dspark-verify-depth.");
     puts("  DS4_TEST_CONTINUED_PREFILL_TOKENS=N  Large suffix size for --glm53-continued-prefill.");
diff --git a/tests/dspark_acceptance_fixture.sh b/tests/dspark_acceptance_fixture.sh
index 07b39f3dc9..4cac4f7d54 100644
--- a/tests/dspark_acceptance_fixture.sh
+++ b/tests/dspark_acceptance_fixture.sh
@@ -23,6 +23,178 @@ fi
 partial_cases=0
 direct_partial_cases=0
 direct_commits=0
+BACKEND=${DS4_DSPARK_FIXTURE_BACKEND:-auto}
+SSD_STREAMING=${DS4_DSPARK_FIXTURE_SSD_STREAMING:-0}
+SSD_CACHE_EXPERTS=${DS4_DSPARK_FIXTURE_SSD_STREAMING_CACHE_EXPERTS:-}
+REQUIRE_ACTIVE=${DS4_DSPARK_FIXTURE_REQUIRE_ACTIVE:-1}
+REQUIRE_EXACT2=${DS4_DSPARK_FIXTURE_REQUIRE_EXACT2:-0}
+REQUIRE_CUDA_EXACTN=${DS4_DSPARK_FIXTURE_REQUIRE_CUDA_EXACTN:-0}
+REQUIRE_CUDA_EXACTN_BATCH_HEAD=${DS4_DSPARK_FIXTURE_REQUIRE_CUDA_EXACTN_BATCH_HEAD:-0}
+REQUIRE_CUDA_EXACTN_GRAPHS=${DS4_DSPARK_FIXTURE_REQUIRE_CUDA_EXACTN_GRAPHS:-0}
+REQUIRE_CUDA_DEVICE_PROPOSER=${DS4_DSPARK_FIXTURE_REQUIRE_CUDA_DEVICE_PROPOSER:-0}
+REQUIRE_METAL_EXACTN_BATCH_HEAD=${DS4_DSPARK_FIXTURE_REQUIRE_METAL_EXACTN_BATCH_HEAD:-0}
+REQUIRE_METAL_EXACTN_PARTIAL=${DS4_DSPARK_FIXTURE_REQUIRE_METAL_EXACTN_PARTIAL:-0}
+REQUIRE_METAL_DEVICE_PROPOSER=${DS4_DSPARK_FIXTURE_REQUIRE_METAL_DEVICE_PROPOSER:-0}
+total_proposed=0
+total_accepted_draft=0
+total_exact2_attempt=0
+total_exact2_fallback=0
+total_cuda_exactn_attempt=0
+total_cuda_exactn_fallback=0
+total_cuda_exactn_error_fallback=0
+total_cuda_exactn_batch_head_attempt=0
+total_cuda_exactn_batch_head_use=0
+total_cuda_exactn_batch_head_fallback=0
+total_cuda_exactn_graph_attempt=0
+total_cuda_exactn_graph_use=0
+total_cuda_exactn_graph_capture=0
+total_cuda_exactn_graph_replay=0
+total_cuda_exactn_graph_warm=0
+total_cuda_exactn_graph_no_slot=0
+total_cuda_exactn_graph_failure=0
+total_cuda_device_proposer_attempt=0
+total_cuda_device_proposer_use=0
+total_cuda_device_proposer_fallback=0
+total_cuda_device_proposer_policy_mismatch=0
+total_exactn_union_error_fallback=0
+total_exactn_union_partial_replay=0
+total_exactn_union_verify_skip=0
+total_metal_exactn_batch_head_attempt=0
+total_metal_exactn_batch_head_use=0
+total_metal_exactn_batch_head_fallback=0
+total_metal_device_proposer_attempt=0
+total_metal_device_proposer_use=0
+total_metal_device_proposer_fallback=0
+total_metal_device_proposer_policy_mismatch=0
+
+stats_field() {
+    printf '%s\n' "$1" | awk -v key="$2" '
+        { prefix = key "="
+          for (i = 1; i <= NF; i++) {
+              if (index($i, prefix) == 1) {
+                  print substr($i, length(prefix) + 1)
+                  exit
+              }
+          }
+        }'
+}
+
+case "$BACKEND" in
+auto|metal|cuda|rocm) ;;
+*)
+    echo "dspark-fixture: invalid DS4_DSPARK_FIXTURE_BACKEND=$BACKEND" >&2
+    exit 1
+    ;;
+esac
+case "$SSD_STREAMING" in
+0|1) ;;
+*)
+    echo "dspark-fixture: DS4_DSPARK_FIXTURE_SSD_STREAMING must be 0 or 1" >&2
+    exit 1
+    ;;
+esac
+case "$REQUIRE_ACTIVE" in
+0|1) ;;
+*)
+    echo "dspark-fixture: DS4_DSPARK_FIXTURE_REQUIRE_ACTIVE must be 0 or 1" >&2
+    exit 1
+    ;;
+esac
+case "$REQUIRE_EXACT2" in
+0|1) ;;
+*)
+    echo "dspark-fixture: DS4_DSPARK_FIXTURE_REQUIRE_EXACT2 must be 0 or 1" >&2
+    exit 1
+    ;;
+esac
+case "$REQUIRE_CUDA_EXACTN" in
+0|1) ;;
+*)
+    echo "dspark-fixture: DS4_DSPARK_FIXTURE_REQUIRE_CUDA_EXACTN must be 0 or 1" >&2
+    exit 1
+    ;;
+esac
+case "$REQUIRE_CUDA_EXACTN_BATCH_HEAD" in
+0|1) ;;
+*)
+    echo "dspark-fixture: DS4_DSPARK_FIXTURE_REQUIRE_CUDA_EXACTN_BATCH_HEAD must be 0 or 1" >&2
+    exit 1
+    ;;
+esac
+if [ "$REQUIRE_CUDA_EXACTN_BATCH_HEAD" != 0 ]; then
+    REQUIRE_CUDA_EXACTN=1
+fi
+case "$REQUIRE_CUDA_EXACTN_GRAPHS" in
+0|1) ;;
+*)
+    echo "dspark-fixture: DS4_DSPARK_FIXTURE_REQUIRE_CUDA_EXACTN_GRAPHS must be 0 or 1" >&2
+    exit 1
+    ;;
+esac
+if [ "$REQUIRE_CUDA_EXACTN_GRAPHS" != 0 ]; then
+    REQUIRE_CUDA_EXACTN=1
+fi
+case "$REQUIRE_CUDA_DEVICE_PROPOSER" in
+0|1) ;;
+*)
+    echo "dspark-fixture: DS4_DSPARK_FIXTURE_REQUIRE_CUDA_DEVICE_PROPOSER must be 0 or 1" >&2
+    exit 1
+    ;;
+esac
+case "$REQUIRE_METAL_EXACTN_BATCH_HEAD" in
+0|1) ;;
+*)
+    echo "dspark-fixture: DS4_DSPARK_FIXTURE_REQUIRE_METAL_EXACTN_BATCH_HEAD must be 0 or 1" >&2
+    exit 1
+    ;;
+esac
+case "$REQUIRE_METAL_EXACTN_PARTIAL" in
+0|1) ;;
+*)
+    echo "dspark-fixture: DS4_DSPARK_FIXTURE_REQUIRE_METAL_EXACTN_PARTIAL must be 0 or 1" >&2
+    exit 1
+    ;;
+esac
+case "$REQUIRE_METAL_DEVICE_PROPOSER" in
+0|1) ;;
+*)
+    echo "dspark-fixture: DS4_DSPARK_FIXTURE_REQUIRE_METAL_DEVICE_PROPOSER must be 0 or 1" >&2
+    exit 1
+    ;;
+esac
+case "$REQUIRE_DIRECT" in
+0|1) ;;
+*)
+    echo "dspark-fixture: DS4_DSPARK_FIXTURE_REQUIRE_DIRECT_COMMIT must be 0 or 1" >&2
+    exit 1
+    ;;
+esac
+case "$REQUIRE_IDENTICAL" in
+0|1) ;;
+*)
+    echo "dspark-fixture: DS4_DSPARK_FIXTURE_REQUIRE_IDENTICAL must be 0 or 1" >&2
+    exit 1
+    ;;
+esac
+if [ "$REQUIRE_EXACT2" != 0 ] || [ "$REQUIRE_CUDA_EXACTN" != 0 ] ||
+   [ "$REQUIRE_CUDA_DEVICE_PROPOSER" != 0 ] ||
+   [ "$REQUIRE_METAL_EXACTN_BATCH_HEAD" != 0 ] ||
+   [ "$REQUIRE_METAL_EXACTN_PARTIAL" != 0 ] ||
+   [ "$REQUIRE_METAL_DEVICE_PROPOSER" != 0 ]; then
+    REQUIRE_IDENTICAL=1
+fi
+case "$SSD_CACHE_EXPERTS" in
+""|*[!0-9]*)
+    if [ -n "$SSD_CACHE_EXPERTS" ]; then
+        echo "dspark-fixture: invalid SSD streaming expert count $SSD_CACHE_EXPERTS" >&2
+        exit 1
+    fi
+    ;;
+esac
+if [ "$SSD_STREAMING" = 0 ] && [ -n "$SSD_CACHE_EXPERTS" ]; then
+    echo "dspark-fixture: SSD cache experts requires DS4_DSPARK_FIXTURE_SSD_STREAMING=1" >&2
+    exit 1
+fi
 
 proposal_quality_guard_enabled() {
     case "$PROPOSAL_QUALITY_GUARD" in
@@ -57,11 +229,6 @@ if proposal_quality_guard_enabled; then
     PROPOSAL_QUALITY_GUARD_ACTIVE=1
 fi
 
-if [ "$REQUIRE_PARTIAL" != 0 ] && [ "${DS4_DSPARK_SCHEDULER_TAIL_MIN_TOKENS+x}" != x ]; then
-    DS4_DSPARK_SCHEDULER_TAIL_MIN_TOKENS=0
-    export DS4_DSPARK_SCHEDULER_TAIL_MIN_TOKENS
-fi
-
 file_bytes() {
     if stat -L -f %z "$1" >/dev/null 2>&1; then
         stat -L -f %z "$1"
@@ -89,12 +256,28 @@ print_metadata() {
     hw_model=$(sysctl -n hw.model 2>/dev/null || true)
     hw_cpu=$(sysctl -n machdep.cpu.brand_string 2>/dev/null || true)
     confidence=${CONFIDENCE:-default}
-    scheduler=${DS4_DSPARK_SCHEDULER:-default}
-    no_draft_skip=${DS4_DSPARK_SCHEDULER_NO_DRAFT_SKIP:-default}
-    short_accept_skip=${DS4_DSPARK_SCHEDULER_SHORT_ACCEPT_NO_DRAFT_SKIP:-default}
-    cold_low_conf_skip=${DS4_DSPARK_SCHEDULER_COLD_LOW_CONFIDENCE_SKIP:-default}
-    cold_low_conf_milli=${DS4_DSPARK_SCHEDULER_COLD_LOW_CONFIDENCE_MILLI:-default}
-    tail_min_tokens=${DS4_DSPARK_SCHEDULER_TAIL_MIN_TOKENS:-default}
+    exact2_cuda=${DS4_CUDA_DSPARK_EXACT2:-unset}
+    exact2_metal=${DS4_METAL_DSPARK_EXACT2:-unset}
+    exactn_cuda=${DS4_CUDA_DSPARK_EXACTN:-unset}
+    exactn_cuda_disable=${DS4_CUDA_DISABLE_DSPARK_EXACTN:-unset}
+    exactn_cuda_batch_head=${DS4_CUDA_DSPARK_EXACTN_BATCH_HEAD:-unset}
+    exactn_cuda_batch_head_disable=${DS4_CUDA_DISABLE_DSPARK_EXACTN_BATCH_HEAD:-unset}
+    exactn_cuda_graphs=${DS4_CUDA_DSPARK_EXACTN_GRAPHS:-unset}
+    exactn_cuda_graphs_disable=${DS4_CUDA_DISABLE_DSPARK_EXACTN_GRAPHS:-unset}
+    cuda_device_proposer=${DS4_CUDA_DSPARK_DEVICE_PROPOSER:-unset}
+    cuda_device_proposer_disable=${DS4_CUDA_DSPARK_NO_DEVICE_PROPOSER:-unset}
+    exactn_union_metal=${DS4_METAL_DSPARK_EXACTN_UNION:-unset}
+    exactn_metal_batch_head=${DS4_METAL_DSPARK_EXACTN_BATCH_HEAD:-unset}
+    exactn_metal_batch_head_disable=${DS4_METAL_DISABLE_DSPARK_EXACTN_BATCH_HEAD:-unset}
+    metal_device_proposer=${DS4_METAL_DSPARK_DEVICE_PROPOSER:-unset}
+    metal_device_proposer_disable=${DS4_METAL_DSPARK_NO_DEVICE_PROPOSER:-unset}
+    noncausal_online_cuda=${DS4_CUDA_ENABLE_DSPARK_NONCAUSAL_ONLINE:-unset}
+    noncausal_online_cuda_disable=${DS4_CUDA_DISABLE_DSPARK_NONCAUSAL_ONLINE:-unset}
+    verify_noncausal=${DS4_DSPARK_VERIFY_NONCAUSAL:-unset}
+    exact_rows_async_tails_metal=${DS4_METAL_DSPARK_EXACT_ROWS_ASYNC_TAILS:-unset}
+    proposer_cap_cuda=${DS4_CUDA_DSPARK_PROPOSER_BLOCK_MAX:-unset}
+    proposer_cap_metal=${DS4_METAL_DSPARK_PROPOSER_BLOCK_MAX:-unset}
+    verifier_cap=${DS4_DSPARK_SSD_VERIFY_BLOCK_MAX:-unset}
 
     printf '# commit=%s\n' "$(git_commit_label)"
     printf '# hardware_os=%s hardware_model=%s hardware_cpu=%s\n' \
@@ -102,19 +285,39 @@ print_metadata() {
     printf '# model=%s model_bytes=%s support=%s support_bytes=%s\n' \
         "$MODEL" "$(file_bytes "$MODEL")" \
         "$SUPPORT" "$(file_bytes "$SUPPORT")"
-    printf '# tokens=%s ctx=default flags="--temp %s --top-p %s --min-p %s --seed %s --nothink" exact_sampling=%s confidence=%s scheduler=%s no_draft_skip=%s short_accept_no_draft_skip=%s cold_low_confidence_skip=%s cold_low_confidence_milli=%s tail_min_tokens=%s proposal_quality_guard=%s proposal_quality_active=%s c_add_min_accepted=%s require_direct=%s require_identical=%s\n' \
+    printf '# tokens=%s ctx=default flags="--temp %s --top-p %s --min-p %s --seed %s --nothink" exact_sampling=%s confidence=%s proposal_quality_guard=%s proposal_quality_active=%s c_add_min_accepted=%s require_direct=%s require_identical=%s\n' \
         "$TOKENS" "$TEMPERATURE" "$TOP_P" "$MIN_P" "$SEED" \
-        "$EXACT_SAMPLING" "$confidence" "$scheduler" "$no_draft_skip" \
-        "$short_accept_skip" "$cold_low_conf_skip" "$cold_low_conf_milli" \
-        "$tail_min_tokens" "$PROPOSAL_QUALITY_GUARD" \
+        "$EXACT_SAMPLING" "$confidence" "$PROPOSAL_QUALITY_GUARD" \
         "$PROPOSAL_QUALITY_GUARD_ACTIVE" "$C_ADD_MIN_ACCEPTED" \
         "$REQUIRE_DIRECT" "$REQUIRE_IDENTICAL"
+    printf '# backend=%s ssd_streaming=%s ssd_cache_experts=%s require_active=%s\n' \
+        "$BACKEND" "$SSD_STREAMING" "${SSD_CACHE_EXPERTS:-auto}" "$REQUIRE_ACTIVE"
     printf '# baseline_command=%s -m %s --tokens %s --temp %s --top-p %s --min-p %s --seed %s --nothink -p <fixture-prompt>\n' \
         "$DS4_BIN" "$MODEL" "$TOKENS" "$TEMPERATURE" "$TOP_P" "$MIN_P" "$SEED"
     printf '# dspark_command=DS4_DSPARK_STATS=1 %s --dspark%s%s -m %s --mtp-model %s --tokens %s --temp %s --top-p %s --min-p %s --seed %s --nothink -p <fixture-prompt>\n' \
         "$DS4_BIN" "${exact_sampling_arg:+ $exact_sampling_arg}" \
         "${CONFIDENCE:+ --dspark-confidence $CONFIDENCE}" \
         "$MODEL" "$SUPPORT" "$TOKENS" "$TEMPERATURE" "$TOP_P" "$MIN_P" "$SEED"
+    printf '# exact2_cuda=%s exact2_metal=%s proposer_block_max_cuda=%s proposer_block_max_metal=%s verifier_block_max=%s require_exact2=%s\n' \
+        "$exact2_cuda" "$exact2_metal" "$proposer_cap_cuda" \
+        "$proposer_cap_metal" "$verifier_cap" "$REQUIRE_EXACT2"
+    printf '# exactn_cuda=%s exactn_cuda_disable=%s require_cuda_exactn=%s exactn_cuda_batch_head=%s exactn_cuda_batch_head_disable=%s require_cuda_exactn_batch_head=%s cuda_device_proposer=%s cuda_device_proposer_disable=%s require_cuda_device_proposer=%s exactn_union_metal=%s noncausal_online_cuda=%s noncausal_online_cuda_disable=%s verify_noncausal=%s exact_rows_async_tails_metal=%s\n' \
+        "$exactn_cuda" "$exactn_cuda_disable" "$REQUIRE_CUDA_EXACTN" \
+        "$exactn_cuda_batch_head" "$exactn_cuda_batch_head_disable" \
+        "$REQUIRE_CUDA_EXACTN_BATCH_HEAD" \
+        "$cuda_device_proposer" "$cuda_device_proposer_disable" \
+        "$REQUIRE_CUDA_DEVICE_PROPOSER" \
+        "$exactn_union_metal" "$noncausal_online_cuda" \
+        "$noncausal_online_cuda_disable" "$verify_noncausal" \
+        "$exact_rows_async_tails_metal"
+    printf '# exactn_cuda_graphs=%s exactn_cuda_graphs_disable=%s require_cuda_exactn_graphs=%s\n' \
+        "$exactn_cuda_graphs" "$exactn_cuda_graphs_disable" \
+        "$REQUIRE_CUDA_EXACTN_GRAPHS"
+    printf '# exactn_metal_batch_head=%s exactn_metal_batch_head_disable=%s require_metal_exactn_batch_head=%s require_metal_exactn_partial=%s metal_device_proposer=%s metal_device_proposer_disable=%s require_metal_device_proposer=%s\n' \
+        "$exactn_metal_batch_head" "$exactn_metal_batch_head_disable" \
+        "$REQUIRE_METAL_EXACTN_BATCH_HEAD" "$REQUIRE_METAL_EXACTN_PARTIAL" \
+        "$metal_device_proposer" "$metal_device_proposer_disable" \
+        "$REQUIRE_METAL_DEVICE_PROPOSER"
 }
 
 if [ ! -x "$DS4_BIN" ]; then
@@ -133,6 +336,44 @@ fi
 tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/ds4-dspark-fixture.XXXXXX")
 trap 'rm -rf "$tmpdir"' EXIT HUP INT TERM
 
+run_variant() {
+    mode=$1
+    prompt=$2
+    stdout_file=$3
+    stderr_file=$4
+
+    set -- "$DS4_BIN"
+    case "$BACKEND" in
+    metal) set -- "$@" --metal ;;
+    cuda)  set -- "$@" --cuda ;;
+    rocm)  set -- "$@" --rocm ;;
+    esac
+    if [ "$SSD_STREAMING" = 1 ]; then
+        set -- "$@" --ssd-streaming
+        if [ -n "$SSD_CACHE_EXPERTS" ]; then
+            set -- "$@" --ssd-streaming-cache-experts "$SSD_CACHE_EXPERTS"
+        fi
+    fi
+    if [ "$mode" = dspark ]; then
+        set -- "$@" --dspark --mtp-model "$SUPPORT"
+        if [ "$EXACT_SAMPLING" != 0 ]; then
+            set -- "$@" --mtp-exact-sampling
+        fi
+        if [ -n "$CONFIDENCE" ]; then
+            set -- "$@" --dspark-confidence "$CONFIDENCE"
+        fi
+    fi
+    set -- "$@" -m "$MODEL" --tokens "$TOKENS" \
+        --temp "$TEMPERATURE" --top-p "$TOP_P" --min-p "$MIN_P" \
+        --seed "$SEED" --nothink -p "$prompt"
+
+    if [ "$mode" = dspark ]; then
+        DS4_DSPARK_STATS=1 "$@" >"$stdout_file" 2>"$stderr_file"
+    else
+        "$@" >"$stdout_file" 2>"$stderr_file"
+    fi
+}
+
 run_case() {
     id=$1
     prompt=$2
@@ -141,27 +382,8 @@ run_case() {
     dspark_out="$tmpdir/$id.dspark.out"
     dspark_err="$tmpdir/$id.dspark.err"
 
-    "$DS4_BIN" -m "$MODEL" \
-        --tokens "$TOKENS" --temp "$TEMPERATURE" --top-p "$TOP_P" \
-        --min-p "$MIN_P" --seed "$SEED" --nothink -p "$prompt" \
-        >"$base_out" 2>"$base_err"
-
-    if [ -n "$CONFIDENCE" ]; then
-        DS4_DSPARK_STATS=1 \
-        "$DS4_BIN" --dspark $exact_sampling_arg \
-            --dspark-confidence "$CONFIDENCE" \
-            -m "$MODEL" --mtp-model "$SUPPORT" \
-            --tokens "$TOKENS" --temp "$TEMPERATURE" --top-p "$TOP_P" \
-            --min-p "$MIN_P" --seed "$SEED" --nothink -p "$prompt" \
-            >"$dspark_out" 2>"$dspark_err"
-    else
-        DS4_DSPARK_STATS=1 \
-        "$DS4_BIN" --dspark $exact_sampling_arg \
-            -m "$MODEL" --mtp-model "$SUPPORT" \
-            --tokens "$TOKENS" --temp "$TEMPERATURE" --top-p "$TOP_P" \
-            --min-p "$MIN_P" --seed "$SEED" --nothink -p "$prompt" \
-            >"$dspark_out" 2>"$dspark_err"
-    fi
+    run_variant baseline "$prompt" "$base_out" "$base_err"
+    run_variant dspark "$prompt" "$dspark_out" "$dspark_err"
 
     output_match=1
     if ! cmp -s "$base_out" "$dspark_out"; then
@@ -184,20 +406,162 @@ run_case() {
         return 1
     fi
 
-    partial=$(printf '%s\n' "$stats" | sed -n 's/.* partial=\([0-9][0-9]*\).*/\1/p')
-    errors=$(printf '%s\n' "$stats" | sed -n 's/.*errors=\([0-9][0-9]*\).*/\1/p')
-    accepted_draft=$(printf '%s\n' "$stats" | sed -n 's/.*accepted_draft=\([0-9][0-9]*\).*/\1/p')
-    direct_full=$(printf '%s\n' "$stats" | sed -n 's/.*direct_full=\([0-9][0-9]*\).*/\1/p')
-    direct_partial=$(printf '%s\n' "$stats" | sed -n 's/.*direct_partial=\([0-9][0-9]*\).*/\1/p')
+    partial=$(stats_field "$stats" partial)
+    errors=$(stats_field "$stats" errors)
+    verifier_unavailable=$(stats_field "$stats" verifier_unavailable)
+    proposed=$(stats_field "$stats" proposed)
+    accepted_draft=$(stats_field "$stats" accepted_draft)
+    direct_full=$(stats_field "$stats" direct_full)
+    direct_partial=$(stats_field "$stats" direct_partial)
+    exact2_attempt=$(stats_field "$stats" exact2_attempt)
+    exact2_fallback=$(stats_field "$stats" exact2_fallback)
+    cuda_exactn_attempt=$(stats_field "$stats" cuda_exactn_attempt)
+    cuda_exactn_fallback=$(stats_field "$stats" cuda_exactn_fallback)
+    cuda_exactn_error_fallback=$(stats_field "$stats" cuda_exactn_error_fallback)
+    cuda_exactn_batch_head_attempt=$(stats_field "$stats" cuda_exactn_batch_head_attempt)
+    cuda_exactn_batch_head_use=$(stats_field "$stats" cuda_exactn_batch_head_use)
+    cuda_exactn_batch_head_fallback=$(stats_field "$stats" cuda_exactn_batch_head_fallback)
+    cuda_exactn_graph_attempt=$(stats_field "$stats" cuda_exactn_graph_attempt)
+    cuda_exactn_graph_use=$(stats_field "$stats" cuda_exactn_graph_use)
+    cuda_exactn_graph_capture=$(stats_field "$stats" cuda_exactn_graph_capture)
+    cuda_exactn_graph_replay=$(stats_field "$stats" cuda_exactn_graph_replay)
+    cuda_exactn_graph_warm=$(stats_field "$stats" cuda_exactn_graph_warm)
+    cuda_exactn_graph_no_slot=$(stats_field "$stats" cuda_exactn_graph_no_slot)
+    cuda_exactn_graph_failure=$(stats_field "$stats" cuda_exactn_graph_failure)
+    cuda_device_proposer_attempt=$(stats_field "$stats" cuda_device_proposer_attempt)
+    cuda_device_proposer_use=$(stats_field "$stats" cuda_device_proposer_use)
+    cuda_device_proposer_fallback=$(stats_field "$stats" cuda_device_proposer_fallback)
+    cuda_device_proposer_policy_mismatch=$(stats_field "$stats" cuda_device_proposer_policy_mismatch)
+    exactn_union_error_fallback=$(stats_field "$stats" exactn_union_error_fallback)
+    exactn_union_partial_replay=$(stats_field "$stats" exactn_union_partial_replay)
+    exactn_union_verify_skip=$(stats_field "$stats" exactn_union_verify_skip)
+    metal_exactn_batch_head_attempt=$(stats_field "$stats" metal_exactn_batch_head_attempt)
+    metal_exactn_batch_head_use=$(stats_field "$stats" metal_exactn_batch_head_use)
+    metal_exactn_batch_head_fallback=$(stats_field "$stats" metal_exactn_batch_head_fallback)
+    metal_device_proposer_attempt=$(stats_field "$stats" metal_device_proposer_attempt)
+    metal_device_proposer_use=$(stats_field "$stats" metal_device_proposer_use)
+    metal_device_proposer_fallback=$(stats_field "$stats" metal_device_proposer_fallback)
+    metal_device_proposer_policy_mismatch=$(stats_field "$stats" metal_device_proposer_policy_mismatch)
+    exact2_full=$(stats_field "$stats" exact2_full)
+    cuda_exactn_full=$(stats_field "$stats" cuda_exactn_full)
+    exactn_union_full=$(stats_field "$stats" exactn_union_full)
+    exactn_full=$(stats_field "$stats" exactn_full)
     partial=${partial:-0}
     errors=${errors:-0}
+    verifier_unavailable=${verifier_unavailable:-0}
+    proposed=${proposed:-0}
     accepted_draft=${accepted_draft:-0}
     direct_full=${direct_full:-0}
     direct_partial=${direct_partial:-0}
+    exact2_attempt=${exact2_attempt:-0}
+    exact2_fallback=${exact2_fallback:-0}
+    cuda_exactn_attempt=${cuda_exactn_attempt:-0}
+    cuda_exactn_fallback=${cuda_exactn_fallback:-0}
+    cuda_exactn_error_fallback=${cuda_exactn_error_fallback:-0}
+    cuda_exactn_batch_head_attempt=${cuda_exactn_batch_head_attempt:-0}
+    cuda_exactn_batch_head_use=${cuda_exactn_batch_head_use:-0}
+    cuda_exactn_batch_head_fallback=${cuda_exactn_batch_head_fallback:-0}
+    cuda_exactn_graph_attempt=${cuda_exactn_graph_attempt:-0}
+    cuda_exactn_graph_use=${cuda_exactn_graph_use:-0}
+    cuda_exactn_graph_capture=${cuda_exactn_graph_capture:-0}
+    cuda_exactn_graph_replay=${cuda_exactn_graph_replay:-0}
+    cuda_exactn_graph_warm=${cuda_exactn_graph_warm:-0}
+    cuda_exactn_graph_no_slot=${cuda_exactn_graph_no_slot:-0}
+    cuda_exactn_graph_failure=${cuda_exactn_graph_failure:-0}
+    cuda_device_proposer_attempt=${cuda_device_proposer_attempt:-0}
+    cuda_device_proposer_use=${cuda_device_proposer_use:-0}
+    cuda_device_proposer_fallback=${cuda_device_proposer_fallback:-0}
+    cuda_device_proposer_policy_mismatch=${cuda_device_proposer_policy_mismatch:-0}
+    exactn_union_error_fallback=${exactn_union_error_fallback:-0}
+    exactn_union_partial_replay=${exactn_union_partial_replay:-0}
+    exactn_union_verify_skip=${exactn_union_verify_skip:-0}
+    metal_exactn_batch_head_attempt=${metal_exactn_batch_head_attempt:-0}
+    metal_exactn_batch_head_use=${metal_exactn_batch_head_use:-0}
+    metal_exactn_batch_head_fallback=${metal_exactn_batch_head_fallback:-0}
+    metal_device_proposer_attempt=${metal_device_proposer_attempt:-0}
+    metal_device_proposer_use=${metal_device_proposer_use:-0}
+    metal_device_proposer_fallback=${metal_device_proposer_fallback:-0}
+    metal_device_proposer_policy_mismatch=${metal_device_proposer_policy_mismatch:-0}
+    exact2_full=${exact2_full:-0}
+    cuda_exactn_full=${cuda_exactn_full:-0}
+    exactn_union_full=${exactn_union_full:-0}
+    exactn_full=${exactn_full:-0}
     if [ "$errors" -ne 0 ]; then
         echo "dspark-fixture: verifier errors for $id: $stats" >&2
         return 1
     fi
+    if [ "$verifier_unavailable" -ne 0 ]; then
+        echo "dspark-fixture: verifier unavailable for $id: $stats" >&2
+        return 1
+    fi
+    total_proposed=$((total_proposed + proposed))
+    total_accepted_draft=$((total_accepted_draft + accepted_draft))
+    total_exact2_attempt=$((total_exact2_attempt + exact2_attempt))
+    total_exact2_fallback=$((total_exact2_fallback + exact2_fallback))
+    total_cuda_exactn_attempt=$((total_cuda_exactn_attempt + cuda_exactn_attempt))
+    total_cuda_exactn_fallback=$((total_cuda_exactn_fallback + cuda_exactn_fallback))
+    total_cuda_exactn_error_fallback=$((total_cuda_exactn_error_fallback + cuda_exactn_error_fallback))
+    total_cuda_exactn_batch_head_attempt=$((total_cuda_exactn_batch_head_attempt + cuda_exactn_batch_head_attempt))
+    total_cuda_exactn_batch_head_use=$((total_cuda_exactn_batch_head_use + cuda_exactn_batch_head_use))
+    total_cuda_exactn_batch_head_fallback=$((total_cuda_exactn_batch_head_fallback + cuda_exactn_batch_head_fallback))
+    total_cuda_exactn_graph_attempt=$((total_cuda_exactn_graph_attempt + cuda_exactn_graph_attempt))
+    total_cuda_exactn_graph_use=$((total_cuda_exactn_graph_use + cuda_exactn_graph_use))
+    total_cuda_exactn_graph_capture=$((total_cuda_exactn_graph_capture + cuda_exactn_graph_capture))
+    total_cuda_exactn_graph_replay=$((total_cuda_exactn_graph_replay + cuda_exactn_graph_replay))
+    total_cuda_exactn_graph_warm=$((total_cuda_exactn_graph_warm + cuda_exactn_graph_warm))
+    total_cuda_exactn_graph_no_slot=$((total_cuda_exactn_graph_no_slot + cuda_exactn_graph_no_slot))
+    total_cuda_exactn_graph_failure=$((total_cuda_exactn_graph_failure + cuda_exactn_graph_failure))
+    total_cuda_device_proposer_attempt=$((total_cuda_device_proposer_attempt + cuda_device_proposer_attempt))
+    total_cuda_device_proposer_use=$((total_cuda_device_proposer_use + cuda_device_proposer_use))
+    total_cuda_device_proposer_fallback=$((total_cuda_device_proposer_fallback + cuda_device_proposer_fallback))
+    total_cuda_device_proposer_policy_mismatch=$((total_cuda_device_proposer_policy_mismatch + cuda_device_proposer_policy_mismatch))
+    total_exactn_union_error_fallback=$((total_exactn_union_error_fallback + exactn_union_error_fallback))
+    total_exactn_union_partial_replay=$((total_exactn_union_partial_replay + exactn_union_partial_replay))
+    total_exactn_union_verify_skip=$((total_exactn_union_verify_skip + exactn_union_verify_skip))
+    total_metal_exactn_batch_head_attempt=$((total_metal_exactn_batch_head_attempt + metal_exactn_batch_head_attempt))
+    total_metal_exactn_batch_head_use=$((total_metal_exactn_batch_head_use + metal_exactn_batch_head_use))
+    total_metal_exactn_batch_head_fallback=$((total_metal_exactn_batch_head_fallback + metal_exactn_batch_head_fallback))
+    total_metal_device_proposer_attempt=$((total_metal_device_proposer_attempt + metal_device_proposer_attempt))
+    total_metal_device_proposer_use=$((total_metal_device_proposer_use + metal_device_proposer_use))
+    total_metal_device_proposer_fallback=$((total_metal_device_proposer_fallback + metal_device_proposer_fallback))
+    total_metal_device_proposer_policy_mismatch=$((total_metal_device_proposer_policy_mismatch + metal_device_proposer_policy_mismatch))
+    if [ "$REQUIRE_EXACT2" != 0 ] && [ "$exact2_fallback" -ne 0 ]; then
+        echo "dspark-fixture: exact2 fallback for $id: $stats" >&2
+        return 1
+    fi
+    if [ "$REQUIRE_CUDA_EXACTN" != 0 ] &&
+       [ "$cuda_exactn_error_fallback" -ne 0 ]; then
+        echo "dspark-fixture: CUDA exact-N error fallback for $id: $stats" >&2
+        return 1
+    fi
+    if [ "$REQUIRE_CUDA_EXACTN_BATCH_HEAD" != 0 ] &&
+       [ "$cuda_exactn_batch_head_fallback" -ne 0 ]; then
+        echo "dspark-fixture: CUDA exact-N batch-head fallback for $id: $stats" >&2
+        return 1
+    fi
+    if [ "$REQUIRE_CUDA_DEVICE_PROPOSER" != 0 ] &&
+       { [ "$cuda_device_proposer_fallback" -ne 0 ] ||
+         [ "$cuda_device_proposer_policy_mismatch" -ne 0 ]; }; then
+        echo "dspark-fixture: CUDA device proposer fallback/mismatch for $id: $stats" >&2
+        return 1
+    fi
+    if [ "$REQUIRE_METAL_EXACTN_BATCH_HEAD" != 0 ] &&
+       [ "$metal_exactn_batch_head_fallback" -ne 0 ]; then
+        echo "dspark-fixture: Metal exact-N batch-head fallback for $id: $stats" >&2
+        return 1
+    fi
+    if [ "$REQUIRE_METAL_EXACTN_PARTIAL" != 0 ] &&
+       { [ "$exactn_union_error_fallback" -ne 0 ] ||
+         [ "$exactn_union_partial_replay" -ne "$exactn_union_verify_skip" ]; }; then
+        echo "dspark-fixture: Metal exact-N partial replay/skip mismatch for $id: $stats" >&2
+        return 1
+    fi
+    if [ "$REQUIRE_METAL_DEVICE_PROPOSER" != 0 ] &&
+       { [ "$metal_device_proposer_fallback" -ne 0 ] ||
+         [ "$metal_device_proposer_policy_mismatch" -ne 0 ]; }; then
+        echo "dspark-fixture: Metal device proposer fallback/mismatch for $id: $stats" >&2
+        return 1
+    fi
     if [ "$PROPOSAL_QUALITY_GUARD_ACTIVE" -ne 0 ] && [ "$id" = c_add ] &&
         [ "$accepted_draft" -lt "$C_ADD_MIN_ACCEPTED" ]; then
         echo "dspark-fixture: c_add accepted_draft $accepted_draft below required $C_ADD_MIN_ACCEPTED: $stats" >&2
@@ -209,7 +573,8 @@ run_case() {
     if [ "$direct_partial" -gt 0 ]; then
         direct_partial_cases=$((direct_partial_cases + 1))
     fi
-    direct_commits=$((direct_commits + direct_full + direct_partial))
+    direct_commits=$((direct_commits + direct_full + direct_partial + \
+        exact2_full + cuda_exactn_full + exactn_union_full + exactn_full))
 
     printf '%s\toutput_match=%s\tbaseline_tps=%s\tdspark_tps=%s\t%s\n' \
         "$id" "$output_match" "${base_tps:-n/a}" "${dspark_tps:-n/a}" "$stats"
@@ -235,3 +600,122 @@ if [ "$REQUIRE_DIRECT" != 0 ] && [ "$direct_commits" -eq 0 ]; then
     echo "dspark-fixture: expected at least one direct verifier-state commit" >&2
     exit 1
 fi
+if [ "$REQUIRE_ACTIVE" != 0 ] &&
+   { [ "$total_proposed" -eq 0 ] || [ "$total_accepted_draft" -eq 0 ]; }; then
+    echo "dspark-fixture: DSpark runtime was not active (proposed=$total_proposed accepted_draft=$total_accepted_draft)" >&2
+    exit 1
+fi
+if [ "$REQUIRE_EXACT2" != 0 ] && [ "$total_exact2_attempt" -eq 0 ]; then
+    echo "dspark-fixture: exact2 was required but never attempted" >&2
+    exit 1
+fi
+if [ "$REQUIRE_EXACT2" != 0 ] && [ "$total_exact2_fallback" -ne 0 ]; then
+    echo "dspark-fixture: exact2 fallback count=$total_exact2_fallback" >&2
+    exit 1
+fi
+if [ "$REQUIRE_CUDA_EXACTN" != 0 ]; then
+    printf '# cuda_exactn_attempt=%s cuda_exactn_fallback=%s cuda_exactn_error_fallback=%s\n' \
+        "$total_cuda_exactn_attempt" "$total_cuda_exactn_fallback" \
+        "$total_cuda_exactn_error_fallback"
+fi
+if [ "$REQUIRE_CUDA_EXACTN_BATCH_HEAD" != 0 ]; then
+    printf '# cuda_exactn_batch_head_attempt=%s cuda_exactn_batch_head_use=%s cuda_exactn_batch_head_fallback=%s\n' \
+        "$total_cuda_exactn_batch_head_attempt" \
+        "$total_cuda_exactn_batch_head_use" \
+        "$total_cuda_exactn_batch_head_fallback"
+fi
+if [ "$REQUIRE_CUDA_EXACTN_GRAPHS" != 0 ]; then
+    printf '# cuda_exactn_graph_attempt=%s cuda_exactn_graph_use=%s cuda_exactn_graph_capture=%s cuda_exactn_graph_replay=%s cuda_exactn_graph_warm=%s cuda_exactn_graph_no_slot=%s cuda_exactn_graph_failure=%s\n' \
+        "$total_cuda_exactn_graph_attempt" \
+        "$total_cuda_exactn_graph_use" \
+        "$total_cuda_exactn_graph_capture" \
+        "$total_cuda_exactn_graph_replay" \
+        "$total_cuda_exactn_graph_warm" \
+        "$total_cuda_exactn_graph_no_slot" \
+        "$total_cuda_exactn_graph_failure"
+fi
+if [ "$REQUIRE_CUDA_DEVICE_PROPOSER" != 0 ]; then
+    printf '# cuda_device_proposer_attempt=%s cuda_device_proposer_use=%s cuda_device_proposer_fallback=%s cuda_device_proposer_policy_mismatch=%s\n' \
+        "$total_cuda_device_proposer_attempt" \
+        "$total_cuda_device_proposer_use" \
+        "$total_cuda_device_proposer_fallback" \
+        "$total_cuda_device_proposer_policy_mismatch"
+fi
+if [ "$REQUIRE_METAL_EXACTN_BATCH_HEAD" != 0 ]; then
+    printf '# metal_exactn_batch_head_attempt=%s metal_exactn_batch_head_use=%s metal_exactn_batch_head_fallback=%s\n' \
+        "$total_metal_exactn_batch_head_attempt" \
+        "$total_metal_exactn_batch_head_use" \
+        "$total_metal_exactn_batch_head_fallback"
+fi
+if [ "$REQUIRE_METAL_EXACTN_PARTIAL" != 0 ]; then
+    printf '# exactn_union_error_fallback=%s exactn_union_partial_replay=%s exactn_union_verify_skip=%s\n' \
+        "$total_exactn_union_error_fallback" \
+        "$total_exactn_union_partial_replay" \
+        "$total_exactn_union_verify_skip"
+fi
+if [ "$REQUIRE_METAL_DEVICE_PROPOSER" != 0 ]; then
+    printf '# metal_device_proposer_attempt=%s metal_device_proposer_use=%s metal_device_proposer_fallback=%s metal_device_proposer_policy_mismatch=%s\n' \
+        "$total_metal_device_proposer_attempt" \
+        "$total_metal_device_proposer_use" \
+        "$total_metal_device_proposer_fallback" \
+        "$total_metal_device_proposer_policy_mismatch"
+fi
+if [ "$REQUIRE_CUDA_EXACTN" != 0 ] &&
+   [ "$total_cuda_exactn_attempt" -eq 0 ]; then
+    echo "dspark-fixture: CUDA exact-N was required but never attempted" >&2
+    exit 1
+fi
+if [ "$REQUIRE_CUDA_EXACTN" != 0 ] &&
+   [ "$total_cuda_exactn_error_fallback" -ne 0 ]; then
+    echo "dspark-fixture: CUDA exact-N error fallback count=$total_cuda_exactn_error_fallback" >&2
+    exit 1
+fi
+if [ "$REQUIRE_CUDA_EXACTN_BATCH_HEAD" != 0 ] &&
+   { [ "$total_cuda_exactn_batch_head_attempt" -eq 0 ] ||
+     [ "$total_cuda_exactn_batch_head_use" -eq 0 ] ||
+     [ "$total_cuda_exactn_batch_head_fallback" -ne 0 ]; }; then
+    echo "dspark-fixture: CUDA exact-N batch head not cleanly exercised (attempt=$total_cuda_exactn_batch_head_attempt use=$total_cuda_exactn_batch_head_use fallback=$total_cuda_exactn_batch_head_fallback)" >&2
+    exit 1
+fi
+if [ "$REQUIRE_CUDA_EXACTN_GRAPHS" != 0 ] &&
+   { [ "$total_cuda_exactn_graph_attempt" -eq 0 ] ||
+     [ "$total_cuda_exactn_graph_use" -eq 0 ] ||
+     [ "$total_cuda_exactn_graph_capture" -eq 0 ] ||
+     [ "$total_cuda_exactn_graph_replay" -eq 0 ] ||
+     [ "$total_cuda_exactn_graph_no_slot" -ne 0 ] ||
+     [ "$total_cuda_exactn_graph_failure" -ne 0 ]; }; then
+    echo "dspark-fixture: CUDA exact-N graphs were not cleanly replayed after warmup (attempt=$total_cuda_exactn_graph_attempt use=$total_cuda_exactn_graph_use warm=$total_cuda_exactn_graph_warm capture=$total_cuda_exactn_graph_capture replay=$total_cuda_exactn_graph_replay no_slot=$total_cuda_exactn_graph_no_slot failure=$total_cuda_exactn_graph_failure)" >&2
+    exit 1
+fi
+if [ "$REQUIRE_CUDA_DEVICE_PROPOSER" != 0 ] &&
+   { [ "$total_cuda_device_proposer_attempt" -eq 0 ] ||
+     [ "$total_cuda_device_proposer_use" -eq 0 ] ||
+     [ "$total_cuda_device_proposer_attempt" -ne "$total_cuda_device_proposer_use" ] ||
+     [ "$total_cuda_device_proposer_fallback" -ne 0 ] ||
+     [ "$total_cuda_device_proposer_policy_mismatch" -ne 0 ]; }; then
+    echo "dspark-fixture: CUDA device proposer not cleanly exercised (attempt=$total_cuda_device_proposer_attempt use=$total_cuda_device_proposer_use fallback=$total_cuda_device_proposer_fallback policy_mismatch=$total_cuda_device_proposer_policy_mismatch)" >&2
+    exit 1
+fi
+if [ "$REQUIRE_METAL_EXACTN_BATCH_HEAD" != 0 ] &&
+   { [ "$total_metal_exactn_batch_head_attempt" -eq 0 ] ||
+     [ "$total_metal_exactn_batch_head_use" -eq 0 ] ||
+     [ "$total_metal_exactn_batch_head_fallback" -ne 0 ]; }; then
+    echo "dspark-fixture: Metal exact-N batch head not cleanly exercised (attempt=$total_metal_exactn_batch_head_attempt use=$total_metal_exactn_batch_head_use fallback=$total_metal_exactn_batch_head_fallback)" >&2
+    exit 1
+fi
+if [ "$REQUIRE_METAL_EXACTN_PARTIAL" != 0 ] &&
+   { [ "$total_exactn_union_partial_replay" -eq 0 ] ||
+     [ "$total_exactn_union_partial_replay" -ne "$total_exactn_union_verify_skip" ] ||
+     [ "$total_exactn_union_error_fallback" -ne 0 ]; }; then
+    echo "dspark-fixture: Metal exact-N partial path not cleanly exercised (replay=$total_exactn_union_partial_replay verify_skip=$total_exactn_union_verify_skip error_fallback=$total_exactn_union_error_fallback)" >&2
+    exit 1
+fi
+if [ "$REQUIRE_METAL_DEVICE_PROPOSER" != 0 ] &&
+   { [ "$total_metal_device_proposer_attempt" -eq 0 ] ||
+     [ "$total_metal_device_proposer_use" -eq 0 ] ||
+     [ "$total_metal_device_proposer_attempt" -ne "$total_metal_device_proposer_use" ] ||
+     [ "$total_metal_device_proposer_fallback" -ne 0 ] ||
+     [ "$total_metal_device_proposer_policy_mismatch" -ne 0 ]; }; then
+    echo "dspark-fixture: Metal device proposer not cleanly exercised (attempt=$total_metal_device_proposer_attempt use=$total_metal_device_proposer_use fallback=$total_metal_device_proposer_fallback policy_mismatch=$total_metal_device_proposer_policy_mismatch)" >&2
+    exit 1
+fi
diff --git a/tests/test_engine_mgpu_placement.c b/tests/test_engine_mgpu_placement.c
index f86db490df..bb84bc7031 100644
--- a/tests/test_engine_mgpu_placement.c
+++ b/tests/test_engine_mgpu_placement.c
@@ -39,6 +39,29 @@ int ds4_test_classify_multi_tier(const ds4_test_fake_tensor *tensors,
                                   int *out_multi_tier,
                                   int *out_n_entries);
 int ds4_test_tensor_to_entry(const char *name, int name_len);
+int ds4_test_dspark_runtime_policy(ds4_backend backend,
+                                   ds4_distributed_role distributed_role);
+bool ds4_test_streaming_manual_cache_cap_count(
+        uint32_t requested_count,
+        uint64_t per_expert_bytes,
+        uint64_t safe_cache_bytes,
+        uint32_t *effective_count_out,
+        uint64_t *requested_bytes_out,
+        uint64_t *effective_bytes_out);
+bool ds4_test_streaming_manual_cache_count_cap_enabled(void);
+bool ds4_test_streaming_manual_cache_count_cap_eligible(
+        ds4_backend backend,
+        bool ssd_streaming,
+        bool dspark_enabled,
+        bool support_is_dspark,
+        uint32_t cache_experts,
+        uint64_t cache_bytes);
+uint64_t ds4_test_streaming_dspark_active_support_reserve_bytes(
+        uint64_t support_model_bytes);
+uint32_t ds4_test_streaming_manual_cache_nonfatal_effective_count(
+        uint32_t requested_count,
+        bool safe_cache_known,
+        uint32_t candidate_count);
 
 /* Ctx-aware variants and calibration helpers. Declared here (not in
  * ds4.h) matching the existing DS4_TEST_HOOKS pattern. */
@@ -184,6 +207,25 @@ static void test_null_config(void) {
     CHECK(n_entries == 0, "NULL cfg -> n_entries 0");
 }
 
+static void test_dspark_runtime_policy(void) {
+    fprintf(stderr, "RUN: test_dspark_runtime_policy\n");
+    CHECK(ds4_test_dspark_runtime_policy(DS4_BACKEND_METAL,
+                                        DS4_DISTRIBUTED_NONE) == 0,
+          "DSpark supports a local Metal graph backend");
+    CHECK(ds4_test_dspark_runtime_policy(DS4_BACKEND_CUDA,
+                                        DS4_DISTRIBUTED_NONE) == 0,
+          "DSpark supports a local CUDA/ROCm graph backend");
+    CHECK(ds4_test_dspark_runtime_policy(DS4_BACKEND_CPU,
+                                        DS4_DISTRIBUTED_NONE) != 0,
+          "DSpark rejects the CPU backend");
+    CHECK(ds4_test_dspark_runtime_policy(DS4_BACKEND_CUDA,
+                                        DS4_DISTRIBUTED_COORDINATOR) != 0,
+          "DSpark rejects a distributed coordinator");
+    CHECK(ds4_test_dspark_runtime_policy(DS4_BACKEND_CUDA,
+                                        DS4_DISTRIBUTED_WORKER) != 0,
+          "DSpark rejects a distributed worker");
+}
+
 /* Build a synthetic, model-shaped tensor list: 1 embedding + 43 layers
  * (each with 2 tensors of equal size) + 1 output head. Used by the
  * multi-tier tests to drive a realistic placement decision. */
@@ -582,6 +624,113 @@ static void test_glm_memory_guard_budget(void) {
     restore_env_value("DS4_GLM_MEMORY_GUARD", old_guard);
 }
 
+static void test_streaming_manual_cache_count_cap(void) {
+    fprintf(stderr, "RUN: test_streaming_manual_cache_count_cap\n");
+    const uint64_t mib = 1024ull * 1024ull;
+    const uint64_t per_expert = 7ull * mib;
+    uint32_t count = UINT32_MAX;
+    uint64_t requested = UINT64_MAX;
+    uint64_t effective = UINT64_MAX;
+
+    CHECK(ds4_test_streaming_manual_cache_cap_count(
+                  100, per_expert, 100ull * per_expert,
+                  &count, &requested, &effective),
+          "numeric cache count converts to bytes without a cap");
+    CHECK(count == 100, "safe numeric cache count is preserved");
+    CHECK(requested == 100ull * per_expert,
+          "numeric request byte conversion is exact");
+    CHECK(effective == requested,
+          "uncapped effective bytes equal requested bytes");
+
+    CHECK(ds4_test_streaming_manual_cache_cap_count(
+                  100, per_expert, 15ull * per_expert + per_expert / 2ull,
+                  &count, &requested, &effective),
+          "numeric cache count accepts a fractional-slot byte limit");
+    CHECK(count == 15, "safe byte limit rounds down to whole expert slots");
+    CHECK(effective == 15ull * per_expert,
+          "capped count converts back to exact bytes");
+
+    CHECK(ds4_test_streaming_manual_cache_cap_count(
+                  3, per_expert, per_expert - 1ull,
+                  &count, &requested, &effective),
+          "sub-slot safe limit is represented without overflow");
+    CHECK(count == 0 && effective == 0,
+          "sub-slot safe limit never forces one unsafe expert");
+
+    count = 123;
+    requested = 456;
+    effective = 789;
+    CHECK(!ds4_test_streaming_manual_cache_cap_count(
+                   UINT32_MAX, UINT64_MAX, UINT64_MAX,
+                   &count, &requested, &effective),
+          "numeric request multiplication overflow is rejected");
+    CHECK(count == 0 && requested == 0 && effective == 0,
+          "overflow failure clears all conversion outputs");
+
+    const char *env = "DS4_METAL_DSPARK_SAFE_EXPERT_COUNT";
+    char *saved = save_env_value(env);
+    unsetenv(env);
+    CHECK(!ds4_test_streaming_manual_cache_count_cap_enabled(),
+          "numeric cache safety cap is disabled by default during A/B");
+    setenv(env, "0", 1);
+    CHECK(!ds4_test_streaming_manual_cache_count_cap_enabled(),
+          "zero does not enable the numeric cache safety cap");
+    setenv(env, "true", 1);
+    CHECK(!ds4_test_streaming_manual_cache_count_cap_enabled(),
+          "ambiguous truthy text does not enable the safety cap");
+    setenv(env, "1", 1);
+    CHECK(ds4_test_streaming_manual_cache_count_cap_enabled(),
+          "the documented exact value 1 enables the safety cap");
+    restore_env_value(env, saved);
+
+    CHECK(ds4_test_streaming_manual_cache_count_cap_eligible(
+                  DS4_BACKEND_METAL, true, true, true, 1032, 0),
+          "Metal SSD+DSpark numeric count is eligible for the opt-in cap");
+    CHECK(!ds4_test_streaming_manual_cache_count_cap_eligible(
+                   DS4_BACKEND_CUDA, true, true, true, 1032, 0),
+          "CUDA numeric count is not eligible");
+    CHECK(!ds4_test_streaming_manual_cache_count_cap_eligible(
+                   DS4_BACKEND_METAL, false, true, true, 1032, 0),
+          "non-streaming Metal is not eligible");
+    CHECK(!ds4_test_streaming_manual_cache_count_cap_eligible(
+                   DS4_BACKEND_METAL, true, false, true, 1032, 0),
+          "Metal support model without --dspark is not eligible");
+    CHECK(!ds4_test_streaming_manual_cache_count_cap_eligible(
+                   DS4_BACKEND_METAL, true, true, false, 1032, 0),
+          "non-DSpark support model is not eligible");
+    CHECK(!ds4_test_streaming_manual_cache_count_cap_eligible(
+                   DS4_BACKEND_METAL, true, true, true, 0, 0),
+          "automatic cache selection is not a numeric-count request");
+    CHECK(!ds4_test_streaming_manual_cache_count_cap_eligible(
+                   DS4_BACKEND_METAL, true, true, true, 0, 32ull << 30),
+          "NGB cache budget is not eligible");
+
+    const uint64_t gib = 1024ull * 1024ull * 1024ull;
+    CHECK(ds4_test_streaming_dspark_active_support_reserve_bytes(0) == 0,
+          "absent support mapping reserves zero bytes");
+    CHECK(ds4_test_streaming_dspark_active_support_reserve_bytes(gib) == gib,
+          "small support mapping is reserved in full");
+    CHECK(ds4_test_streaming_dspark_active_support_reserve_bytes(2ull * gib) ==
+                  2ull * gib,
+          "support reserve includes the exact 2 GiB boundary");
+    CHECK(ds4_test_streaming_dspark_active_support_reserve_bytes(6ull * gib) ==
+                  2ull * gib,
+          "large mmap-backed support model is limited to 2 GiB active reserve");
+
+    CHECK(ds4_test_streaming_manual_cache_nonfatal_effective_count(
+                  1032, true, 341) == 341,
+          "measured non-zero candidate may cap a numeric request");
+    CHECK(ds4_test_streaming_manual_cache_nonfatal_effective_count(
+                  1032, false, 341) == 1032,
+          "unknown safe budget preserves the explicit request");
+    CHECK(ds4_test_streaming_manual_cache_nonfatal_effective_count(
+                  1032, true, 0) == 1032,
+          "sub-slot safe budget remains non-fatal");
+    CHECK(ds4_test_streaming_manual_cache_nonfatal_effective_count(
+                  100, true, 200) == 100,
+          "policy never grows an explicit numeric request");
+}
+
 static void test_cuda_tp_prefill_default_accounting(void) {
     fprintf(stderr, "RUN: test_cuda_tp_prefill_default_accounting\n");
 
@@ -706,6 +855,7 @@ static void test_cuda_tp_output_head_moves_to_lower_half(void) {
 int main(void) {
     test_tensor_to_entry();
     test_null_config();
+    test_dspark_runtime_policy();
     test_forced_two_tier_no_spill();
     test_cpu_spill();
     test_zero_budget_guard();
@@ -715,6 +865,7 @@ int main(void) {
     test_glm_per_layer_cache_accounting();
     test_glm_session_count_accounting();
     test_glm_memory_guard_budget();
+    test_streaming_manual_cache_count_cap();
     test_cuda_tp_prefill_default_accounting();
     test_cuda_tp_output_head_moves_to_lower_half();
 
diff --git a/tests/test_gpu_model_cache.c b/tests/test_gpu_model_cache.c
index ebbd0c4e05..c8cd7324d1 100644
--- a/tests/test_gpu_model_cache.c
+++ b/tests/test_gpu_model_cache.c
@@ -6,6 +6,7 @@
  *   - ds4_gpu_lookup_cache at range bases and at interior offsets
  *     (proves the subrange pointer offset arithmetic is right)
  *   - device-id resolution
+ *   - selected-expert batched-I/O policy, planner, scatter and byte oracle
  *   - on multi-GPU boxes: caching on device 1 and active-device
  *     preference in lookup */
 
@@ -26,6 +27,234 @@
     } while (0)
 
 int main(void) {
+    int enabled = -1;
+    int required = -1;
+    int oracle = -1;
+    int fused = -1;
+    int owned_forced = -1;
+    int multi_gpu_forced = -1;
+    int capture = -1;
+    CHECK(!ds4_cuda_test_q8_hc_expand_env_value(NULL) &&
+          !ds4_cuda_test_q8_hc_expand_env_value("") &&
+          !ds4_cuda_test_q8_hc_expand_env_value("0") &&
+          !ds4_cuda_test_q8_hc_expand_env_value("false") &&
+          !ds4_cuda_test_q8_hc_expand_env_value("NO") &&
+          !ds4_cuda_test_q8_hc_expand_env_value("off") &&
+          ds4_cuda_test_q8_hc_expand_env_value("1") &&
+          ds4_cuda_test_q8_hc_expand_env_value("true"),
+          "Q8 HC value-aware environment parser");
+    CHECK(ds4_cuda_test_q8_hc_expand_policy(
+              0, 0, 1, 0, 0, &fused, &owned_forced,
+              &multi_gpu_forced, &capture) &&
+          fused == 1 && owned_forced == 0 && multi_gpu_forced == 0 &&
+          capture == 0,
+          "Q8 HC defaults to fused");
+    CHECK(ds4_cuda_test_q8_hc_expand_policy(
+              0, 1, 1, 0, 0, &fused, &owned_forced,
+              &multi_gpu_forced, &capture) &&
+          fused == 0 && owned_forced == 0 && multi_gpu_forced == 0,
+          "Q8 HC split opt-in on single GPU");
+    CHECK(ds4_cuda_test_q8_hc_expand_policy(
+              1, 1, 1, 0, 0, &fused, &owned_forced,
+              &multi_gpu_forced, &capture) &&
+          fused == 1 && owned_forced == 0 && multi_gpu_forced == 0,
+          "Q8 HC force-fused dominates conflicting split request");
+    CHECK(ds4_cuda_test_q8_hc_expand_policy(
+              0, 1, 2, 0, 0, &fused, &owned_forced,
+              &multi_gpu_forced, &capture) &&
+          fused == 1 && owned_forced == 0 && multi_gpu_forced == 1,
+          "Q8 HC multi-GPU remains fused");
+    CHECK(ds4_cuda_test_q8_hc_expand_policy(
+              0, 1, 1, 1, 0, &fused, &owned_forced,
+              &multi_gpu_forced, &capture) &&
+          fused == 1 && owned_forced == 1 && multi_gpu_forced == 0,
+          "Q8 HC owned dispatch remains fused");
+    CHECK(ds4_cuda_test_q8_hc_expand_policy(
+              0, 1, 1, 0, 1, &fused, &owned_forced,
+              &multi_gpu_forced, &capture) &&
+          fused == 0 && capture == 1,
+          "Q8 HC split graph-capture policy matches eager policy");
+    CHECK(!ds4_cuda_test_stream_selected_batch_env_value(NULL) &&
+          !ds4_cuda_test_stream_selected_batch_env_value("") &&
+          !ds4_cuda_test_stream_selected_batch_env_value("0") &&
+          !ds4_cuda_test_stream_selected_batch_env_value("false") &&
+          !ds4_cuda_test_stream_selected_batch_env_value("FALSE") &&
+          !ds4_cuda_test_stream_selected_batch_env_value("no") &&
+          !ds4_cuda_test_stream_selected_batch_env_value("off") &&
+          ds4_cuda_test_stream_selected_batch_env_value("1") &&
+          ds4_cuda_test_stream_selected_batch_env_value("true"),
+          "selected-expert batched-I/O value-aware environment parser");
+    CHECK(ds4_cuda_test_stream_selected_batch_policy(
+              1, 0, 0, 0, &enabled, &required, &oracle) &&
+          enabled == 1 && required == 0 && oracle == 0,
+          "selected-expert batched-I/O enable policy");
+    CHECK(ds4_cuda_test_stream_selected_batch_policy(
+              0, 0, 1, 0, &enabled, &required, &oracle) &&
+          enabled == 1 && required == 1 && oracle == 0,
+          "selected-expert batched-I/O require policy");
+    CHECK(ds4_cuda_test_stream_selected_batch_policy(
+              1, 1, 1, 1, &enabled, &required, &oracle) &&
+          enabled == 0 && required == 0 && oracle == 0,
+          "selected-expert batched-I/O disable-dominant policy");
+    CHECK(!ds4_cuda_test_stream_selected_event_env_value(NULL) &&
+          !ds4_cuda_test_stream_selected_event_env_value("") &&
+          !ds4_cuda_test_stream_selected_event_env_value("0") &&
+          !ds4_cuda_test_stream_selected_event_env_value("false") &&
+          !ds4_cuda_test_stream_selected_event_env_value("NO") &&
+          !ds4_cuda_test_stream_selected_event_env_value("off") &&
+          ds4_cuda_test_stream_selected_event_env_value("1") &&
+          ds4_cuda_test_stream_selected_event_env_value("true"),
+          "selected-expert event-pipeline value-aware environment parser");
+    CHECK(ds4_cuda_test_stream_selected_event_pipeline_policy(
+              1, 0, 0, 0, &enabled, &required, &oracle) &&
+          enabled == 1 && required == 0 && oracle == 0,
+          "selected-expert event-pipeline enable policy");
+    CHECK(ds4_cuda_test_stream_selected_event_pipeline_policy(
+              0, 0, 1, 0, &enabled, &required, &oracle) &&
+          enabled == 1 && required == 1 && oracle == 0,
+          "selected-expert event-pipeline require policy");
+    CHECK(ds4_cuda_test_stream_selected_event_pipeline_policy(
+              0, 0, 0, 1, &enabled, &required, &oracle) &&
+          enabled == 1 && required == 0 && oracle == 1,
+          "selected-expert event-pipeline oracle policy");
+    CHECK(ds4_cuda_test_stream_selected_event_pipeline_policy(
+              1, 1, 1, 1, &enabled, &required, &oracle) &&
+          enabled == 0 && required == 0 && oracle == 0,
+          "selected-expert event-pipeline disable-dominant policy");
+    int stats = -1;
+    CHECK(!ds4_cuda_test_stream_expert_persistent_env_value(NULL) &&
+          !ds4_cuda_test_stream_expert_persistent_env_value("") &&
+          !ds4_cuda_test_stream_expert_persistent_env_value("0") &&
+          !ds4_cuda_test_stream_expert_persistent_env_value("false") &&
+          !ds4_cuda_test_stream_expert_persistent_env_value("NO") &&
+          !ds4_cuda_test_stream_expert_persistent_env_value("off") &&
+          ds4_cuda_test_stream_expert_persistent_env_value("1") &&
+          ds4_cuda_test_stream_expert_persistent_env_value("true"),
+          "persistent expert planner value-aware environment parser");
+    CHECK(ds4_cuda_test_stream_expert_persistent_policy(
+              1, 0, 0, 0, 0,
+              &enabled, &required, &stats, &oracle) &&
+          enabled == 1 && required == 0 && stats == 0 && oracle == 0,
+          "persistent expert planner enable policy");
+    CHECK(ds4_cuda_test_stream_expert_persistent_policy(
+              0, 0, 1, 1, 1,
+              &enabled, &required, &stats, &oracle) &&
+          enabled == 1 && required == 1 && stats == 1 && oracle == 1,
+          "persistent expert planner require/stats/oracle policy");
+    CHECK(ds4_cuda_test_stream_expert_persistent_policy(
+              0, 0, 0, 1, 0,
+              &enabled, &required, &stats, &oracle) &&
+          enabled == 0 && required == 0 && stats == 1 && oracle == 0,
+          "persistent expert planner stats-only policy");
+    CHECK(ds4_cuda_test_stream_expert_persistent_policy(
+              1, 1, 1, 1, 1,
+              &enabled, &required, &stats, &oracle) &&
+          enabled == 0 && required == 0 && stats == 0 && oracle == 0,
+          "persistent expert planner disable-dominant policy");
+    ds4_cuda_stream_expert_persistent_report persistent_before;
+    memset(&persistent_before, 0, sizeof(persistent_before));
+    ds4_cuda_stream_expert_persistent_get_report(&persistent_before);
+    CHECK(ds4_cuda_test_stream_expert_persistent_planner(),
+          "persistent expert LRU/free-list transaction planner oracle");
+    ds4_cuda_stream_expert_persistent_report persistent_after;
+    memset(&persistent_after, 0, sizeof(persistent_after));
+    ds4_cuda_stream_expert_persistent_get_report(&persistent_after);
+    CHECK(persistent_after.oracle_runs == persistent_before.oracle_runs + 1u &&
+          persistent_after.oracle_failures ==
+              persistent_before.oracle_failures &&
+          persistent_after.plan_attempts > persistent_before.plan_attempts &&
+          persistent_after.plans_built > persistent_before.plans_built &&
+          persistent_after.commits > persistent_before.commits &&
+          persistent_after.rollbacks > persistent_before.rollbacks &&
+          persistent_after.hits > persistent_before.hits &&
+          persistent_after.misses > persistent_before.misses &&
+          persistent_after.duplicates > persistent_before.duplicates &&
+          persistent_after.free_assignments >
+              persistent_before.free_assignments &&
+          persistent_after.evictions > persistent_before.evictions &&
+          persistent_after.rejects > persistent_before.rejects &&
+          persistent_after.budget_rejects >
+              persistent_before.budget_rejects &&
+          persistent_after.class_rejects >
+              persistent_before.class_rejects &&
+          persistent_after.protected_rejects >
+              persistent_before.protected_rejects &&
+          persistent_after.key_misses > persistent_before.key_misses &&
+          persistent_after.overflow_rejects >
+              persistent_before.overflow_rejects,
+          "persistent expert planner invariant coverage counters");
+    CHECK(ds4_cuda_test_iq2_ssd_grouped_policy(
+              1, 0, 0, 0, &enabled, &required, &stats) &&
+          enabled == 1 && required == 0 && stats == 0,
+          "IQ2 SSD grouped-MMQ enable policy");
+    CHECK(ds4_cuda_test_iq2_ssd_grouped_policy(
+              0, 0, 1, 1, &enabled, &required, &stats) &&
+          enabled == 1 && required == 1 && stats == 1,
+          "IQ2 SSD grouped-MMQ require and stats policy");
+    CHECK(ds4_cuda_test_iq2_ssd_grouped_policy(
+              1, 1, 1, 1, &enabled, &required, &stats) &&
+          enabled == 0 && required == 0 && stats == 0,
+          "IQ2 SSD grouped-MMQ disable-dominant policy");
+    CHECK(ds4_cuda_test_iq2_ssd_grouped_eligibility(
+              1, 1, 1, 1, 0, 0, 0, 1,
+              32u, 6u, 1, 1, 1),
+          "IQ2 SSD grouped-MMQ canonical eligibility");
+    CHECK(!ds4_cuda_test_iq2_ssd_grouped_candidate(
+               1, 1, 1, 0, 1u, 6u, 1, 1, 1) &&
+          !ds4_cuda_test_iq2_ssd_grouped_candidate(
+               1, 1, 1, 0, 31u, 6u, 1, 1, 1) &&
+          ds4_cuda_test_iq2_ssd_grouped_candidate(
+               1, 1, 1, 0, 32u, 6u, 1, 1, 1),
+          "IQ2 SSD grouped-MMQ REQUIRE candidate excludes decode/tail");
+    CHECK(!ds4_cuda_test_iq2_ssd_grouped_candidate(
+               1, 1, 1, 0, 32u, 5u, 1, 1, 1) &&
+          !ds4_cuda_test_iq2_ssd_grouped_candidate(
+               1, 1, 1, 0, 32u, 6u, 0, 1, 1) &&
+          !ds4_cuda_test_iq2_ssd_grouped_candidate(
+               1, 1, 1, 0, 32u, 6u, 1, 0, 1) &&
+          !ds4_cuda_test_iq2_ssd_grouped_candidate(
+               1, 1, 1, 0, 32u, 6u, 1, 1, 0),
+          "IQ2 SSD grouped-MMQ REQUIRE excludes non-candidate layouts");
+    CHECK(!ds4_cuda_test_iq2_ssd_grouped_eligibility(
+               1, 1, 1, 1, 0, 0, 0, 1,
+               31u, 6u, 1, 1, 1) &&
+          !ds4_cuda_test_iq2_ssd_grouped_eligibility(
+               1, 1, 1, 1, 0, 0, 0, 1,
+               32u, 5u, 1, 1, 1) &&
+          !ds4_cuda_test_iq2_ssd_grouped_eligibility(
+               1, 1, 1, 1, 0, 0, 0, 1,
+               32u, 6u, 0, 1, 1) &&
+          !ds4_cuda_test_iq2_ssd_grouped_eligibility(
+               1, 1, 1, 1, 1, 0, 0, 1,
+               32u, 6u, 1, 1, 1) &&
+          !ds4_cuda_test_iq2_ssd_grouped_eligibility(
+               1, 1, 1, 1, 0, 0, 1, 1,
+               32u, 6u, 1, 1, 1) &&
+          !ds4_cuda_test_iq2_ssd_grouped_eligibility(
+               1, 1, 1, 1, 0, 0, 0, 1,
+               32u, 6u, 1, 1, 0),
+          "IQ2 SSD grouped-MMQ exclusion matrix");
+    const uint64_t iq2_row = 16u * 66u;
+    const uint64_t iq2_expert = 2048u * iq2_row;
+    const uint64_t q2_row = 8u * 84u;
+    const uint64_t q2_expert = 4096u * q2_row;
+    CHECK(ds4_cuda_test_iq2_ssd_grouped_raw_layout(
+              16u, 10u, iq2_expert, iq2_row,
+              q2_expert, q2_row, 4096u, 2048u, 4096u),
+          "IQ2 SSD grouped-MMQ canonical raw layout");
+    CHECK(!ds4_cuda_test_iq2_ssd_grouped_raw_layout(
+               12u, 10u, iq2_expert, iq2_row,
+               q2_expert, q2_row, 4096u, 2048u, 4096u) &&
+          !ds4_cuda_test_iq2_ssd_grouped_raw_layout(
+               16u, 10u, iq2_expert, iq2_row + 2u,
+               q2_expert, q2_row, 4096u, 2048u, 4096u) &&
+          !ds4_cuda_test_iq2_ssd_grouped_raw_layout(
+               16u, 10u, iq2_expert, iq2_row,
+               q2_expert + 84u, q2_row, 4096u, 2048u, 4096u),
+          "IQ2 SSD grouped-MMQ raw-layout rejection matrix");
+    CHECK(ds4_cuda_test_stream_selected_batch_plan(),
+          "selected-expert batched-I/O planner");
+
     int dev_count = 0;
     (void)cudaGetDeviceCount(&dev_count);
     fprintf(stderr, "test_gpu_model_cache: %d CUDA devices visible\n",
@@ -36,6 +265,109 @@ int main(void) {
     }
 
     CHECK(ds4_gpu_init(), "ds4_gpu_init");
+    ds4_cuda_q8_hc_expand_report q8_hc_before;
+    memset(&q8_hc_before, 0, sizeof(q8_hc_before));
+    ds4_cuda_q8_hc_expand_get_report(&q8_hc_before);
+    CHECK(ds4_cuda_test_q8_hc_expand_oracle(),
+          "Q8 HC fused/split graph-capture parity oracle");
+    ds4_cuda_q8_hc_expand_report q8_hc_after;
+    memset(&q8_hc_after, 0, sizeof(q8_hc_after));
+    ds4_cuda_q8_hc_expand_get_report(&q8_hc_after);
+    CHECK(q8_hc_after.oracle_runs == q8_hc_before.oracle_runs + 1u &&
+          q8_hc_after.oracle_failures == q8_hc_before.oracle_failures,
+          "Q8 HC oracle coverage counters");
+    ds4_gpu_set_streaming_expert_cache_budget(3u);
+    ds4_gpu_set_streaming_expert_cache_expert_bytes(40u);
+    CHECK(ds4_gpu_stream_expert_cache_configured_count() == 0u &&
+          ds4_gpu_stream_expert_cache_budget_for_expert_size(16u, 8u) == 0u &&
+          ds4_gpu_stream_expert_cache_current_count() == 0u,
+          "persistent expert arena remains runtime-inert before loader wiring");
+    ds4_cuda_stream_expert_persistent_report arena_before;
+    memset(&arena_before, 0, sizeof(arena_before));
+    ds4_cuda_stream_expert_persistent_get_report(&arena_before);
+    CHECK(ds4_cuda_test_stream_expert_persistent_arena(),
+          "persistent expert device arena allocation/reuse/reinit oracle");
+    ds4_cuda_stream_expert_persistent_report arena_after;
+    memset(&arena_after, 0, sizeof(arena_after));
+    ds4_cuda_stream_expert_persistent_get_report(&arena_after);
+    CHECK(arena_after.arena_allocations >=
+              arena_before.arena_allocations + 2u &&
+          arena_after.arena_reuses > arena_before.arena_reuses &&
+          arena_after.arena_releases >= arena_before.arena_releases + 2u &&
+          arena_after.arena_failures == arena_before.arena_failures &&
+          arena_after.arena_oracle_runs ==
+              arena_before.arena_oracle_runs + 1u &&
+          arena_after.arena_oracle_failures ==
+              arena_before.arena_oracle_failures,
+          "persistent expert device arena coverage counters");
+    ds4_cuda_stream_expert_persistent_report runtime_before;
+    memset(&runtime_before, 0, sizeof(runtime_before));
+    ds4_cuda_stream_expert_persistent_get_report(&runtime_before);
+    CHECK(ds4_cuda_test_stream_expert_persistent_runtime(),
+          "persistent expert cold/hit/mixed/eviction/fault runtime oracle");
+    ds4_cuda_stream_expert_persistent_report runtime_after;
+    memset(&runtime_after, 0, sizeof(runtime_after));
+    ds4_cuda_stream_expert_persistent_get_report(&runtime_after);
+    CHECK(runtime_after.runtime_oracle_runs ==
+              runtime_before.runtime_oracle_runs + 1u &&
+          runtime_after.runtime_oracle_failures ==
+              runtime_before.runtime_oracle_failures &&
+          runtime_after.epochs_attempted >=
+              runtime_before.epochs_attempted + 7u &&
+          runtime_after.epochs_published ==
+              runtime_before.epochs_published + 5u &&
+          runtime_after.all_hit_epochs ==
+              runtime_before.all_hit_epochs + 2u &&
+          runtime_after.miss_epochs == runtime_before.miss_epochs + 3u &&
+          runtime_after.miss_experts >= runtime_before.miss_experts + 5u &&
+          runtime_after.weight_bytes_uploaded >=
+              runtime_before.weight_bytes_uploaded + 200u &&
+          runtime_after.remap_bytes_uploaded >=
+              runtime_before.remap_bytes_uploaded + 44u &&
+          runtime_after.upload_failures ==
+              runtime_before.upload_failures + 1u &&
+          runtime_after.slot_invalidations >=
+              runtime_before.slot_invalidations + 2u &&
+          runtime_after.poisons >= runtime_before.poisons + 1u &&
+          runtime_after.persistent_dispatches >=
+              runtime_before.persistent_dispatches + 5u,
+          "persistent expert runtime coverage counters");
+    ds4_cuda_iq2_ssd_grouped_report lease_before;
+    memset(&lease_before, 0, sizeof(lease_before));
+    ds4_cuda_iq2_ssd_grouped_get_report(&lease_before);
+    CHECK(ds4_cuda_test_iq2_ssd_grouped_lease(),
+          "IQ2 SSD compact-binding consume/reuse lease oracle");
+    ds4_cuda_iq2_ssd_grouped_report lease_after;
+    memset(&lease_after, 0, sizeof(lease_after));
+    ds4_cuda_iq2_ssd_grouped_get_report(&lease_after);
+    CHECK(lease_after.lease_records > lease_before.lease_records &&
+          lease_after.lease_waits > lease_before.lease_waits &&
+          lease_after.lease_drains > lease_before.lease_drains,
+          "IQ2 SSD compact-binding lease coverage counters");
+    CHECK(ds4_cuda_test_stream_selected_event_pipeline(),
+          "selected-expert compute/readback/upload event ordering oracle");
+    ds4_cuda_stream_selected_event_pipeline_report event_report;
+    memset(&event_report, 0, sizeof(event_report));
+    ds4_cuda_stream_selected_event_pipeline_get_report(&event_report);
+    CHECK(event_report.candidates >= 1 &&
+          event_report.signals >= 1 &&
+          event_report.readbacks >= 1 &&
+          event_report.uploads >= 1 &&
+          event_report.compute_waits >= 1 &&
+          event_report.oracle_runs >= 1 &&
+          event_report.oracle_failures == 0,
+          "selected-expert event-pipeline coverage counters");
+    CHECK(ds4_cuda_test_stream_selected_batch_copy(),
+          "selected-expert batched-I/O scatter + byte oracle");
+    ds4_cuda_stream_selected_batch_io_report batch_report;
+    memset(&batch_report, 0, sizeof(batch_report));
+    ds4_cuda_stream_selected_batch_io_get_report(&batch_report);
+    CHECK(batch_report.oracle_runs >= 2 &&
+          batch_report.oracle_failures == 0 &&
+          batch_report.tasks >= 9 &&
+          batch_report.segments >= batch_report.tasks &&
+          batch_report.reads >= 7 && batch_report.bytes >= 18u * 1024u,
+          "selected-expert batched-I/O coverage counters");
 
     /* Build a synthetic 1-MiB "model" in host memory. */
     const size_t total = 1024 * 1024;
diff --git a/tests/test_metal_argmax_top1 b/tests/test_metal_argmax_top1
new file mode 100755
index 0000000000..d99aabd0d8
Binary files /dev/null and b/tests/test_metal_argmax_top1 differ
diff --git a/tests/test_metal_argmax_top1.c b/tests/test_metal_argmax_top1.c
new file mode 100644
index 0000000000..80a7f2a095
--- /dev/null
+++ b/tests/test_metal_argmax_top1.c
@@ -0,0 +1,561 @@
+#define _DARWIN_C_SOURCE
+
+#include "ds4_gpu.h"
+
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+
+#ifdef __APPLE__
+
+enum {
+    PROD_VOCAB = 129280u,
+    GUARD_WORDS = 8u,
+    OVERLAP_CALLS = 17u,
+    BENCH_CALLS = 512u,
+    TAIL_BENCH_CALLS = 128u,
+    BENCH_SAMPLES = 8u,
+};
+
+static const uint32_t k_guard = 0x7fc12345u;
+static const char *k_disable = "DS4_METAL_DISABLE_DECODE_ARGMAX_TOP1";
+static const char *k_require = "DS4_METAL_REQUIRE_DECODE_ARGMAX_TOP1";
+
+bool ds4_log_is_tty(FILE *fp) {
+    (void)fp;
+    return false;
+}
+
+static double now_sec(void) {
+    struct timespec ts;
+    clock_gettime(CLOCK_MONOTONIC, &ts);
+    return (double)ts.tv_sec + (double)ts.tv_nsec / 1.0e9;
+}
+
+static float f32_from_bits(uint32_t bits) {
+    float value = 0.0f;
+    memcpy(&value, &bits, sizeof(value));
+    return value;
+}
+
+static int cpu_argmax(const float *v, uint32_t n) {
+    int best = 0;
+    float best_v = f32_from_bits(0xff800000u);
+    for (uint32_t i = 0; i < n; i++) {
+        if (v[i] > best_v) {
+            best_v = v[i];
+            best = (int)i;
+        }
+    }
+    return best;
+}
+
+/* Mirrors the eight-way ILP used by production sample_argmax for the finite
+ * benchmark row.  The correctness oracle above intentionally stays scalar so
+ * its edge-case behavior remains obvious. */
+static int cpu_argmax_decode(const float *v, uint32_t n) {
+    int best = 0;
+    float best_v = f32_from_bits(0xff800000u);
+    int bi[8] = {0};
+    float bv[8] = {
+        best_v, best_v, best_v, best_v,
+        best_v, best_v, best_v, best_v,
+    };
+    uint32_t i = 0;
+    for (; n - i >= 8u; i += 8u) {
+        for (uint32_t lane = 0; lane < 8u; lane++) {
+            const float x = v[i + lane];
+            if (x > bv[lane]) {
+                bv[lane] = x;
+                bi[lane] = (int)(i + lane);
+            }
+        }
+    }
+    for (uint32_t lane = 0; lane < 8u; lane++) {
+        if (bv[lane] > best_v ||
+            (bv[lane] == best_v && bi[lane] < best)) {
+            best_v = bv[lane];
+            best = bi[lane];
+        }
+    }
+    for (; i < n; i++) {
+        if (v[i] > best_v) {
+            best_v = v[i];
+            best = (int)i;
+        }
+    }
+    return best;
+}
+
+static void fill_case(float *v, uint32_t n, uint32_t kind) {
+    for (uint32_t i = 0; i < n; i++) {
+        const int32_t centered = (int32_t)((i * 2654435761u + 17u) % 8191u) - 4095;
+        v[i] = (float)centered / 257.0f;
+    }
+    switch (kind) {
+        case 0:
+            v[91357u] = 1000.0f;
+            break;
+        case 1:
+            v[17u] = 2000.0f;
+            v[80003u] = 2000.0f;
+            break;
+        case 2:
+            v[9u] = f32_from_bits(0x7f800000u);
+            v[42000u] = f32_from_bits(0x7f800000u);
+            break;
+        case 3:
+            for (uint32_t i = 0; i < n; i++) {
+                v[i] = f32_from_bits(0xff800000u);
+            }
+            break;
+        case 4:
+            for (uint32_t i = 0; i < n; i++) {
+                v[i] = f32_from_bits(0x7fc00001u);
+            }
+            break;
+        case 5:
+            for (uint32_t i = 0; i < n; i++) {
+                v[i] = f32_from_bits((i & 1u) ? 0x7fc00001u : 0xff800000u);
+            }
+            v[777u] = -100.0f;
+            break;
+        default:
+            abort();
+    }
+}
+
+static int set_candidate(bool candidate) {
+    if (candidate) {
+        return unsetenv(k_disable) == 0 &&
+               setenv(k_require, "1", 1) == 0;
+    }
+    return unsetenv(k_require) == 0 &&
+           setenv(k_disable, "1", 1) == 0;
+}
+
+static int run_one(ds4_gpu_tensor *out, const ds4_gpu_tensor *logits,
+                   uint32_t n_vocab, bool candidate, int32_t *result) {
+    return set_candidate(candidate) &&
+           ds4_gpu_argmax_tensor(out, logits, n_vocab) &&
+           ds4_gpu_tensor_read(out, 0, result, sizeof(*result));
+}
+
+static int check_guarded_cases(void) {
+    const size_t total = (size_t)PROD_VOCAB + 2u * GUARD_WORDS;
+    uint32_t *host = malloc(total * sizeof(*host));
+    ds4_gpu_tensor *input_base = ds4_gpu_tensor_alloc(total * sizeof(float));
+    ds4_gpu_tensor *input = input_base ? ds4_gpu_tensor_view(
+        input_base, (uint64_t)GUARD_WORDS * sizeof(float),
+        (uint64_t)PROD_VOCAB * sizeof(float)) : NULL;
+    ds4_gpu_tensor *out_base = ds4_gpu_tensor_alloc(3u * sizeof(uint32_t));
+    ds4_gpu_tensor *out = out_base ? ds4_gpu_tensor_view(
+        out_base, sizeof(uint32_t), sizeof(uint32_t)) : NULL;
+    int ok = host && input_base && input && out_base && out;
+
+    for (uint32_t kind = 0; ok && kind < 6u; kind++) {
+        for (size_t i = 0; i < total; i++) host[i] = k_guard;
+        float *values = (float *)(host + GUARD_WORDS);
+        fill_case(values, PROD_VOCAB, kind);
+        const int expected = cpu_argmax(values, PROD_VOCAB);
+        uint32_t out_words[3] = {k_guard, k_guard, k_guard};
+        int32_t control = -1;
+        int32_t candidate = -1;
+        ok = ds4_gpu_tensor_write(input_base, 0, host,
+                                  total * sizeof(*host)) &&
+             ds4_gpu_tensor_write(out_base, 0, out_words,
+                                  sizeof(out_words)) &&
+             run_one(out, input, PROD_VOCAB, false, &control) &&
+             ds4_gpu_tensor_write(out_base, 0, out_words,
+                                  sizeof(out_words)) &&
+             run_one(out, input, PROD_VOCAB, true, &candidate) &&
+             ds4_gpu_tensor_read(out_base, 0, out_words,
+                                 sizeof(out_words)) &&
+             ds4_gpu_tensor_read(input_base, 0, host,
+                                 total * sizeof(*host));
+        if (!ok) break;
+        if (candidate != expected ||
+            (kind < 3u && control != candidate) ||
+            out_words[0] != k_guard || out_words[2] != k_guard) {
+            fprintf(stderr,
+                    "argmax case %u failed: expected=%d control=%d candidate=%d "
+                    "guards=%08x/%08x\n",
+                    kind, expected, control, candidate,
+                    out_words[0], out_words[2]);
+            ok = 0;
+        }
+        for (uint32_t i = 0; ok && i < GUARD_WORDS; i++) {
+            if (host[i] != k_guard ||
+                host[GUARD_WORDS + PROD_VOCAB + i] != k_guard) {
+                fprintf(stderr, "argmax input guard changed in case %u\n", kind);
+                ok = 0;
+            }
+        }
+    }
+
+    ds4_gpu_tensor_free(out);
+    ds4_gpu_tensor_free(out_base);
+    ds4_gpu_tensor_free(input);
+    ds4_gpu_tensor_free(input_base);
+    free(host);
+    if (ok) fprintf(stderr, "Metal top-1 production-shape oracle: PASS\n");
+    return ok;
+}
+
+static int check_vocab_shapes(void) {
+    static const uint32_t sizes[] = {4096u, 4097u, 129279u, 129281u};
+    const uint32_t max_n = sizes[sizeof(sizes) / sizeof(sizes[0]) - 1u];
+    float *host = malloc((size_t)max_n * sizeof(*host));
+    ds4_gpu_tensor *logits = ds4_gpu_tensor_alloc(
+        (uint64_t)max_n * sizeof(float));
+    ds4_gpu_tensor *out = ds4_gpu_tensor_alloc(sizeof(int32_t));
+    int ok = host && logits && out;
+    for (size_t shape = 0;
+         ok && shape < sizeof(sizes) / sizeof(sizes[0]);
+         shape++) {
+        const uint32_t n = sizes[shape];
+        for (uint32_t i = 0; i < n; i++) {
+            host[i] = -(float)((i * 17u + 3u) % 8191u);
+        }
+        host[n - 1u] = 2000.0f;
+        const int expected = cpu_argmax(host, n);
+        int32_t control = -1;
+        int32_t candidate = -1;
+        ok = ds4_gpu_tensor_write(logits, 0, host,
+                                  (uint64_t)n * sizeof(*host)) &&
+             run_one(out, logits, n, false, &control) &&
+             run_one(out, logits, n, true, &candidate);
+        if (ok && (control != expected || candidate != expected)) {
+            fprintf(stderr,
+                    "Metal top-1 shape %u failed: expected=%d control=%d candidate=%d\n",
+                    n, expected, control, candidate);
+            ok = 0;
+        }
+    }
+    ds4_gpu_tensor_free(out);
+    ds4_gpu_tensor_free(logits);
+    free(host);
+    if (ok) fprintf(stderr, "Metal top-1 tail/empty-group shapes: PASS\n");
+    return ok;
+}
+
+static int check_scratch_wrap(void) {
+    const size_t values = (size_t)OVERLAP_CALLS * PROD_VOCAB;
+    float *host = malloc(values * sizeof(*host));
+    int32_t expected[OVERLAP_CALLS];
+    int32_t results[OVERLAP_CALLS + 2u];
+    ds4_gpu_tensor *logits = ds4_gpu_tensor_alloc(values * sizeof(float));
+    ds4_gpu_tensor *out_base = ds4_gpu_tensor_alloc(sizeof(results));
+    ds4_gpu_tensor *rows[OVERLAP_CALLS] = {0};
+    ds4_gpu_tensor *outs[OVERLAP_CALLS] = {0};
+    int ok = host && logits && out_base;
+
+    for (uint32_t row = 0; ok && row < OVERLAP_CALLS; row++) {
+        float *v = host + (size_t)row * PROD_VOCAB;
+        for (uint32_t i = 0; i < PROD_VOCAB; i++) {
+            v[i] = -(float)((i + row * 13u) % 4093u);
+        }
+        const uint32_t winner = (row * 7919u + 23u) % PROD_VOCAB;
+        v[winner] = 1000.0f + (float)row;
+        expected[row] = (int32_t)winner;
+        rows[row] = ds4_gpu_tensor_view(
+            logits, (uint64_t)row * PROD_VOCAB * sizeof(float),
+            (uint64_t)PROD_VOCAB * sizeof(float));
+        outs[row] = ds4_gpu_tensor_view(
+            out_base, (uint64_t)(row + 1u) * sizeof(int32_t),
+            sizeof(int32_t));
+        ok = rows[row] && outs[row];
+    }
+    for (uint32_t i = 0; i < OVERLAP_CALLS + 2u; i++) {
+        results[i] = (int32_t)k_guard;
+    }
+    if (ok) {
+        ok = ds4_gpu_tensor_write(logits, 0, host,
+                                  values * sizeof(float)) &&
+             ds4_gpu_tensor_write(out_base, 0, results, sizeof(results)) &&
+             set_candidate(true) && ds4_gpu_begin_commands();
+    }
+    for (uint32_t row = 0; ok && row < OVERLAP_CALLS; row++) {
+        ok = ds4_gpu_argmax_tensor(outs[row], rows[row], PROD_VOCAB);
+        if (ok && row + 1u < OVERLAP_CALLS) {
+            ok = ds4_gpu_flush_commands();
+        }
+    }
+    if (ok) ok = ds4_gpu_end_commands();
+    else (void)ds4_gpu_synchronize();
+    if (ok) ok = ds4_gpu_tensor_read(out_base, 0, results, sizeof(results));
+    if (ok && ((uint32_t)results[0] != k_guard ||
+               (uint32_t)results[OVERLAP_CALLS + 1u] != k_guard)) {
+        fprintf(stderr, "argmax overlap output guard changed\n");
+        ok = 0;
+    }
+    for (uint32_t row = 0; ok && row < OVERLAP_CALLS; row++) {
+        if (results[row + 1u] != expected[row]) {
+            fprintf(stderr,
+                    "argmax scratch wrap failed at call %u: expected=%d got=%d\n",
+                    row, expected[row], results[row + 1u]);
+            ok = 0;
+        }
+    }
+
+    for (uint32_t row = 0; row < OVERLAP_CALLS; row++) {
+        ds4_gpu_tensor_free(outs[row]);
+        ds4_gpu_tensor_free(rows[row]);
+    }
+    ds4_gpu_tensor_free(out_base);
+    ds4_gpu_tensor_free(logits);
+    free(host);
+    if (ok) {
+        fprintf(stderr,
+                "Metal top-1 scratch-ring wrap (%u command-buffer submissions "
+                "without host waits): PASS\n",
+                OVERLAP_CALLS);
+    }
+    return ok;
+}
+
+static int timed_batch(ds4_gpu_tensor *out,
+                       const ds4_gpu_tensor *logits,
+                       bool candidate,
+                       uint32_t calls,
+                       double *elapsed) {
+    if (!set_candidate(candidate) || !ds4_gpu_begin_commands()) return 0;
+    const double t0 = now_sec();
+    int ok = 1;
+    for (uint32_t i = 0; ok && i < calls; i++) {
+        ok = ds4_gpu_argmax_tensor(out, logits, PROD_VOCAB);
+    }
+    if (ok) ok = ds4_gpu_end_commands();
+    else (void)ds4_gpu_synchronize();
+    const double t1 = now_sec();
+    if (ok && elapsed) *elapsed = t1 - t0;
+    return ok;
+}
+
+static int cmp_double(const void *a, const void *b) {
+    const double x = *(const double *)a;
+    const double y = *(const double *)b;
+    return (x > y) - (x < y);
+}
+
+static int timed_selection_tail(
+        ds4_gpu_tensor *out,
+        ds4_gpu_tensor *logits,
+        const ds4_gpu_tensor *producer,
+        float *readback,
+        int expected,
+        bool candidate,
+        uint32_t calls,
+        double *elapsed) {
+    if (!set_candidate(true)) return 0;
+    const double t0 = now_sec();
+    int ok = 1;
+    for (uint32_t i = 0; ok && i < calls; i++) {
+        int32_t got = -1;
+        ok = ds4_gpu_begin_commands();
+        if (ok) {
+            ok = ds4_gpu_tensor_copy(
+                logits, 0, producer, 0,
+                (uint64_t)PROD_VOCAB * sizeof(float));
+        }
+        if (ok && candidate) {
+            ok = ds4_gpu_argmax_tensor(out, logits, PROD_VOCAB);
+        }
+        if (ok) ok = ds4_gpu_end_commands();
+        else (void)ds4_gpu_synchronize();
+        if (ok && candidate) {
+            ok = ds4_gpu_tensor_read(out, 0, &got, sizeof(got));
+        } else if (ok) {
+            ok = ds4_gpu_tensor_read(logits, 0, readback,
+                                     (uint64_t)PROD_VOCAB * sizeof(*readback));
+            if (ok) got = cpu_argmax_decode(readback, PROD_VOCAB);
+        }
+        if (ok && got != expected) {
+            fprintf(stderr,
+                    "Metal selection-tail mismatch: candidate=%d expected=%d got=%d\n",
+                    candidate, expected, got);
+            ok = 0;
+        }
+    }
+    const double t1 = now_sec();
+    if (ok && elapsed) *elapsed = t1 - t0;
+    return ok;
+}
+
+static int run_selection_tail_benchmark(
+        ds4_gpu_tensor *out,
+        ds4_gpu_tensor *logits,
+        const ds4_gpu_tensor *producer,
+        const float *values) {
+    float *readback = malloc((size_t)PROD_VOCAB * sizeof(*readback));
+    int ok = readback != NULL;
+    const int expected = cpu_argmax_decode(values, PROD_VOCAB);
+    double ignored = 0.0;
+    if (ok) {
+        ok = timed_selection_tail(out, logits, producer, readback, expected,
+                                  false, 16u, &ignored);
+    }
+    if (ok) {
+        ok = timed_selection_tail(out, logits, producer, readback, expected,
+                                  true, 16u, &ignored);
+    }
+
+    double control[BENCH_SAMPLES] = {0};
+    double candidate[BENCH_SAMPLES] = {0};
+    uint32_t nc = 0;
+    uint32_t nn = 0;
+    for (uint32_t cycle = 0; ok && cycle < BENCH_SAMPLES / 2u; cycle++) {
+        const bool order[4] = {false, true, true, false};
+        for (uint32_t j = 0; ok && j < 4u; j++) {
+            double elapsed = 0.0;
+            ok = timed_selection_tail(out, logits, producer,
+                                      readback, expected,
+                                      order[j], TAIL_BENCH_CALLS, &elapsed);
+            if (!ok) break;
+            if (order[j]) candidate[nn++] = elapsed;
+            else control[nc++] = elapsed;
+        }
+    }
+    if (ok && (nc != BENCH_SAMPLES || nn != BENCH_SAMPLES)) ok = 0;
+    if (ok) {
+        qsort(control, BENCH_SAMPLES, sizeof(control[0]), cmp_double);
+        qsort(candidate, BENCH_SAMPLES, sizeof(candidate[0]), cmp_double);
+        const double control_median =
+            0.5 * (control[BENCH_SAMPLES / 2u - 1u] +
+                   control[BENCH_SAMPLES / 2u]);
+        const double candidate_median =
+            0.5 * (candidate[BENCH_SAMPLES / 2u - 1u] +
+                   candidate[BENCH_SAMPLES / 2u]);
+        const double control_us =
+            control_median * 1.0e6 / TAIL_BENCH_CALLS;
+        const double candidate_us =
+            candidate_median * 1.0e6 / TAIL_BENCH_CALLS;
+        const double saved_us = control_us - candidate_us;
+        fprintf(stderr,
+                "Metal resident greedy-selection tail A/B (%u logits, "
+                "%u calls/sample, %u samples): full-read+CPU=%.3f us "
+                "p95=%.3f us device-top1+4B=%.3f us p95=%.3f us "
+                "saved=%.3f us speedup=%.2fx reduction=%.1f%%\n",
+                PROD_VOCAB, TAIL_BENCH_CALLS, BENCH_SAMPLES,
+                control_us,
+                control[BENCH_SAMPLES - 1u] * 1.0e6 / TAIL_BENCH_CALLS,
+                candidate_us,
+                candidate[BENCH_SAMPLES - 1u] * 1.0e6 / TAIL_BENCH_CALLS,
+                saved_us, control_us / candidate_us,
+                saved_us * 100.0 / control_us);
+        fprintf(stderr,
+                "Metal resident greedy-selection tail excludes GGUF/model access and SSD I/O.\n");
+    }
+    free(readback);
+    return ok;
+}
+
+static int run_benchmark(void) {
+    float *host = malloc((size_t)PROD_VOCAB * sizeof(*host));
+    ds4_gpu_tensor *logits = ds4_gpu_tensor_alloc(
+        (uint64_t)PROD_VOCAB * sizeof(float));
+    ds4_gpu_tensor *producer = ds4_gpu_tensor_alloc(
+        (uint64_t)PROD_VOCAB * sizeof(float));
+    ds4_gpu_tensor *out = ds4_gpu_tensor_alloc(sizeof(int32_t));
+    int ok = host && logits && producer && out;
+    if (ok) {
+        fill_case(host, PROD_VOCAB, 0u);
+        ok = ds4_gpu_tensor_write(logits, 0, host,
+                                  (uint64_t)PROD_VOCAB * sizeof(float)) &&
+             ds4_gpu_tensor_write(producer, 0, host,
+                                  (uint64_t)PROD_VOCAB * sizeof(float));
+    }
+
+    double ignored = 0.0;
+    if (ok) ok = timed_batch(out, logits, false, 64u, &ignored);
+    if (ok) ok = timed_batch(out, logits, true, 64u, &ignored);
+
+    double control[BENCH_SAMPLES] = {0};
+    double candidate[BENCH_SAMPLES] = {0};
+    uint32_t nc = 0;
+    uint32_t nn = 0;
+    for (uint32_t cycle = 0; ok && cycle < BENCH_SAMPLES / 2u; cycle++) {
+        const bool order[4] = {false, true, true, false};
+        for (uint32_t j = 0; ok && j < 4u; j++) {
+            double elapsed = 0.0;
+            ok = timed_batch(out, logits, order[j], BENCH_CALLS, &elapsed);
+            if (!ok) break;
+            int32_t got = -1;
+            ok = ds4_gpu_tensor_read(out, 0, &got, sizeof(got));
+            if (!ok || got != 91357) {
+                fprintf(stderr,
+                        "Metal timed top-1 output mismatch: candidate=%d expected=91357 got=%d\n",
+                        order[j], got);
+                ok = 0;
+                break;
+            }
+            if (order[j]) candidate[nn++] = elapsed;
+            else control[nc++] = elapsed;
+        }
+    }
+    if (ok && (nc != BENCH_SAMPLES || nn != BENCH_SAMPLES)) ok = 0;
+    if (ok) {
+        qsort(control, BENCH_SAMPLES, sizeof(control[0]), cmp_double);
+        qsort(candidate, BENCH_SAMPLES, sizeof(candidate[0]), cmp_double);
+        const double control_median =
+            0.5 * (control[BENCH_SAMPLES / 2u - 1u] +
+                   control[BENCH_SAMPLES / 2u]);
+        const double candidate_median =
+            0.5 * (candidate[BENCH_SAMPLES / 2u - 1u] +
+                   candidate[BENCH_SAMPLES / 2u]);
+        const double control_us = control_median * 1.0e6 / BENCH_CALLS;
+        const double candidate_us = candidate_median * 1.0e6 / BENCH_CALLS;
+        const double saved_us = control_us - candidate_us;
+        const double speedup = control_us / candidate_us;
+        const double gain = saved_us * 100.0 / control_us;
+        fprintf(stderr,
+                "Metal resident warm top-1 A/B (%u logits, %u calls/sample, "
+                "%u samples): control=%.3f us p95=%.3f us "
+                "candidate=%.3f us p95=%.3f us saved=%.3f us "
+                "speedup=%.2fx reduction=%.1f%%\n",
+                PROD_VOCAB, BENCH_CALLS, BENCH_SAMPLES,
+                control_us,
+                control[BENCH_SAMPLES - 1u] * 1.0e6 / BENCH_CALLS,
+                candidate_us,
+                candidate[BENCH_SAMPLES - 1u] * 1.0e6 / BENCH_CALLS,
+                saved_us, speedup, gain);
+        fprintf(stderr,
+                "Metal resident top-1 A/B excludes GGUF/model access and SSD I/O.\n");
+    }
+
+    if (ok) {
+        ok = run_selection_tail_benchmark(out, logits, producer, host);
+    }
+
+    ds4_gpu_tensor_free(out);
+    ds4_gpu_tensor_free(producer);
+    ds4_gpu_tensor_free(logits);
+    free(host);
+    return ok;
+}
+
+int main(void) {
+    int ok = ds4_gpu_init();
+    if (ok) ok = check_guarded_cases();
+    if (ok) ok = check_vocab_shapes();
+    if (ok) ok = check_scratch_wrap();
+    if (ok && getenv("DS4_TEST_METAL_ARGMAX_TOP1_TIMING") != NULL) {
+        ok = run_benchmark();
+    }
+    (void)unsetenv(k_disable);
+    (void)unsetenv(k_require);
+    ds4_gpu_cleanup();
+    return ok ? 0 : 1;
+}
+
+#else
+
+int main(void) {
+    fprintf(stderr, "test_metal_argmax_top1: skipped (Metal requires macOS)\n");
+    return 0;
+}
+
+#endif
diff --git a/tests/test_metal_dspark_capture b/tests/test_metal_dspark_capture
new file mode 100755
index 0000000000..2d881efef7
Binary files /dev/null and b/tests/test_metal_dspark_capture differ
diff --git a/tests/test_metal_dspark_capture.c b/tests/test_metal_dspark_capture.c
new file mode 100644
index 0000000000..dcbc850bfe
--- /dev/null
+++ b/tests/test_metal_dspark_capture.c
@@ -0,0 +1,204 @@
+#define _DARWIN_C_SOURCE
+
+#include "ds4_gpu.h"
+
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+enum {
+    TEST_HC = 4,
+    GUARD_FLOATS = 11,
+};
+
+static const uint32_t guard_bits = 0x7fc12345u;
+
+bool ds4_log_is_tty(FILE *fp) {
+    (void)fp;
+    return false;
+}
+
+static void fill_guard(uint32_t *dst, size_t count) {
+    for (size_t i = 0; i < count; i++) dst[i] = guard_bits;
+}
+
+static int guard_is_intact(const char *name,
+                           const uint32_t *storage,
+                           size_t payload_floats) {
+    for (size_t i = 0; i < GUARD_FLOATS; i++) {
+        if (storage[i] != guard_bits) {
+            fprintf(stderr, "%s prefix guard changed at %zu: 0x%08x\n",
+                    name, i, storage[i]);
+            return 0;
+        }
+    }
+    for (size_t i = 0; i < GUARD_FLOATS; i++) {
+        const size_t at = GUARD_FLOATS + payload_floats + i;
+        if (storage[at] != guard_bits) {
+            fprintf(stderr, "%s suffix guard changed at %zu: 0x%08x\n",
+                    name, i, storage[at]);
+            return 0;
+        }
+    }
+    return 1;
+}
+
+static ds4_gpu_tensor *make_guarded_tensor(size_t payload_floats,
+                                            uint32_t **host_storage) {
+    const size_t total = payload_floats + 2u * GUARD_FLOATS;
+    uint32_t *storage = malloc(total * sizeof(*storage));
+    if (!storage) return NULL;
+    fill_guard(storage, total);
+
+    ds4_gpu_tensor *base = ds4_gpu_tensor_alloc(total * sizeof(float));
+    if (!base || !ds4_gpu_tensor_write(base, 0, storage,
+                                       total * sizeof(*storage))) {
+        ds4_gpu_tensor_free(base);
+        free(storage);
+        return NULL;
+    }
+    *host_storage = storage;
+    return base;
+}
+
+static int read_guarded_tensor(ds4_gpu_tensor *base,
+                               uint32_t *storage,
+                               size_t payload_floats) {
+    const size_t total = payload_floats + 2u * GUARD_FLOATS;
+    return ds4_gpu_tensor_read(base, 0, storage,
+                               total * sizeof(*storage));
+}
+
+static int run_case(uint32_t rows, uint32_t n_embd) {
+    const size_t x_floats = (size_t)rows * TEST_HC * n_embd;
+    const size_t weights_floats = (size_t)rows * TEST_HC;
+    const size_t out_floats = (size_t)rows * n_embd;
+    const size_t last_floats = n_embd;
+    uint32_t *x_storage = NULL;
+    uint32_t *weights_storage = NULL;
+    uint32_t *ref_storage = NULL;
+    uint32_t *fused_storage = NULL;
+    uint32_t *last_storage = NULL;
+    ds4_gpu_tensor *x_base = make_guarded_tensor(x_floats, &x_storage);
+    ds4_gpu_tensor *weights_base =
+        make_guarded_tensor(weights_floats, &weights_storage);
+    ds4_gpu_tensor *ref_base = make_guarded_tensor(out_floats, &ref_storage);
+    ds4_gpu_tensor *fused_base =
+        make_guarded_tensor(out_floats, &fused_storage);
+    ds4_gpu_tensor *last_base =
+        make_guarded_tensor(last_floats, &last_storage);
+    ds4_gpu_tensor *x = NULL;
+    ds4_gpu_tensor *weights = NULL;
+    ds4_gpu_tensor *ref = NULL;
+    ds4_gpu_tensor *fused = NULL;
+    ds4_gpu_tensor *last = NULL;
+    int ok = x_base && weights_base && ref_base && fused_base && last_base;
+
+    if (ok) {
+        float *x_values = (float *)(x_storage + GUARD_FLOATS);
+        float *weight_values = (float *)(weights_storage + GUARD_FLOATS);
+        for (size_t i = 0; i < x_floats; i++) {
+            const int32_t centered = (int32_t)((i * 37u + 13u) % 257u) - 128;
+            x_values[i] = (float)centered / 32.0f;
+        }
+        for (size_t i = 0; i < weights_floats; i++) {
+            weight_values[i] = 0.25f;
+        }
+        ok = ds4_gpu_tensor_write(x_base, 0, x_storage,
+                                  (x_floats + 2u * GUARD_FLOATS) * sizeof(float)) &&
+             ds4_gpu_tensor_write(weights_base, 0, weights_storage,
+                                  (weights_floats + 2u * GUARD_FLOATS) * sizeof(float));
+    }
+    if (ok) {
+        x = ds4_gpu_tensor_view(x_base,
+                (uint64_t)GUARD_FLOATS * sizeof(float),
+                (uint64_t)x_floats * sizeof(float));
+        weights = ds4_gpu_tensor_view(weights_base,
+                (uint64_t)GUARD_FLOATS * sizeof(float),
+                (uint64_t)weights_floats * sizeof(float));
+        ref = ds4_gpu_tensor_view(ref_base,
+                (uint64_t)GUARD_FLOATS * sizeof(float),
+                (uint64_t)out_floats * sizeof(float));
+        fused = ds4_gpu_tensor_view(fused_base,
+                (uint64_t)GUARD_FLOATS * sizeof(float),
+                (uint64_t)out_floats * sizeof(float));
+        last = ds4_gpu_tensor_view(last_base,
+                (uint64_t)GUARD_FLOATS * sizeof(float),
+                (uint64_t)last_floats * sizeof(float));
+        ok = x && weights && ref && fused && last;
+    }
+    if (ok) {
+        ok = ds4_gpu_hc_weighted_sum_tensor(ref, x, weights,
+                                             n_embd, TEST_HC) &&
+             ds4_gpu_hc_weighted_sum_capture_last_tensor(fused, last,
+                                                          x, weights,
+                                                          n_embd, TEST_HC) &&
+             ds4_gpu_synchronize();
+    }
+    if (ok) {
+        ok = read_guarded_tensor(ref_base, ref_storage, out_floats) &&
+             read_guarded_tensor(fused_base, fused_storage, out_floats) &&
+             read_guarded_tensor(last_base, last_storage, last_floats) &&
+             read_guarded_tensor(x_base, x_storage, x_floats) &&
+             read_guarded_tensor(weights_base, weights_storage, weights_floats);
+    }
+    if (ok && memcmp(ref_storage + GUARD_FLOATS,
+                     fused_storage + GUARD_FLOATS,
+                     out_floats * sizeof(float)) != 0) {
+        fprintf(stderr, "DSpark capture batch mismatch rows=%u embd=%u\n",
+                rows, n_embd);
+        ok = 0;
+    }
+    if (ok && memcmp(ref_storage + GUARD_FLOATS +
+                         (size_t)(rows - 1u) * n_embd,
+                     last_storage + GUARD_FLOATS,
+                     last_floats * sizeof(float)) != 0) {
+        fprintf(stderr, "DSpark capture last-row mismatch rows=%u embd=%u\n",
+                rows, n_embd);
+        ok = 0;
+    }
+    if (ok) {
+        ok = guard_is_intact("input", x_storage, x_floats) &&
+             guard_is_intact("weights", weights_storage, weights_floats) &&
+             guard_is_intact("reference", ref_storage, out_floats) &&
+             guard_is_intact("fused batch", fused_storage, out_floats) &&
+             guard_is_intact("fused last", last_storage, last_floats);
+    }
+
+    ds4_gpu_tensor_free(last);
+    ds4_gpu_tensor_free(fused);
+    ds4_gpu_tensor_free(ref);
+    ds4_gpu_tensor_free(weights);
+    ds4_gpu_tensor_free(x);
+    ds4_gpu_tensor_free(last_base);
+    ds4_gpu_tensor_free(fused_base);
+    ds4_gpu_tensor_free(ref_base);
+    ds4_gpu_tensor_free(weights_base);
+    ds4_gpu_tensor_free(x_base);
+    free(last_storage);
+    free(fused_storage);
+    free(ref_storage);
+    free(weights_storage);
+    free(x_storage);
+
+    if (ok) {
+        fprintf(stderr, "DSpark Metal capture rows=%u embd=%u bitwise PASS\n",
+                rows, n_embd);
+    }
+    return ok;
+}
+
+int main(void) {
+    int ok = ds4_gpu_init();
+    const uint32_t rows[] = {1u, 2u, 5u};
+    const uint32_t embd[] = {17u, 4096u};
+    for (size_t d = 0; ok && d < sizeof(embd) / sizeof(embd[0]); d++) {
+        for (size_t r = 0; ok && r < sizeof(rows) / sizeof(rows[0]); r++) {
+            ok = run_case(rows[r], embd[d]);
+        }
+    }
+    ds4_gpu_cleanup();
+    return ok ? 0 : 1;
+}
diff --git a/tests/test_metal_exactn_oracle.c b/tests/test_metal_exactn_oracle.c
new file mode 100644
index 0000000000..6198caa141
--- /dev/null
+++ b/tests/test_metal_exactn_oracle.c
@@ -0,0 +1,544 @@
+/* Model-backed correctness oracle for the Metal exact-N speculative path.
+ *
+ * The DSpark proposer is intentionally not involved: the test derives five
+ * target-greedy tokens, injects controlled full/partial/EOS draft blocks, and
+ * sends them through the same production verifier/commit function.  Each
+ * resulting session is compared with ordinary one-token decode at three
+ * levels: serialized KV/compressor state, continuation logits, and a short
+ * greedy continuation.
+ *
+ * Run with:
+ *   DS4_TEST_MODEL=/path/to/model.gguf make test-metal-exactn-oracle
+ *
+ * Running the binary directly without a model is a developer-friendly skip.
+ * The Make target is the release gate: it sets DS4_TEST_REQUIRE_MODEL=1 and
+ * therefore fails when the configured model is absent.
+ */
+
+#include "ds4.h"
+
+#include <float.h>
+#include <math.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#define TEST_CTX 512
+#define TEST_PREFILL_CHUNK 128u
+/* Partial-N deliberately falls back to the legacy five-row verifier, whose
+ * routed union may need 5 * top6 slots before replaying the accepted prefix. */
+#define TEST_EXPERT_CACHE 32u
+#define MAX_DRAFT 5
+#define CONTINUATION_TOKENS 4
+
+/* This symbol exists only in the ds4 core compiled with DS4_TEST_HOOKS. */
+int ds4_test_session_eval_exact_drafts(
+        ds4_session *s,
+        const int   *drafts,
+        int          draft_n,
+        int          eos_token,
+        int         *accepted,
+        int          accepted_cap,
+        char        *err,
+        size_t       errlen);
+
+enum {
+    EXACTN_UNION_ATTEMPTS = 0,
+    EXACTN_UNION_FULL_ACCEPTS,
+    EXACTN_UNION_FALLBACKS,
+    EXACTN_UNION_PARTIAL_FALLBACKS,
+    EXACTN_UNION_ERROR_FALLBACKS,
+    EXACTN_UNION_PARTIAL_REPLAYS,
+    EXACTN_UNION_VERIFY_SKIPS,
+    EXACTN_UNION_BATCH_HEAD_ATTEMPTS,
+    EXACTN_UNION_BATCH_HEAD_USES,
+    EXACTN_UNION_BATCH_HEAD_FALLBACKS,
+    EXACTN_UNION_COUNTER_COUNT
+};
+
+int ds4_test_session_exactn_union_stats(
+        const ds4_session *s,
+        uint64_t           out[EXACTN_UNION_COUNTER_COUNT]);
+
+typedef struct {
+    const char *name;
+    int draft_n;
+    int reject_at; /* -1 means every draft is target-greedy. */
+    int eos_at;    /* -1 uses the model EOS; otherwise a synthetic EOS row. */
+} exactn_case;
+
+static void fail(const char *what, const char *case_name, const char *detail) {
+    fprintf(stderr, "FAIL: %s case=%s%s%s\n",
+            what,
+            case_name ? case_name : "setup",
+            detail && detail[0] ? ": " : "",
+            detail && detail[0] ? detail : "");
+    exit(1);
+}
+
+static void restore_snapshot(ds4_session *s,
+                             const ds4_session_snapshot *snap,
+                             const char *case_name) {
+    char err[256] = "";
+    if (ds4_session_load_snapshot(s, snap, err, sizeof(err)) != 0) {
+        fail("snapshot restore", case_name, err);
+    }
+}
+
+static void save_snapshot(ds4_session *s,
+                          ds4_session_snapshot *snap,
+                          const char *case_name) {
+    char err[256] = "";
+    if (ds4_session_save_snapshot(s, snap, err, sizeof(err)) != 0) {
+        fail("snapshot save", case_name, err);
+    }
+}
+
+static void eval_tokens(ds4_session *s,
+                        const int *tokens,
+                        int count,
+                        const char *case_name) {
+    char err[256] = "";
+    for (int i = 0; i < count; i++) {
+        if (ds4_session_eval(s, tokens[i], err, sizeof(err)) != 0) {
+            char detail[320];
+            snprintf(detail, sizeof(detail), "row=%d token=%d err=%s",
+                     i, tokens[i], err);
+            fail("sequential decode", case_name, detail);
+        }
+    }
+}
+
+static int copy_logits(ds4_session *s, float *out, int vocab,
+                       const char *case_name) {
+    const int copied = ds4_session_copy_logits(s, out, vocab);
+    if (copied != vocab) {
+        char detail[96];
+        snprintf(detail, sizeof(detail), "copied=%d vocab=%d", copied, vocab);
+        fail("logits read", case_name, detail);
+    }
+    return copied;
+}
+
+static int lowest_finite_token(const float *logits, int vocab,
+                               int excluded_a, int excluded_b) {
+    int token = -1;
+    float value = FLT_MAX;
+    for (int i = 0; i < vocab; i++) {
+        if (i == excluded_a || i == excluded_b) continue;
+        if (token < 0 || logits[i] < value) {
+            token = i;
+            value = logits[i];
+        }
+    }
+    return token;
+}
+
+static int greedy_continuation(ds4_session *s, int eos,
+                               int out[CONTINUATION_TOKENS],
+                               const char *case_name) {
+    char err[256] = "";
+    int n = 0;
+    while (n < CONTINUATION_TOKENS) {
+        const int token = ds4_session_argmax(s);
+        if (token < 0) fail("continuation argmax", case_name, "negative token");
+        out[n++] = token;
+        if (token == eos || n == CONTINUATION_TOKENS) break;
+        if (ds4_session_eval(s, token, err, sizeof(err)) != 0) {
+            fail("continuation decode", case_name, err);
+        }
+    }
+    return n;
+}
+
+static void compare_logits(const float *expected, const float *actual,
+                           int vocab, const char *case_name) {
+    if (memcmp(expected, actual, (size_t)vocab * sizeof(*actual)) == 0) return;
+
+    int differing = 0;
+    int first = -1;
+    float max_abs = 0.0f;
+    for (int i = 0; i < vocab; i++) {
+        if (memcmp(&expected[i], &actual[i], sizeof(actual[i])) != 0) {
+            if (first < 0) first = i;
+            differing++;
+        }
+        float delta = fabsf(expected[i] - actual[i]);
+        if (!isfinite(delta)) delta = FLT_MAX;
+        if (delta > max_abs) max_abs = delta;
+    }
+    char detail[256];
+    snprintf(detail, sizeof(detail),
+             "differing=%d first=%d expected=%g actual=%g max_abs=%g",
+             differing, first,
+             first >= 0 ? expected[first] : 0.0f,
+             first >= 0 ? actual[first] : 0.0f,
+             max_abs);
+    fail("continuation logits mismatch", case_name, detail);
+}
+
+static void compare_snapshots(const ds4_session_snapshot *expected,
+                              const ds4_session_snapshot *actual,
+                              const char *case_name) {
+    if (expected->len != actual->len) {
+        char detail[160];
+        snprintf(detail, sizeof(detail), "expected_bytes=%llu actual_bytes=%llu",
+                 (unsigned long long)expected->len,
+                 (unsigned long long)actual->len);
+        fail("state snapshot length mismatch", case_name, detail);
+    }
+    if (memcmp(expected->ptr, actual->ptr, (size_t)expected->len) == 0) return;
+
+    uint64_t first = 0;
+    while (first < expected->len &&
+           expected->ptr[first] == actual->ptr[first]) {
+        first++;
+    }
+    char detail[192];
+    snprintf(detail, sizeof(detail),
+             "first_byte=%llu expected=0x%02x actual=0x%02x bytes=%llu",
+             (unsigned long long)first,
+             first < expected->len ? expected->ptr[first] : 0,
+             first < actual->len ? actual->ptr[first] : 0,
+             (unsigned long long)expected->len);
+    fail("KV/compressor snapshot mismatch", case_name, detail);
+}
+
+static void compare_continuations(const int *expected, int expected_n,
+                                  const int *actual, int actual_n,
+                                  const char *case_name) {
+    if (expected_n == actual_n &&
+        memcmp(expected, actual, (size_t)expected_n * sizeof(*actual)) == 0) {
+        return;
+    }
+    int first = 0;
+    const int common = expected_n < actual_n ? expected_n : actual_n;
+    while (first < common && expected[first] == actual[first]) first++;
+    char detail[192];
+    snprintf(detail, sizeof(detail),
+             "expected_n=%d actual_n=%d first=%d expected=%d actual=%d",
+             expected_n, actual_n, first,
+             first < expected_n ? expected[first] : -1,
+             first < actual_n ? actual[first] : -1);
+    fail("greedy continuation mismatch", case_name, detail);
+}
+
+static void read_union_stats(
+        const ds4_session *session,
+        uint64_t           out[EXACTN_UNION_COUNTER_COUNT],
+        const char        *case_name) {
+    if (ds4_test_session_exactn_union_stats(session, out) != 0) {
+        fail("exact-N union stats", case_name, "hook failed");
+    }
+}
+
+static void check_union_stats_delta(
+    const uint64_t before[EXACTN_UNION_COUNTER_COUNT],
+    const uint64_t after[EXACTN_UNION_COUNTER_COUNT],
+    const exactn_case *tc,
+    bool expect_batch_head) {
+    uint64_t expected[EXACTN_UNION_COUNTER_COUNT] = {0};
+    /* EOS in the first draft row truncates the block to N=1 before exact-N
+     * dispatch.  Every other full block (including middle EOS) is committed
+     * by the union path; a deliberately wrong row restores once and exactly
+     * replays the already verified prefix. */
+    if (tc->eos_at != 0) {
+        expected[EXACTN_UNION_ATTEMPTS] = 1;
+        if (tc->reject_at >= 0) {
+            expected[EXACTN_UNION_FALLBACKS] = 1;
+            expected[EXACTN_UNION_PARTIAL_FALLBACKS] = 1;
+            expected[EXACTN_UNION_PARTIAL_REPLAYS] = 1;
+            expected[EXACTN_UNION_VERIFY_SKIPS] = 1;
+        } else {
+            expected[EXACTN_UNION_FULL_ACCEPTS] = 1;
+        }
+        if (expect_batch_head) {
+            expected[EXACTN_UNION_BATCH_HEAD_ATTEMPTS] = 1;
+            expected[EXACTN_UNION_BATCH_HEAD_USES] = 1;
+        }
+    }
+
+    static const char *const names[EXACTN_UNION_COUNTER_COUNT] = {
+        "attempt", "full", "fallback", "partial", "error",
+        "partial-replay", "verify-skip", "batch-head-attempt",
+        "batch-head-use", "batch-head-fallback"
+    };
+    for (int i = 0; i < EXACTN_UNION_COUNTER_COUNT; i++) {
+        if (after[i] < before[i] || after[i] - before[i] != expected[i]) {
+            char detail[192];
+            snprintf(detail, sizeof(detail),
+                     "%s before=%llu after=%llu expected_delta=%llu",
+                     names[i],
+                     (unsigned long long)before[i],
+                     (unsigned long long)after[i],
+                     (unsigned long long)expected[i]);
+            fail("exact-N union stats delta", tc->name, detail);
+        }
+    }
+}
+
+static void run_case(ds4_session *session,
+                     const ds4_session_snapshot *base,
+                     int base_pos,
+                     ds4_session_snapshot *expected_state,
+                     ds4_session_snapshot *actual_state,
+                     const int correct[MAX_DRAFT],
+                     const int wrong[MAX_DRAFT],
+                     int model_eos,
+                      int vocab,
+                      float *expected_logits,
+                      float *actual_logits,
+                      const exactn_case *tc,
+                      bool expect_batch_head) {
+    int drafts[MAX_DRAFT];
+    memcpy(drafts, correct, (size_t)tc->draft_n * sizeof(drafts[0]));
+    if (tc->reject_at >= 0) drafts[tc->reject_at] = wrong[tc->reject_at];
+
+    int cycle_eos = model_eos;
+    if (tc->eos_at >= 0) {
+        cycle_eos = drafts[tc->eos_at];
+    }
+
+    /* Run the production path first.  A partial union result already proves
+     * the complete correct prefix and skips the legacy batch verifier, so the
+     * commit must stop exactly at the deliberately wrong row. */
+    restore_snapshot(session, base, tc->name);
+    int accepted[MAX_DRAFT] = {-1, -1, -1, -1, -1};
+    char err[256] = "";
+    uint64_t union_before[EXACTN_UNION_COUNTER_COUNT];
+    uint64_t union_after[EXACTN_UNION_COUNTER_COUNT];
+    read_union_stats(session, union_before, tc->name);
+    const int accepted_n = ds4_test_session_eval_exact_drafts(
+            session, drafts, tc->draft_n, cycle_eos,
+            accepted, MAX_DRAFT, err, sizeof(err));
+    if (accepted_n < 0) fail("exact-N cycle", tc->name, err);
+    read_union_stats(session, union_after, tc->name);
+    check_union_stats_delta(union_before, union_after, tc,
+                            expect_batch_head);
+    int full_expected = tc->draft_n;
+    if (tc->eos_at >= 0) {
+        /* Production truncates at the first occurrence of EOS.  The fixture
+         * chooses a unique middle token below, but computing the first row
+         * here keeps the oracle correct if a future prompt changes. */
+        for (int i = 0; i < tc->draft_n; i++) {
+            if (drafts[i] == cycle_eos) {
+                full_expected = i + 1;
+                break;
+            }
+        }
+    }
+    if (tc->reject_at < 0 && accepted_n != full_expected) {
+        char detail[160];
+        snprintf(detail, sizeof(detail), "expected=%d actual=%d err=%s",
+                 full_expected, accepted_n, err);
+        fail("accepted prefix length", tc->name, detail);
+    }
+    if (tc->reject_at >= 0 && accepted_n != tc->reject_at) {
+        char detail[160];
+        snprintf(detail, sizeof(detail),
+                 "reject_at=%d accepted=%d err=%s",
+                 tc->reject_at, accepted_n, err);
+        fail("partial accepted wrong row", tc->name, detail);
+    }
+    for (int i = 0; i < accepted_n; i++) {
+        if (accepted[i] != correct[i]) {
+            char detail[128];
+            snprintf(detail, sizeof(detail),
+                     "row=%d expected=%d actual=%d",
+                     i, correct[i], accepted[i]);
+            fail("accepted token", tc->name, detail);
+        }
+    }
+    if (ds4_session_pos(session) != base_pos + accepted_n) {
+        fail("checkpoint length", tc->name, "unexpected committed position");
+    }
+
+    copy_logits(session, actual_logits, vocab, tc->name);
+    save_snapshot(session, actual_state, tc->name);
+    int actual_cont[CONTINUATION_TOKENS];
+    const int actual_cont_n = tc->eos_at >= 0 ? 0 :
+        greedy_continuation(session, model_eos, actual_cont, tc->name);
+
+    restore_snapshot(session, base, tc->name);
+    eval_tokens(session, correct, accepted_n, tc->name);
+    copy_logits(session, expected_logits, vocab, tc->name);
+    save_snapshot(session, expected_state, tc->name);
+    int expected_cont[CONTINUATION_TOKENS];
+    const int expected_cont_n = tc->eos_at >= 0 ? 0 :
+        greedy_continuation(session, model_eos, expected_cont, tc->name);
+
+    compare_logits(expected_logits, actual_logits, vocab, tc->name);
+    compare_snapshots(expected_state, actual_state, tc->name);
+    compare_continuations(expected_cont, expected_cont_n,
+                          actual_cont, actual_cont_n, tc->name);
+    fprintf(stderr,
+            "PASS: exact-N oracle case=%s drafted=%d committed=%d "
+            "snapshot_bytes=%llu continuation=%d\n",
+            tc->name, tc->draft_n, accepted_n,
+            (unsigned long long)actual_state->len, actual_cont_n);
+}
+
+int main(void) {
+#ifndef __APPLE__
+    fprintf(stderr, "test_metal_exactn_oracle: skipped (Metal requires macOS)\n");
+    return 0;
+#else
+    const char *model = getenv("DS4_TEST_MODEL");
+    if (!model || !model[0] || access(model, R_OK) != 0) {
+        const char *required = getenv("DS4_TEST_REQUIRE_MODEL");
+        fprintf(stderr,
+                "test_metal_exactn_oracle: %s "
+                "(set DS4_TEST_MODEL to a readable target GGUF)\n",
+                required && required[0] && strcmp(required, "0") != 0
+                    ? "FAIL: required model is missing"
+                    : "skipped");
+        return required && required[0] && strcmp(required, "0") != 0 ? 1 : 0;
+    }
+
+    const char *batch_head_env =
+        getenv("DS4_TEST_METAL_EXACTN_BATCH_HEAD");
+    const bool expect_batch_head =
+        batch_head_env && batch_head_env[0] &&
+        strcmp(batch_head_env, "0") != 0;
+
+    setenv("DS4_TEST_METAL_EXACTN_ORACLE", "1", 1);
+    setenv("DS4_METAL_DSPARK_EXACTN_UNION", "1", 1);
+    setenv("DS4_METAL_DSPARK_EXACTN", "1", 1);
+    if (expect_batch_head) {
+        setenv("DS4_METAL_DSPARK_EXACTN_BATCH_HEAD", "1", 1);
+        unsetenv("DS4_METAL_DISABLE_DSPARK_EXACTN_BATCH_HEAD");
+    } else {
+        unsetenv("DS4_METAL_DSPARK_EXACTN_BATCH_HEAD");
+    }
+    setenv("DS4_DSPARK_STATS", "1", 1);
+    setenv("DS4_DSPARK_SSD_VERIFY_BLOCK_MAX", "5", 1);
+    setenv("DS4_METAL_DSPARK_ACCEPTANCE_ONLY_VERIFY", "0", 1);
+    setenv("DS4_METAL_DSPARK_EXACT2", "0", 1);
+
+    ds4_engine_options opt;
+    memset(&opt, 0, sizeof(opt));
+    opt.model_path = model;
+    opt.backend = DS4_BACKEND_METAL;
+    opt.context_size = TEST_CTX;
+    opt.prefill_chunk = TEST_PREFILL_CHUNK;
+    opt.ssd_streaming = true;
+    opt.ssd_streaming_cold = true;
+    opt.ssd_streaming_cache_experts = TEST_EXPERT_CACHE;
+
+    ds4_engine *engine = NULL;
+    if (ds4_engine_open(&engine, &opt) != 0 || !engine) {
+        fail("engine open", NULL, model);
+    }
+
+    ds4_tokens prompt = {0};
+    ds4_encode_chat_prompt(
+            engine,
+            NULL,
+            "Continue this sequence concisely: 1, 2, 3, 4, 5,",
+            DS4_THINK_NONE,
+            &prompt);
+    if (prompt.len <= 0 || prompt.len >= TEST_CTX - 16) {
+        fail("prompt tokenization", NULL, "invalid prompt length");
+    }
+
+    ds4_session *session = NULL;
+    if (ds4_session_create(&session, engine, TEST_CTX) != 0 || !session) {
+        fail("session create", NULL, "failed");
+    }
+    char err[256] = "";
+    if (ds4_session_sync(session, &prompt, err, sizeof(err)) != 0) {
+        fail("session prefill", NULL, err);
+    }
+
+    ds4_session_snapshot base = {0};
+    ds4_session_snapshot expected_state = {0};
+    ds4_session_snapshot actual_state = {0};
+    save_snapshot(session, &base, "setup");
+
+    const int vocab = ds4_engine_vocab_size(engine);
+    const int model_eos = ds4_token_eos(engine);
+    float *probe_logits = malloc((size_t)vocab * sizeof(*probe_logits));
+    float *expected_logits = malloc((size_t)vocab * sizeof(*expected_logits));
+    float *actual_logits = malloc((size_t)vocab * sizeof(*actual_logits));
+    if (!probe_logits || !expected_logits || !actual_logits) {
+        fail("host allocation", NULL, "logits");
+    }
+
+    int correct[MAX_DRAFT];
+    int wrong[MAX_DRAFT];
+    for (int i = 0; i < MAX_DRAFT; i++) {
+        copy_logits(session, probe_logits, vocab, "draft derivation");
+        correct[i] = ds4_session_argmax(session);
+        if (correct[i] < 0 || correct[i] == model_eos) {
+            fail("draft derivation", NULL, "unexpected early EOS");
+        }
+        wrong[i] = lowest_finite_token(probe_logits, vocab,
+                                       correct[i], model_eos);
+        if (wrong[i] < 0) fail("draft derivation", NULL, "no rejection token");
+        eval_tokens(session, &correct[i], 1, "draft derivation");
+    }
+    restore_snapshot(session, &base, "setup");
+
+    int eos_middle_at = -1;
+    for (int candidate = 2; candidate < MAX_DRAFT && eos_middle_at < 0;
+         candidate++) {
+        bool unique = true;
+        for (int previous = 0; previous < candidate; previous++) {
+            if (correct[previous] == correct[candidate]) {
+                unique = false;
+                break;
+            }
+        }
+        if (unique) eos_middle_at = candidate;
+    }
+    for (int candidate = 1; candidate < 2 && eos_middle_at < 0; candidate++) {
+        if (correct[candidate] != correct[0]) eos_middle_at = candidate;
+    }
+    if (eos_middle_at < 1) {
+        fail("EOS fixture derivation", NULL,
+             "no target token unique after the first draft row");
+    }
+
+    const exactn_case cases[] = {
+        {"full-2",       2, -1, -1},
+        {"full-3",       3, -1, -1},
+        {"full-4",       4, -1, -1},
+        /* Five drafts plus the already-generated target token exercise the
+         * requested six-token speculative cycle. */
+        {"full-5",       5, -1, -1},
+        {"partial-5-at1", 5,  1, -1},
+        {"partial-5-at2", 5,  2, -1},
+        {"partial-5-at3", 5,  3, -1},
+        {"partial-5-at4", 5,  4, -1},
+        {"eos-first",     5, -1,  0},
+        {"eos-middle",    5, -1, eos_middle_at},
+    };
+
+    for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) {
+        run_case(session, &base, prompt.len, &expected_state, &actual_state,
+                 correct, wrong, model_eos, vocab,
+                 expected_logits, actual_logits, &cases[i],
+                 expect_batch_head);
+    }
+
+    fprintf(stderr,
+            "test_metal_exactn_oracle PASS cases=%zu N=2..5 "
+            "(six-token cycle at N=5) partial_prefixes=1..4 "
+            "eos=first,middle batch_head=%s\n",
+            sizeof(cases) / sizeof(cases[0]),
+            expect_batch_head ? "required" : "disabled");
+
+    free(actual_logits);
+    free(expected_logits);
+    free(probe_logits);
+    ds4_session_snapshot_free(&actual_state);
+    ds4_session_snapshot_free(&expected_state);
+    ds4_session_snapshot_free(&base);
+    ds4_session_free(session);
+    ds4_tokens_free(&prompt);
+    ds4_engine_close(engine);
+    return 0;
+#endif
+}
diff --git a/tests/test_metal_indexer_q4.c b/tests/test_metal_indexer_q4.c
new file mode 100644
index 0000000000..b45dc4de13
--- /dev/null
+++ b/tests/test_metal_indexer_q4.c
@@ -0,0 +1,436 @@
+#define _DARWIN_C_SOURCE
+
+/* GGUF-free production-shape oracle for the DeepSeek Flash Q4_K indexer
+ * query projection.  The fixture deliberately uses the real 1024 -> 8192
+ * geometry and checks the dispatch boundaries around the Metal matvec,
+ * generic matmul, and 32-token matrix paths. */
+
+#include "ds4_gpu.h"
+
+#include <math.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+bool ds4_log_is_tty(FILE *fp) {
+    (void)fp;
+    return false;
+}
+
+#ifdef __APPLE__
+
+enum {
+    Q4_K_TYPE = 12u,
+    QK_K = 256u,
+    INDEXER_IN_DIM = 1024u,
+    INDEXER_OUT_DIM = 8192u,
+    BLOCKS_PER_ROW = INDEXER_IN_DIM / QK_K,
+    GROUPS_PER_BLOCK = 8u,
+    GROUP_SIZE = 32u,
+    Q_PATTERNS = 16u,
+    MAX_TOKENS = 33u,
+    GUARD_FLOATS = 64u,
+};
+
+typedef struct {
+    uint16_t d;
+    uint16_t dmin;
+    uint8_t scales[12];
+    uint8_t qs[QK_K / 2u];
+} block_q4_K;
+
+static const float k_abs_tolerance = 2.0e-3f;
+static const float k_rel_tolerance = 3.0e-5f;
+static const uint32_t k_poison_base = 0x7fc10000u;
+
+static void fail(const char *what) {
+    fprintf(stderr, "Metal indexer Q4_K oracle FAIL: %s\n", what);
+    exit(1);
+}
+
+#define CHECK(expr, what) do { if (!(expr)) fail(what); } while (0)
+
+static uint64_t align_up(uint64_t value, uint64_t alignment) {
+    return (value + alignment - 1u) / alignment * alignment;
+}
+
+static float f16_to_f32(uint16_t h) {
+    const uint32_t sign = (uint32_t)(h & 0x8000u) << 16u;
+    uint32_t exp = (h >> 10u) & 0x1fu;
+    uint32_t mant = h & 0x03ffu;
+    uint32_t bits;
+
+    if (exp == 0u) {
+        if (mant == 0u) {
+            bits = sign;
+        } else {
+            exp = 1u;
+            while ((mant & 0x0400u) == 0u) {
+                mant <<= 1u;
+                exp--;
+            }
+            mant &= 0x03ffu;
+            bits = sign | ((exp + 127u - 15u) << 23u) | (mant << 13u);
+        }
+    } else if (exp == 31u) {
+        bits = sign | 0x7f800000u | (mant << 13u);
+    } else {
+        bits = sign | ((exp + 127u - 15u) << 23u) | (mant << 13u);
+    }
+
+    float value;
+    memcpy(&value, &bits, sizeof(value));
+    return value;
+}
+
+static void q4_pack_scales(uint8_t packed[12],
+                           const uint8_t scales[GROUPS_PER_BLOCK],
+                           const uint8_t minima[GROUPS_PER_BLOCK]) {
+    memset(packed, 0, 12u);
+    for (uint32_t group = 0; group < 4u; group++) {
+        packed[group] = scales[group] & 63u;
+        packed[group + 4u] = minima[group] & 63u;
+    }
+    for (uint32_t group = 4u; group < GROUPS_PER_BLOCK; group++) {
+        packed[group + 4u] = (scales[group] & 15u) |
+                             ((minima[group] & 15u) << 4u);
+        packed[group - 4u] |= (scales[group] >> 4u) << 6u;
+        packed[group] |= (minima[group] >> 4u) << 6u;
+    }
+}
+
+static void q4_scale_min(const uint8_t packed[12], uint32_t group,
+                         uint8_t *scale, uint8_t *minimum) {
+    if (group < 4u) {
+        *scale = packed[group] & 63u;
+        *minimum = packed[group + 4u] & 63u;
+    } else {
+        *scale = (packed[group + 4u] & 15u) |
+                 ((packed[group - 4u] >> 6u) << 4u);
+        *minimum = (packed[group + 4u] >> 4u) |
+                   ((packed[group] >> 6u) << 4u);
+    }
+}
+
+static uint32_t q4_value(const block_q4_K *block,
+                         uint32_t group, uint32_t lane) {
+    const uint32_t byte_offset = (group >> 1u) * GROUP_SIZE + lane;
+    const uint32_t shift = (group & 1u) * 4u;
+    return (block->qs[byte_offset] >> shift) & 15u;
+}
+
+static void fill_q4_indexer(block_q4_K *matrix) {
+    CHECK(sizeof(block_q4_K) == 144u, "unexpected Q4_K block size");
+
+    for (uint32_t row = 0; row < INDEXER_OUT_DIM; row++) {
+        const uint32_t pattern = row & (Q_PATTERNS - 1u);
+        for (uint32_t block = 0; block < BLOCKS_PER_ROW; block++) {
+            block_q4_K *b = matrix +
+                (uint64_t)row * BLOCKS_PER_ROW + block;
+            uint8_t scales[GROUPS_PER_BLOCK];
+            uint8_t minima[GROUPS_PER_BLOCK];
+
+            for (uint32_t group = 0; group < GROUPS_PER_BLOCK; group++) {
+                scales[group] = (uint8_t)(1u +
+                    (row * 13u + block * 7u + group * 5u + 3u) % 31u);
+                minima[group] = (uint8_t)(
+                    (row * 11u + block * 3u + group * 7u + 1u) % 17u);
+            }
+            q4_pack_scales(b->scales, scales, minima);
+            memset(b->qs, 0, sizeof(b->qs));
+
+            for (uint32_t group = 0; group < GROUPS_PER_BLOCK; group++) {
+                for (uint32_t lane = 0; lane < GROUP_SIZE; lane++) {
+                    const uint32_t q =
+                        (lane * 5u + pattern * 3u + block * 7u +
+                         group * 11u) & 15u;
+                    const uint32_t byte_offset =
+                        (group >> 1u) * GROUP_SIZE + lane;
+                    const uint32_t shift = (group & 1u) * 4u;
+                    b->qs[byte_offset] |= (uint8_t)(q << shift);
+                }
+            }
+
+            /* Exact binary scales: d=2^-8 and dmin=2^-9. */
+            b->d = 0x1c00u;
+            b->dmin = 0x1800u;
+        }
+    }
+}
+
+static void fill_inputs(float *inputs) {
+    for (uint32_t token = 0; token < MAX_TOKENS; token++) {
+        for (uint32_t col = 0; col < INDEXER_IN_DIM; col++) {
+            const uint32_t key = token * 131u + col * 17u +
+                                 ((col >> 3u) ^ (token * 29u));
+            /* Multiples of 2^-6 are exactly representable by the half RHS
+             * used in the Metal matrix kernels. */
+            inputs[(uint64_t)token * INDEXER_IN_DIM + col] =
+                (float)((int)(key % 129u) - 64) / 64.0f;
+        }
+    }
+}
+
+static void build_cpu_dequant_oracle(const block_q4_K *matrix,
+                                     const float *inputs,
+                                     float *reference) {
+    float group_sum[MAX_TOKENS][BLOCKS_PER_ROW][GROUPS_PER_BLOCK];
+    float q_dot[MAX_TOKENS][Q_PATTERNS]
+               [BLOCKS_PER_ROW][GROUPS_PER_BLOCK];
+
+    memset(group_sum, 0, sizeof(group_sum));
+    memset(q_dot, 0, sizeof(q_dot));
+
+    /* q nibbles repeat every Q_PATTERNS rows, while scales/minima remain
+     * row-specific.  Factoring these sums preserves the exact dequantized
+     * dot-product algebra and keeps the production-size oracle inexpensive. */
+    for (uint32_t token = 0; token < MAX_TOKENS; token++) {
+        const float *x = inputs + (uint64_t)token * INDEXER_IN_DIM;
+        for (uint32_t block = 0; block < BLOCKS_PER_ROW; block++) {
+            for (uint32_t group = 0; group < GROUPS_PER_BLOCK; group++) {
+                const uint32_t col0 = block * QK_K + group * GROUP_SIZE;
+                float x_sum = 0.0f;
+                for (uint32_t lane = 0; lane < GROUP_SIZE; lane++) {
+                    x_sum += x[col0 + lane];
+                }
+                group_sum[token][block][group] = x_sum;
+
+                for (uint32_t pattern = 0; pattern < Q_PATTERNS; pattern++) {
+                    const block_q4_K *b = matrix +
+                        (uint64_t)pattern * BLOCKS_PER_ROW + block;
+                    float sum = 0.0f;
+                    for (uint32_t lane = 0; lane < GROUP_SIZE; lane++) {
+                        sum += (float)q4_value(b, group, lane) *
+                               x[col0 + lane];
+                    }
+                    q_dot[token][pattern][block][group] = sum;
+                }
+            }
+        }
+    }
+
+    for (uint32_t token = 0; token < MAX_TOKENS; token++) {
+        for (uint32_t row = 0; row < INDEXER_OUT_DIM; row++) {
+            const uint32_t pattern = row & (Q_PATTERNS - 1u);
+            const block_q4_K *row_blocks = matrix +
+                (uint64_t)row * BLOCKS_PER_ROW;
+            float acc = 0.0f;
+
+            for (uint32_t block = 0; block < BLOCKS_PER_ROW; block++) {
+                const block_q4_K *b = row_blocks + block;
+                const float d = f16_to_f32(b->d);
+                const float dmin = f16_to_f32(b->dmin);
+                for (uint32_t group = 0; group < GROUPS_PER_BLOCK; group++) {
+                    uint8_t scale = 0;
+                    uint8_t minimum = 0;
+                    q4_scale_min(b->scales, group, &scale, &minimum);
+                    acc += d * (float)scale *
+                               q_dot[token][pattern][block][group] -
+                           dmin * (float)minimum *
+                               group_sum[token][block][group];
+                }
+            }
+            reference[(uint64_t)token * INDEXER_OUT_DIM + row] = acc;
+        }
+    }
+}
+
+static uint32_t poison_bits(uint64_t index) {
+    return k_poison_base + (uint32_t)(index & 0xffffu);
+}
+
+static void poison(float *values, uint64_t count) {
+    for (uint64_t i = 0; i < count; i++) {
+        const uint32_t bits = poison_bits(i);
+        memcpy(&values[i], &bits, sizeof(bits));
+    }
+}
+
+static uint64_t count_canary_failures(const float *values,
+                                      uint64_t begin, uint64_t end,
+                                      uint64_t *first) {
+    uint64_t failures = 0;
+    *first = UINT64_MAX;
+    for (uint64_t i = begin; i < end; i++) {
+        uint32_t actual = 0;
+        memcpy(&actual, &values[i], sizeof(actual));
+        if (actual != poison_bits(i)) {
+            if (*first == UINT64_MAX) *first = i;
+            failures++;
+        }
+    }
+    return failures;
+}
+
+static bool compare_case(const float *actual, const float *reference,
+                         uint32_t n_tokens) {
+    const uint64_t count = (uint64_t)n_tokens * INDEXER_OUT_DIM;
+    uint64_t raw_mismatches = 0;
+    uint64_t tolerance_failures = 0;
+    uint64_t worst = 0;
+    float max_abs = 0.0f;
+    float max_rel = 0.0f;
+
+    for (uint64_t i = 0; i < count; i++) {
+        if (memcmp(&actual[i], &reference[i], sizeof(float)) != 0) {
+            raw_mismatches++;
+        }
+        const float diff = fabsf(actual[i] - reference[i]);
+        const float rel = diff / fmaxf(1.0f, fabsf(reference[i]));
+        const float limit = k_abs_tolerance +
+                            k_rel_tolerance * fabsf(reference[i]);
+        if (diff > max_abs) {
+            max_abs = diff;
+            worst = i;
+        }
+        if (rel > max_rel) max_rel = rel;
+        if (!isfinite(actual[i]) || diff > limit) tolerance_failures++;
+    }
+
+    fprintf(stderr,
+            "Metal indexer Q4_K n_tok=%u: raw=%llu/%llu "
+            "max_abs=%g max_rel=%g tol(abs=%g rel=%g) %s\n",
+            n_tokens,
+            (unsigned long long)raw_mismatches,
+            (unsigned long long)count,
+            max_abs, max_rel, k_abs_tolerance, k_rel_tolerance,
+            tolerance_failures == 0 ? "PASS" : "FAIL");
+    if (tolerance_failures != 0) {
+        fprintf(stderr,
+                "  worst token=%llu row=%llu gpu=%g cpu=%g delta=%g "
+                "failures=%llu\n",
+                (unsigned long long)(worst / INDEXER_OUT_DIM),
+                (unsigned long long)(worst % INDEXER_OUT_DIM),
+                actual[worst], reference[worst],
+                actual[worst] - reference[worst],
+                (unsigned long long)tolerance_failures);
+    }
+    return tolerance_failures == 0;
+}
+
+int main(void) {
+    static const uint32_t token_cases[] = {
+        1u, 8u, 9u, 16u, 17u, 31u, 32u, 33u,
+    };
+    const uint64_t row_bytes =
+        (INDEXER_IN_DIM / QK_K) * sizeof(block_q4_K);
+    const uint64_t weight_bytes = (uint64_t)INDEXER_OUT_DIM * row_bytes;
+    const uint64_t page = (uint64_t)getpagesize();
+    const uint64_t model_bytes = align_up(weight_bytes, page);
+    const uint64_t input_count = (uint64_t)MAX_TOKENS * INDEXER_IN_DIM;
+    const uint64_t max_output_count =
+        (uint64_t)MAX_TOKENS * INDEXER_OUT_DIM;
+    const uint64_t storage_count =
+        GUARD_FLOATS + max_output_count + GUARD_FLOATS;
+
+    CHECK(Q4_K_TYPE == 12u, "Q4_K GGUF type must be 12");
+    CHECK(row_bytes == 576u, "unexpected production Q4_K row size");
+    CHECK(weight_bytes == 4718592u,
+          "unexpected production Q4_K indexer size");
+    CHECK(unsetenv("DS4_METAL_DISABLE_Q4_MV_CLASSIC") == 0,
+          "clear classic Q4_K kill switch");
+    CHECK(ds4_gpu_init() != 0, "Metal init");
+
+    void *model = NULL;
+    CHECK(posix_memalign(&model, (size_t)page, (size_t)model_bytes) == 0,
+          "page-aligned model allocation");
+    memset(model, 0, (size_t)model_bytes);
+    fill_q4_indexer(model);
+
+    float *inputs = malloc((size_t)input_count * sizeof(float));
+    float *reference = malloc((size_t)max_output_count * sizeof(float));
+    float *storage = malloc((size_t)storage_count * sizeof(float));
+    CHECK(inputs && reference && storage, "host tensor allocation");
+    fill_inputs(inputs);
+    build_cpu_dequant_oracle(model, inputs, reference);
+
+    ds4_gpu_tensor *x = ds4_gpu_tensor_alloc(input_count * sizeof(float));
+    ds4_gpu_tensor *out_base =
+        ds4_gpu_tensor_alloc(storage_count * sizeof(float));
+    CHECK(x && out_base, "Metal tensor allocation");
+    CHECK(ds4_gpu_tensor_write(x, 0, inputs,
+                               input_count * sizeof(float)) != 0,
+          "input upload");
+    CHECK(ds4_gpu_set_model_map(model, model_bytes) != 0, "model map");
+    ds4_gpu_set_quality(false);
+    ds4_gpu_set_ssd_streaming(false);
+
+    bool passed = true;
+    for (uint32_t case_i = 0;
+         case_i < sizeof(token_cases) / sizeof(token_cases[0]);
+         case_i++) {
+        const uint32_t n_tokens = token_cases[case_i];
+        const uint64_t output_count =
+            (uint64_t)n_tokens * INDEXER_OUT_DIM;
+        const uint64_t output_bytes = output_count * sizeof(float);
+
+        poison(storage, storage_count);
+        CHECK(ds4_gpu_tensor_write(out_base, 0, storage,
+                                   storage_count * sizeof(float)) != 0,
+              "output poison upload");
+        ds4_gpu_tensor *out = ds4_gpu_tensor_view(
+            out_base, GUARD_FLOATS * sizeof(float), output_bytes);
+        CHECK(out != NULL, "exact-size output view");
+
+        /* This is the production entry point and Q4_K's GGUF type is 12. */
+        CHECK(ds4_gpu_matmul_quant_tensor(
+                  out, model, model_bytes, 0, Q4_K_TYPE,
+                  INDEXER_IN_DIM, INDEXER_OUT_DIM, x, n_tokens) != 0,
+              "ds4_gpu_matmul_quant_tensor(type=12)");
+        ds4_gpu_tensor_free(out);
+
+        CHECK(ds4_gpu_tensor_read(out_base, 0, storage,
+                                  storage_count * sizeof(float)) != 0,
+              "output readback");
+
+        const bool values_ok = compare_case(
+            storage + GUARD_FLOATS, reference, n_tokens);
+        uint64_t first_prefix = UINT64_MAX;
+        uint64_t first_suffix = UINT64_MAX;
+        const uint64_t prefix_failures = count_canary_failures(
+            storage, 0, GUARD_FLOATS, &first_prefix);
+        const uint64_t suffix_begin = GUARD_FLOATS + output_count;
+        const uint64_t suffix_failures = count_canary_failures(
+            storage, suffix_begin, storage_count, &first_suffix);
+        const bool canary_ok = prefix_failures == 0 && suffix_failures == 0;
+
+        fprintf(stderr,
+                "Metal indexer Q4_K n_tok=%u canary: prefix=%llu "
+                "suffix=%llu %s\n",
+                n_tokens,
+                (unsigned long long)prefix_failures,
+                (unsigned long long)suffix_failures,
+                canary_ok ? "PASS" : "FAIL");
+        if (!canary_ok) {
+            fprintf(stderr, "  first prefix=%llu first suffix=%llu\n",
+                    (unsigned long long)first_prefix,
+                    (unsigned long long)first_suffix);
+        }
+        if (!values_ok || !canary_ok) passed = false;
+    }
+
+    ds4_gpu_tensor_free(out_base);
+    ds4_gpu_tensor_free(x);
+    ds4_gpu_cleanup();
+    free(storage);
+    free(reference);
+    free(inputs);
+    free(model);
+
+    fprintf(stderr,
+            "Metal indexer Q4_K production geometry 1024x8192: %s\n",
+            passed ? "PASS" : "FAIL");
+    return passed ? 0 : 1;
+}
+
+#else
+
+int main(void) {
+    fprintf(stderr, "Metal indexer Q4_K oracle SKIP: requires macOS\n");
+    return 0;
+}
+
+#endif
diff --git a/tests/test_metal_iq2_live_index b/tests/test_metal_iq2_live_index
new file mode 100755
index 0000000000..6306bbeaa8
Binary files /dev/null and b/tests/test_metal_iq2_live_index differ
diff --git a/tests/test_metal_iq2_live_index.c b/tests/test_metal_iq2_live_index.c
new file mode 100644
index 0000000000..2098a71a7c
--- /dev/null
+++ b/tests/test_metal_iq2_live_index.c
@@ -0,0 +1,240 @@
+#define _DARWIN_C_SOURCE
+
+#include "ds4_gpu.h"
+
+#include <fcntl.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/mman.h>
+#include <unistd.h>
+
+#ifdef __APPLE__
+
+enum {
+    N_TOTAL_EXPERT = 6,
+    N_SELECTED = 6,
+    CACHE_BUDGET = 3,
+};
+
+static const uint64_t GATE_EXPERT_BYTES = UINT64_C(2162688);
+static const uint64_t DOWN_EXPERT_BYTES = UINT64_C(2752512);
+static const uint64_t TOTAL_EXPERT_BYTES = UINT64_C(7077888);
+
+bool ds4_log_is_tty(FILE *fp) {
+    (void)fp;
+    return false;
+}
+
+static int check_policy(void) {
+    int ok = 1;
+#define CHECK_POLICY(expected, ssd, gate, down, enable, disable) do {       \
+    const int got = ds4_gpu_test_stream_expert_live_index_policy(          \
+            (ssd), (gate), (down), (enable), (disable));                   \
+    if (got != (expected)) {                                                \
+        fprintf(stderr,                                                     \
+                "policy FAIL ssd=%d gate=%llu down=%llu enable=%d "        \
+                "disable=%d got=%d expected=%d\n",                         \
+                (ssd), (unsigned long long)(gate),                          \
+                (unsigned long long)(down), (enable), (disable),            \
+                got, (expected));                                           \
+        ok = 0;                                                             \
+    }                                                                       \
+} while (0)
+
+    CHECK_POLICY(1, 1, GATE_EXPERT_BYTES, DOWN_EXPERT_BYTES, -1, -1);
+    CHECK_POLICY(1, 1, GATE_EXPERT_BYTES, DOWN_EXPERT_BYTES, 1, 0);
+    CHECK_POLICY(0, 0, GATE_EXPERT_BYTES, DOWN_EXPERT_BYTES, 1, 0);
+    CHECK_POLICY(0, 1, GATE_EXPERT_BYTES - 1u,
+                 DOWN_EXPERT_BYTES + 2u, 1, 0);
+    CHECK_POLICY(0, 1, GATE_EXPERT_BYTES, DOWN_EXPERT_BYTES, 0, 0);
+    CHECK_POLICY(0, 1, GATE_EXPERT_BYTES, DOWN_EXPERT_BYTES, 1, 1);
+#undef CHECK_POLICY
+    return ok;
+}
+
+static int seed_route(const ds4_gpu_stream_expert_table *table,
+                      const int32_t route[N_SELECTED]) {
+    return ds4_gpu_stream_expert_cache_seed_selected(table,
+                                                      route,
+                                                      N_SELECTED);
+}
+
+static int run_churn(const ds4_gpu_stream_expert_table *table) {
+    static const int32_t route_a[N_SELECTED] = {0, 1, 2, 0, 1, 2};
+    static const int32_t route_b[N_SELECTED] = {3, 4, 5, 3, 4, 5};
+    return seed_route(table, route_a) &&
+           seed_route(table, route_b) &&
+           seed_route(table, route_a);
+}
+
+static void print_report(
+        const char *name,
+        const ds4_gpu_stream_expert_live_index_report *r) {
+    fprintf(stderr,
+            "%s scans=%llu entries=%llu fallbacks=%llu inserts=%llu "
+            "removes=%llu reuse_calls=%llu reuse_entries=%llu "
+            "live=%u cache=%u eligible=%u active=%u broken=%u hash=%016llx\n",
+            name,
+            (unsigned long long)r->scans,
+            (unsigned long long)r->entries,
+            (unsigned long long)r->fallbacks,
+            (unsigned long long)r->inserts,
+            (unsigned long long)r->removes,
+            (unsigned long long)r->reuse_scan_calls,
+            (unsigned long long)r->reuse_scan_entries,
+            r->live_count,
+            r->cache_entries,
+            r->eligible,
+            r->active,
+            r->broken,
+            (unsigned long long)r->resident_hash);
+}
+
+int main(void) {
+    int ok = check_policy();
+    int fd = -1;
+    void *model = MAP_FAILED;
+    char path[] = "/tmp/ds4-metal-iq2-live-index.XXXXXX";
+
+    if (2u * GATE_EXPERT_BYTES + DOWN_EXPERT_BYTES !=
+        TOTAL_EXPERT_BYTES) {
+        fprintf(stderr, "production size constants FAIL\n");
+        return 1;
+    }
+
+    const uint64_t gate_tensor_bytes =
+        (uint64_t)N_TOTAL_EXPERT * GATE_EXPERT_BYTES;
+    const uint64_t down_tensor_bytes =
+        (uint64_t)N_TOTAL_EXPERT * DOWN_EXPERT_BYTES;
+    const uint64_t gate_offset = 0;
+    const uint64_t up_offset = gate_tensor_bytes;
+    const uint64_t down_offset = up_offset + gate_tensor_bytes;
+    const uint64_t model_size = down_offset + down_tensor_bytes;
+
+    fd = mkstemp(path);
+    if (fd < 0 || ftruncate(fd, (off_t)model_size) != 0) {
+        perror("IQ2 live-index fixture");
+        ok = 0;
+        goto cleanup;
+    }
+    model = mmap(NULL, (size_t)model_size,
+                 PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
+    if (model == MAP_FAILED) {
+        perror("IQ2 live-index mmap");
+        ok = 0;
+        goto cleanup;
+    }
+
+    if (!ds4_gpu_init() || !ds4_gpu_set_model_map(model, model_size)) {
+        fprintf(stderr, "Metal initialization/model map FAIL\n");
+        ok = 0;
+        goto cleanup;
+    }
+    ds4_gpu_set_ssd_streaming(true);
+    ds4_gpu_set_model_fd(fd);
+    ds4_gpu_set_streaming_expert_cache_expert_bytes(TOTAL_EXPERT_BYTES);
+    setenv("DS4_METAL_STREAMING_EXPERT_TIMING_SUMMARY", "1", 1);
+
+    const ds4_gpu_stream_expert_table table = {
+        .model_map = model,
+        .model_size = model_size,
+        .layer = 0,
+        .n_total_expert = N_TOTAL_EXPERT,
+        .gate_offset = gate_offset,
+        .up_offset = up_offset,
+        .down_offset = down_offset,
+        .gate_expert_bytes = GATE_EXPERT_BYTES,
+        .down_expert_bytes = DOWN_EXPERT_BYTES,
+    };
+
+    setenv("DS4_METAL_DISABLE_STREAMING_EXPERT_LIVE_INDEX", "1", 1);
+    unsetenv("DS4_METAL_ENABLE_STREAMING_EXPERT_LIVE_INDEX");
+    ds4_gpu_set_streaming_expert_cache_budget(CACHE_BUDGET);
+    ok = ok && run_churn(&table);
+    ds4_gpu_stream_expert_live_index_report control = {0};
+    ds4_gpu_test_stream_expert_live_index_report(&control);
+    print_report("control", &control);
+    if (control.scans != 0 || control.entries != 0 ||
+        control.cache_entries != CACHE_BUDGET || control.broken != 0 ||
+        control.reuse_scan_calls == 0 ||
+        control.reuse_scan_entries < 30720u) {
+        fprintf(stderr, "control coverage FAIL\n");
+        ok = 0;
+    }
+
+    unsetenv("DS4_METAL_DISABLE_STREAMING_EXPERT_LIVE_INDEX");
+    unsetenv("DS4_METAL_ENABLE_STREAMING_EXPERT_LIVE_INDEX");
+    ds4_gpu_set_streaming_expert_cache_budget(CACHE_BUDGET);
+    ok = ok && run_churn(&table);
+    ds4_gpu_stream_expert_live_index_report candidate = {0};
+    ds4_gpu_test_stream_expert_live_index_report(&candidate);
+    print_report("candidate", &candidate);
+    if (candidate.scans == 0 || candidate.entries == 0 ||
+        candidate.fallbacks != 0 || candidate.inserts < CACHE_BUDGET ||
+        candidate.removes == 0 ||
+        candidate.live_count != candidate.cache_entries ||
+        candidate.cache_entries != CACHE_BUDGET ||
+        candidate.eligible == 0 || candidate.active == 0 ||
+        candidate.broken != 0 ||
+        candidate.reuse_scan_calls == 0 ||
+        candidate.reuse_scan_entries >= control.reuse_scan_entries ||
+        candidate.entries > candidate.scans * CACHE_BUDGET ||
+        candidate.resident_hash != control.resident_hash) {
+        fprintf(stderr, "candidate coverage/state FAIL\n");
+        ok = 0;
+    }
+
+    ds4_gpu_set_streaming_expert_cache_budget(CACHE_BUDGET);
+    static const int32_t route_a[N_SELECTED] = {0, 1, 2, 0, 1, 2};
+    static const int32_t route_b[N_SELECTED] = {3, 4, 5, 3, 4, 5};
+    ok = ok && seed_route(&table, route_a);
+    ds4_gpu_test_set_flags(DS4_GPU_TEST_STREAMING_LIVE_INDEX_FAILURE);
+    ok = ok && seed_route(&table, route_b);
+    ds4_gpu_test_set_flags(0);
+    ds4_gpu_stream_expert_live_index_report fault = {0};
+    ds4_gpu_test_stream_expert_live_index_report(&fault);
+    print_report("fault-fallback", &fault);
+    if (fault.fallbacks == 0 || fault.broken == 0 ||
+        fault.active != 0 || fault.cache_entries != CACHE_BUDGET) {
+        fprintf(stderr, "fault fallback FAIL\n");
+        ok = 0;
+    }
+
+    ds4_gpu_set_streaming_expert_cache_budget(CACHE_BUDGET);
+    ds4_gpu_stream_expert_live_index_report reset = {0};
+    ds4_gpu_test_stream_expert_live_index_report(&reset);
+    print_report("post-reset", &reset);
+    if (reset.broken != 0 || reset.live_count != 0 ||
+        reset.cache_entries != 0 || reset.fallbacks != 0) {
+        fprintf(stderr, "reset lifecycle FAIL\n");
+        ok = 0;
+    }
+
+cleanup:
+    ds4_gpu_test_set_flags(0);
+    unsetenv("DS4_METAL_ENABLE_STREAMING_EXPERT_LIVE_INDEX");
+    unsetenv("DS4_METAL_DISABLE_STREAMING_EXPERT_LIVE_INDEX");
+    unsetenv("DS4_METAL_STREAMING_EXPERT_TIMING_SUMMARY");
+    ds4_gpu_set_streaming_expert_cache_budget(0);
+    ds4_gpu_set_model_fd(-1);
+    ds4_gpu_set_ssd_streaming(false);
+    ds4_gpu_cleanup();
+    if (model != MAP_FAILED) munmap(model, (size_t)model_size);
+    if (fd >= 0) close(fd);
+    unlink(path);
+
+    fprintf(stderr, "IQ2 Metal streaming expert live-index %s\n",
+            ok ? "PASS" : "FAIL");
+    return ok ? 0 : 1;
+}
+
+#else
+
+int main(void) {
+    fprintf(stderr, "test_metal_iq2_live_index: SKIP (requires Apple Metal)\n");
+    return 0;
+}
+
+#endif
diff --git a/tests/test_metal_iq2_midonly b/tests/test_metal_iq2_midonly
new file mode 100755
index 0000000000..a1277392fc
Binary files /dev/null and b/tests/test_metal_iq2_midonly differ
diff --git a/tests/test_metal_iq2_midonly.c b/tests/test_metal_iq2_midonly.c
new file mode 100644
index 0000000000..6ff74c4a4a
--- /dev/null
+++ b/tests/test_metal_iq2_midonly.c
@@ -0,0 +1,76 @@
+#include "ds4_gpu.h"
+
+#include <inttypes.h>
+#include <stdio.h>
+
+#ifdef __APPLE__
+
+/* ds4_metal.m references the CLI logger hook; the standalone tensor oracle
+ * does not need terminal detection. */
+bool ds4_log_is_tty(FILE *fp) {
+    (void)fp;
+    return false;
+}
+
+int main(void) {
+    ds4_gpu_iq2_mid_only_oracle_report report;
+    if (!ds4_gpu_test_iq2_addr_mid_only_oracle(&report)) {
+        fprintf(stderr, "test_metal_iq2_midonly: setup/execution failed\n");
+        ds4_gpu_cleanup();
+        return 1;
+    }
+
+    const int pass =
+        report.mid_words == 6u * 2048u &&
+        report.mid_mismatches == 0 &&
+        report.canonical_gate_unwritten == 0 &&
+        report.canonical_up_unwritten == 0 &&
+        report.candidate_gate_writes == 0 &&
+        report.candidate_up_writes == 0 &&
+        report.masked_mid_mismatches == 0 &&
+        report.masked_inactive_writes == 0 &&
+        report.masked_canonical_gate_unwritten == 0 &&
+        report.masked_canonical_up_unwritten == 0 &&
+        report.masked_gate_writes == 0 &&
+        report.masked_up_writes == 0 &&
+        report.guard_byte_mismatches == 0;
+    fprintf(stderr,
+            "test_metal_iq2_midonly: %s mid_words=%" PRIu64
+            " mid_mismatches=%" PRIu64
+            " canonical_gate_unwritten=%" PRIu64
+            " canonical_up_unwritten=%" PRIu64
+            " candidate_gate_writes=%" PRIu64
+            " candidate_up_writes=%" PRIu64
+            " masked_mid_mismatches=%" PRIu64
+            " masked_inactive_writes=%" PRIu64
+            " masked_canonical_gate_unwritten=%" PRIu64
+            " masked_canonical_up_unwritten=%" PRIu64
+            " masked_gate_writes=%" PRIu64
+            " masked_up_writes=%" PRIu64
+            " guard_byte_mismatches=%" PRIu64 "\n",
+            pass ? "PASS" : "FAIL",
+            report.mid_words,
+            report.mid_mismatches,
+            report.canonical_gate_unwritten,
+            report.canonical_up_unwritten,
+            report.candidate_gate_writes,
+            report.candidate_up_writes,
+            report.masked_mid_mismatches,
+            report.masked_inactive_writes,
+            report.masked_canonical_gate_unwritten,
+            report.masked_canonical_up_unwritten,
+            report.masked_gate_writes,
+            report.masked_up_writes,
+            report.guard_byte_mismatches);
+    ds4_gpu_cleanup();
+    return pass ? 0 : 1;
+}
+
+#else
+
+int main(void) {
+    fprintf(stderr, "test_metal_iq2_midonly: skipped (Metal requires macOS)\n");
+    return 0;
+}
+
+#endif
diff --git a/tests/test_metal_iq2_ssd_grouped_mm b/tests/test_metal_iq2_ssd_grouped_mm
new file mode 100755
index 0000000000..abe4063ebf
Binary files /dev/null and b/tests/test_metal_iq2_ssd_grouped_mm differ
diff --git a/tests/test_metal_iq2_ssd_grouped_mm.c b/tests/test_metal_iq2_ssd_grouped_mm.c
new file mode 100644
index 0000000000..038c0bc3be
--- /dev/null
+++ b/tests/test_metal_iq2_ssd_grouped_mm.c
@@ -0,0 +1,1826 @@
+#define _DARWIN_C_SOURCE
+
+#include "ds4_gpu.h"
+
+#include <fcntl.h>
+#include <math.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#ifdef __APPLE__
+
+#define IQ2_XXS_TYPE 16u
+#define Q2_K_TYPE 10u
+#define QK_K 256u
+#define IN_DIM 256u
+#define MID_DIM 256u
+#define OUT_DIM 64u
+#define N_TOTAL_EXPERT 8u
+#define N_EXPERT 6u
+#define MAX_TOKENS 33u
+#define N_TOTAL_EXPERT_256 256u
+#define HIGH_EXPERT_ID 255u
+#define CLAMP 4.0f
+#define SENTINEL 1234567.0f
+#define TAIL_CULL_ENV \
+    "DS4_METAL_ENABLE_IQ2_XXS_SSD_PREFILL_MM_ADDR_TAIL_CULL"
+#define TAIL_CULL_DISABLE_ENV \
+    "DS4_METAL_DISABLE_IQ2_XXS_SSD_PREFILL_MM_ADDR_TAIL_CULL"
+
+/* Production Flash routed-expert geometry.  Eight physical experts keep the
+ * standalone fixture bounded while retaining the production top-6 routing,
+ * 32-token grouped tile, and exact quantized row strides. */
+#define FULL_IN_DIM 4096u
+#define FULL_MID_DIM 2048u
+#define FULL_OUT_DIM 4096u
+#define FULL_TOKENS 32u
+#define FULL_GUARD_WORDS 64u
+#define FULL_GUARD_BITS 0x51a7c3e9u
+
+typedef struct {
+    uint16_t d;
+    uint16_t qs[QK_K / 8u];
+} block_iq2_xxs;
+
+typedef struct {
+    uint8_t scales[QK_K / 16u];
+    uint8_t qs[QK_K / 4u];
+    uint16_t d;
+    uint16_t dmin;
+} block_q2_K;
+
+typedef struct {
+    uint32_t tokens;
+    uint64_t pair_count;
+    uint64_t out_count;
+    float *gate;
+    float *up;
+    float *mid;
+    float *out;
+    uint64_t guard_mismatches;
+} run_result;
+
+typedef struct {
+    uint64_t pair_count;
+    uint64_t out_count;
+    float *gate;
+    float *up;
+    float *mid;
+    float *out;
+    uint64_t guard_mismatches;
+} full_run_result;
+
+typedef struct {
+    uint64_t candidate_calls;
+    uint64_t calls;
+    uint64_t tokens;
+    uint64_t rows;
+    uint64_t require_failures;
+    uint32_t min_tokens;
+    uint32_t max_tokens;
+} mm_stats_snapshot;
+
+static uint32_t *full_guarded_payload(uint64_t payload_words);
+static int full_check_guard(const char *run_name,
+                            const char *tensor_name,
+                            const ds4_gpu_tensor *tensor,
+                            uint64_t payload_words,
+                            uint64_t *mismatches);
+
+/* Test-only counter reader implemented by the Metal backend. */
+int ds4_gpu_test_iq2_stream_addr_mm_stats(
+    uint64_t *candidate_calls, uint64_t *calls, uint64_t *tokens,
+    uint64_t *rows, uint64_t *require_failures, uint32_t *min_tokens,
+    uint32_t *max_tokens);
+int ds4_gpu_test_iq2_stream_addr_mm_policy(
+    int enable, int require, int disable, int material_ready,
+    int *requested_out, int *required_out);
+
+bool ds4_log_is_tty(FILE *fp) {
+    (void)fp;
+    return false;
+}
+
+static uint64_t align_up(uint64_t value, uint64_t alignment) {
+    return (value + alignment - 1u) / alignment * alignment;
+}
+
+static int check_grouped_mm_policy(void) {
+    int ok = 1;
+#define CHECK_POLICY(label, valid_expected, request_expected,                \
+                     require_expected, enable, require, disable, ready) do { \
+    int requested = -1;                                                      \
+    int required = -1;                                                       \
+    const int valid = ds4_gpu_test_iq2_stream_addr_mm_policy(                \
+        (enable), (require), (disable), (ready), &requested, &required);     \
+    const int case_ok = valid == (valid_expected) &&                         \
+        requested == (request_expected) && required == (require_expected);  \
+    fprintf(stderr,                                                          \
+            "IQ2 grouped-MM policy %-28s %s valid=%d request=%d "           \
+            "require=%d\n",                                                \
+            (label), case_ok ? "PASS" : "FAIL", valid, requested, required);\
+    ok = case_ok && ok;                                                      \
+} while (0)
+
+    /* Unset defaults: automatic selection, with fail-closed coverage only
+     * after the complete selected-address domain is materially ready. */
+    CHECK_POLICY("default-ready",       1, 1, 1, -1, -1, -1, 1);
+    CHECK_POLICY("default-small-cache", 1, 1, 0, -1, -1, -1, 0);
+
+    /* Value-aware rollback and explicit fallback controls. */
+    CHECK_POLICY("enable-zero",         1, 0, 0,  0, -1, -1, 1);
+    CHECK_POLICY("enable-one",          1, 1, 1,  1, -1, -1, 1);
+    CHECK_POLICY("require-zero",        1, 1, 0, -1,  0, -1, 1);
+    CHECK_POLICY("require-one-not-ready", 1, 1, 1, -1, 1, -1, 0);
+    CHECK_POLICY("require-over-enable-zero", 1, 1, 1, 0, 1, -1, 0);
+    CHECK_POLICY("disable-one",         1, 0, 0, -1, -1,  1, 1);
+    CHECK_POLICY("disable-zero",        1, 1, 1, -1, -1,  0, 1);
+    /* The explicit REQUIRE is retained so an eligible IQ2 prefill fails at
+     * the candidate boundary; unrelated shapes and short tails stay valid. */
+    CHECK_POLICY("require-disable-conflict", 1, 0, 1, -1, 1, 1, 1);
+#undef CHECK_POLICY
+    return ok;
+}
+
+static void fill_iq2(block_iq2_xxs *matrix, uint32_t salt,
+                     uint32_t n_total_expert) {
+    for (uint32_t expert = 0; expert < n_total_expert; expert++) {
+        for (uint32_t row = 0; row < MID_DIM; row++) {
+            block_iq2_xxs *b = matrix + (uint64_t)expert * MID_DIM + row;
+            const uint32_t key = salt * 977u + expert * 431u + row * 37u;
+            b->d = (uint16_t)(0x1800u + ((key & 1u) ? 0x0200u : 0u));
+            for (uint32_t i = 0; i < QK_K / 8u; i++) {
+                b->qs[i] = (uint16_t)(key + i * 509u + (i >> 2u) * 131u);
+            }
+        }
+    }
+}
+
+static void fill_q2(block_q2_K *matrix, uint32_t n_total_expert) {
+    for (uint32_t expert = 0; expert < n_total_expert; expert++) {
+        for (uint32_t row = 0; row < OUT_DIM; row++) {
+            block_q2_K *b = matrix + (uint64_t)expert * OUT_DIM + row;
+            const uint32_t key = expert * 617u + row * 73u;
+            for (uint32_t group = 0; group < QK_K / 16u; group++) {
+                const uint8_t scale =
+                    (uint8_t)(1u + (key + 3u * group) % 7u);
+                const uint8_t min = (uint8_t)((key / 5u + group) % 4u);
+                b->scales[group] = (uint8_t)(scale | (min << 4u));
+            }
+            for (uint32_t i = 0; i < QK_K / 4u; i++) {
+                b->qs[i] = (uint8_t)(key + 29u * i + (i >> 1u) * 7u);
+            }
+            b->d = 0x1800u;
+            b->dmin = 0x1400u;
+        }
+    }
+}
+
+static void fill_iq2_full(block_iq2_xxs *matrix, uint32_t salt) {
+    const uint32_t blocks_per_row = FULL_IN_DIM / QK_K;
+    for (uint32_t expert = 0; expert < N_TOTAL_EXPERT; expert++) {
+        for (uint32_t row = 0; row < FULL_MID_DIM; row++) {
+            for (uint32_t block = 0; block < blocks_per_row; block++) {
+                block_iq2_xxs *b = matrix +
+                    ((uint64_t)expert * FULL_MID_DIM + row) *
+                        blocks_per_row + block;
+                const uint32_t key = salt * 977u + expert * 431u +
+                                     row * 37u + block * 811u;
+                b->d = (uint16_t)(0x1800u +
+                                  ((key & 1u) ? 0x0200u : 0u));
+                for (uint32_t i = 0; i < QK_K / 8u; i++) {
+                    b->qs[i] = (uint16_t)(key + i * 509u +
+                                          (i >> 2u) * 131u);
+                }
+            }
+        }
+    }
+}
+
+static void fill_q2_full(block_q2_K *matrix) {
+    const uint32_t blocks_per_row = FULL_MID_DIM / QK_K;
+    for (uint32_t expert = 0; expert < N_TOTAL_EXPERT; expert++) {
+        for (uint32_t row = 0; row < FULL_OUT_DIM; row++) {
+            for (uint32_t block = 0; block < blocks_per_row; block++) {
+                block_q2_K *b = matrix +
+                    ((uint64_t)expert * FULL_OUT_DIM + row) *
+                        blocks_per_row + block;
+                const uint32_t key = expert * 617u + row * 73u +
+                                     block * 991u;
+                for (uint32_t group = 0; group < QK_K / 16u; group++) {
+                    const uint8_t scale =
+                        (uint8_t)(1u + (key + 3u * group) % 7u);
+                    const uint8_t min =
+                        (uint8_t)((key / 5u + group) % 4u);
+                    b->scales[group] =
+                        (uint8_t)(scale | (uint8_t)(min << 4u));
+                }
+                for (uint32_t i = 0; i < QK_K / 4u; i++) {
+                    b->qs[i] =
+                        (uint8_t)(key + 29u * i + (i >> 1u) * 7u);
+                }
+                b->d = 0x1800u;
+                b->dmin = 0x1400u;
+            }
+        }
+    }
+}
+
+static int result_alloc(run_result *r, uint32_t tokens) {
+    memset(r, 0, sizeof(*r));
+    r->tokens = tokens;
+    r->pair_count = (uint64_t)tokens * N_EXPERT * MID_DIM;
+    r->out_count = (uint64_t)tokens * OUT_DIM;
+    r->gate = calloc((size_t)r->pair_count, sizeof(float));
+    r->up = calloc((size_t)r->pair_count, sizeof(float));
+    r->mid = calloc((size_t)r->pair_count, sizeof(float));
+    r->out = calloc((size_t)r->out_count, sizeof(float));
+    return r->gate && r->up && r->mid && r->out;
+}
+
+static void result_free(run_result *r) {
+    free(r->gate);
+    free(r->up);
+    free(r->mid);
+    free(r->out);
+    memset(r, 0, sizeof(*r));
+}
+
+static int run_once(
+        const char *run_name,
+        run_result *result,
+        const void *model,
+        uint64_t model_size,
+        uint64_t gate_offset,
+        uint64_t up_offset,
+        uint64_t down_offset,
+        uint64_t gate_expert_bytes,
+        uint64_t gate_row_bytes,
+        uint64_t down_expert_bytes,
+        uint64_t down_row_bytes,
+        uint32_t n_total_expert,
+        const float *x,
+        const int32_t *selected,
+        const float *weights,
+        bool allow_mid_f16) {
+    const uint32_t tokens = result->tokens;
+    const uint64_t x_count = (uint64_t)tokens * IN_DIM;
+    const uint64_t route_count = (uint64_t)tokens * N_EXPERT;
+    const uint64_t expert_count = (uint64_t)tokens * N_EXPERT * OUT_DIM;
+    const uint64_t guard_bytes =
+        (uint64_t)FULL_GUARD_WORDS * sizeof(uint32_t);
+    uint32_t *pair_init = full_guarded_payload(result->pair_count);
+    uint32_t *expert_init = full_guarded_payload(expert_count);
+    uint32_t *out_init = full_guarded_payload(result->out_count);
+    if (!pair_init || !expert_init || !out_init) {
+        free(pair_init);
+        free(expert_init);
+        free(out_init);
+        return 0;
+    }
+
+    ds4_gpu_tensor *x_t = ds4_gpu_tensor_alloc(x_count * sizeof(float));
+    ds4_gpu_tensor *selected_t =
+        ds4_gpu_tensor_alloc(route_count * sizeof(int32_t));
+    ds4_gpu_tensor *weights_t =
+        ds4_gpu_tensor_alloc(route_count * sizeof(float));
+    ds4_gpu_tensor *gate_t =
+        ds4_gpu_tensor_alloc(result->pair_count * sizeof(float) + guard_bytes);
+    ds4_gpu_tensor *up_t =
+        ds4_gpu_tensor_alloc(result->pair_count * sizeof(float) + guard_bytes);
+    ds4_gpu_tensor *mid_t =
+        ds4_gpu_tensor_alloc(result->pair_count * sizeof(float) + guard_bytes);
+    ds4_gpu_tensor *experts_t =
+        ds4_gpu_tensor_alloc(expert_count * sizeof(float) + guard_bytes);
+    ds4_gpu_tensor *out_t =
+        ds4_gpu_tensor_alloc(result->out_count * sizeof(float) + guard_bytes);
+    int ok = x_t && selected_t && weights_t && gate_t && up_t && mid_t &&
+             experts_t && out_t;
+    ok = ok && ds4_gpu_tensor_write(x_t, 0, x, x_count * sizeof(float));
+    ok = ok && ds4_gpu_tensor_write(selected_t, 0, selected,
+                                     route_count * sizeof(int32_t));
+    ok = ok && ds4_gpu_tensor_write(weights_t, 0, weights,
+                                     route_count * sizeof(float));
+    ok = ok && ds4_gpu_tensor_write(
+        gate_t, 0, pair_init,
+        result->pair_count * sizeof(float) + guard_bytes);
+    ok = ok && ds4_gpu_tensor_write(
+        up_t, 0, pair_init,
+        result->pair_count * sizeof(float) + guard_bytes);
+    ok = ok && ds4_gpu_tensor_write(
+        mid_t, 0, pair_init,
+        result->pair_count * sizeof(float) + guard_bytes);
+    ok = ok && ds4_gpu_tensor_write(
+        experts_t, 0, expert_init,
+        expert_count * sizeof(float) + guard_bytes);
+    ok = ok && ds4_gpu_tensor_write(
+        out_t, 0, out_init,
+        result->out_count * sizeof(float) + guard_bytes);
+
+    bool mid_is_f16 = true;
+    if (ok) {
+        ok = ds4_gpu_routed_moe_batch_tensor(
+            out_t, gate_t, up_t, mid_t, experts_t, model, model_size,
+            gate_offset, up_offset, down_offset, IQ2_XXS_TYPE, Q2_K_TYPE,
+            gate_expert_bytes, gate_row_bytes, down_expert_bytes,
+            down_row_bytes, IN_DIM, MID_DIM, OUT_DIM, selected_t, weights_t,
+            n_total_expert, N_EXPERT, CLAMP, x_t, 0u, tokens,
+            &mid_is_f16, false);
+    }
+    if (ok && mid_is_f16 && !allow_mid_f16) {
+        fprintf(stderr,
+                "IQ2_XXS SSD grouped-MM oracle unexpectedly selected f16 mid\n");
+        ok = 0;
+    }
+    ok = ok && ds4_gpu_tensor_read(gate_t, 0, result->gate,
+                                    result->pair_count * sizeof(float));
+    ok = ok && ds4_gpu_tensor_read(up_t, 0, result->up,
+                                    result->pair_count * sizeof(float));
+    ok = ok && ds4_gpu_tensor_read(mid_t, 0, result->mid,
+                                    result->pair_count * sizeof(float));
+    ok = ok && ds4_gpu_tensor_read(out_t, 0, result->out,
+                                    result->out_count * sizeof(float));
+
+    result->guard_mismatches = 0;
+    if (gate_t) {
+        const int guard_ok = full_check_guard(
+            run_name, "gate", gate_t, result->pair_count,
+            &result->guard_mismatches);
+        ok = guard_ok && ok;
+    }
+    if (up_t) {
+        const int guard_ok = full_check_guard(
+            run_name, "up", up_t, result->pair_count,
+            &result->guard_mismatches);
+        ok = guard_ok && ok;
+    }
+    if (mid_t) {
+        const int guard_ok = full_check_guard(
+            run_name, "mid", mid_t, result->pair_count,
+            &result->guard_mismatches);
+        ok = guard_ok && ok;
+    }
+    if (experts_t) {
+        const int guard_ok = full_check_guard(
+            run_name, "experts", experts_t, expert_count,
+            &result->guard_mismatches);
+        ok = guard_ok && ok;
+    }
+    if (out_t) {
+        const int guard_ok = full_check_guard(
+            run_name, "out", out_t, result->out_count,
+            &result->guard_mismatches);
+        ok = guard_ok && ok;
+    }
+
+    ds4_gpu_tensor_free(x_t);
+    ds4_gpu_tensor_free(selected_t);
+    ds4_gpu_tensor_free(weights_t);
+    ds4_gpu_tensor_free(gate_t);
+    ds4_gpu_tensor_free(up_t);
+    ds4_gpu_tensor_free(mid_t);
+    ds4_gpu_tensor_free(experts_t);
+    ds4_gpu_tensor_free(out_t);
+    free(pair_init);
+    free(expert_init);
+    free(out_init);
+    return ok;
+}
+
+static int compare_array(const char *case_name, const char *tensor_name,
+                         const float *candidate, const float *control,
+                         uint64_t count, double max_limit,
+                         double rms_limit) {
+    double max_abs = 0.0;
+    long double sum_sq = 0.0;
+    uint64_t nonfinite = 0;
+    uint64_t unwritten = 0;
+    uint32_t sentinel_bits = 0;
+    memcpy(&sentinel_bits, &(float){ SENTINEL }, sizeof(sentinel_bits));
+    for (uint64_t i = 0; i < count; i++) {
+        uint32_t candidate_bits = 0;
+        uint32_t control_bits = 0;
+        memcpy(&candidate_bits, candidate + i, sizeof(candidate_bits));
+        memcpy(&control_bits, control + i, sizeof(control_bits));
+        if (candidate_bits == sentinel_bits || control_bits == sentinel_bits) {
+            unwritten++;
+            continue;
+        }
+        if ((candidate_bits & 0x7f800000u) == 0x7f800000u ||
+            (control_bits & 0x7f800000u) == 0x7f800000u) {
+            nonfinite++;
+            continue;
+        }
+        const double diff = fabs((double)candidate[i] - (double)control[i]);
+        if (diff > max_abs) max_abs = diff;
+        sum_sq += (long double)diff * (long double)diff;
+    }
+    const double rms = count ? sqrt((double)(sum_sq / count)) : 0.0;
+    const int pass = nonfinite == 0 && unwritten == 0 &&
+                     max_abs <= max_limit && rms <= rms_limit;
+    fprintf(stderr,
+            "IQ2_XXS SSD grouped-MM %-12s %-4s %s count=%llu "
+            "max_abs=%.9g rms=%.9g limits=%.9g/%.9g nonfinite=%llu "
+            "unwritten=%llu\n",
+            case_name, tensor_name, pass ? "PASS" : "FAIL",
+            (unsigned long long)count, max_abs, rms, max_limit, rms_limit,
+            (unsigned long long)nonfinite,
+            (unsigned long long)unwritten);
+    return pass;
+}
+
+static int compare_results(const char *name, const run_result *candidate,
+                           const run_result *control) {
+    if (candidate->pair_count != control->pair_count ||
+        candidate->out_count != control->out_count ||
+        candidate->guard_mismatches != 0 ||
+        control->guard_mismatches != 0) {
+        return 0;
+    }
+    int ok = compare_array(name, "gate", candidate->gate, control->gate,
+                           candidate->pair_count, 0.025, 0.004);
+    ok = compare_array(name, "up", candidate->up, control->up,
+                       candidate->pair_count, 0.025, 0.004) && ok;
+    ok = compare_array(name, "mid", candidate->mid, control->mid,
+                       candidate->pair_count, 0.08, 0.015) && ok;
+    ok = compare_array(name, "out", candidate->out, control->out,
+                       candidate->out_count, 0.08, 0.015) && ok;
+    return ok;
+}
+
+static void clear_tail_cull_test_env(void) {
+    unsetenv(TAIL_CULL_ENV);
+    unsetenv(TAIL_CULL_DISABLE_ENV);
+    unsetenv("DS4_METAL_ENABLE_IQ2_XXS_SSD_PREFILL_MM");
+    unsetenv("DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM");
+    unsetenv("DS4_METAL_DISABLE_IQ2_XXS_SSD_PREFILL_MM");
+}
+
+static void configure_tail_cull_test_env(bool enable_tail_cull) {
+    unsetenv(TAIL_CULL_DISABLE_ENV);
+    unsetenv("DS4_METAL_DISABLE_IQ2_XXS_SSD_PREFILL_MM");
+    setenv("DS4_METAL_ENABLE_IQ2_XXS_SSD_PREFILL_MM", "1", 1);
+    setenv("DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM", "1", 1);
+    if (enable_tail_cull) {
+        setenv(TAIL_CULL_ENV, "1", 1);
+    } else {
+        unsetenv(TAIL_CULL_ENV);
+    }
+}
+
+static int compare_array_bit_exact(const char *case_name,
+                                   const char *tensor_name,
+                                   const float *candidate,
+                                   const float *control,
+                                   uint64_t count) {
+    uint32_t sentinel_bits = 0;
+    memcpy(&sentinel_bits, &(float){ SENTINEL }, sizeof(sentinel_bits));
+    uint64_t mismatches = 0;
+    uint64_t nonfinite = 0;
+    uint64_t unwritten = 0;
+    uint64_t first_mismatch = UINT64_MAX;
+    uint32_t first_candidate = 0;
+    uint32_t first_control = 0;
+    for (uint64_t i = 0; i < count; i++) {
+        uint32_t candidate_bits = 0;
+        uint32_t control_bits = 0;
+        memcpy(&candidate_bits, candidate + i, sizeof(candidate_bits));
+        memcpy(&control_bits, control + i, sizeof(control_bits));
+        if (candidate_bits == sentinel_bits || control_bits == sentinel_bits) {
+            unwritten++;
+        }
+        if ((candidate_bits & 0x7f800000u) == 0x7f800000u ||
+            (control_bits & 0x7f800000u) == 0x7f800000u) {
+            nonfinite++;
+        }
+        if (candidate_bits != control_bits) {
+            if (first_mismatch == UINT64_MAX) {
+                first_mismatch = i;
+                first_candidate = candidate_bits;
+                first_control = control_bits;
+            }
+            mismatches++;
+        }
+    }
+    const int pass = mismatches == 0 && nonfinite == 0 && unwritten == 0;
+    fprintf(stderr,
+            "IQ2_XXS SSD tail-cull %-20s %-4s %s count=%llu "
+            "bit_mismatches=%llu nonfinite=%llu unwritten=%llu",
+            case_name, tensor_name, pass ? "PASS" : "FAIL",
+            (unsigned long long)count,
+            (unsigned long long)mismatches,
+            (unsigned long long)nonfinite,
+            (unsigned long long)unwritten);
+    if (first_mismatch != UINT64_MAX) {
+        fprintf(stderr, " first=%llu candidate=0x%08x control=0x%08x",
+                (unsigned long long)first_mismatch,
+                first_candidate, first_control);
+    }
+    fputc('\n', stderr);
+    return pass;
+}
+
+static int compare_results_bit_exact(const char *name,
+                                     const run_result *candidate,
+                                     const run_result *control) {
+    const int shape_ok =
+        candidate->pair_count == control->pair_count &&
+        candidate->out_count == control->out_count &&
+        candidate->guard_mismatches == 0 &&
+        control->guard_mismatches == 0;
+    if (!shape_ok) {
+        fprintf(stderr,
+                "IQ2_XXS SSD tail-cull %-20s shape/guard FAIL\n", name);
+        return 0;
+    }
+    int ok = compare_array_bit_exact(
+        name, "gate", candidate->gate, control->gate,
+        candidate->pair_count);
+    ok = compare_array_bit_exact(
+        name, "up", candidate->up, control->up,
+        candidate->pair_count) && ok;
+    ok = compare_array_bit_exact(
+        name, "mid", candidate->mid, control->mid,
+        candidate->pair_count) && ok;
+    ok = compare_array_bit_exact(
+        name, "out", candidate->out, control->out,
+        candidate->out_count) && ok;
+    return ok;
+}
+
+static int run_tail_cull_bit_exact(
+        const void *model,
+        uint64_t model_size,
+        uint64_t gate_offset,
+        uint64_t up_offset,
+        uint64_t down_offset,
+        uint64_t gate_expert_bytes,
+        uint64_t gate_row_bytes,
+        uint64_t down_expert_bytes,
+        uint64_t down_row_bytes,
+        uint32_t n_total_expert,
+        uint32_t cache_budget,
+        const float *x,
+        const int32_t *selected,
+        const float *weights) {
+    run_result control;
+    run_result candidate;
+    run_result repeat;
+    memset(&control, 0, sizeof(control));
+    memset(&candidate, 0, sizeof(candidate));
+    memset(&repeat, 0, sizeof(repeat));
+    int ok = 0;
+
+    clear_tail_cull_test_env();
+    if (!result_alloc(&control, MAX_TOKENS) ||
+        !result_alloc(&candidate, MAX_TOKENS) ||
+        !result_alloc(&repeat, MAX_TOKENS)) {
+        goto cleanup;
+    }
+
+    configure_tail_cull_test_env(false);
+    ds4_gpu_set_streaming_expert_cache_budget(cache_budget);
+    const int control_ok = run_once(
+        "tail-cull-control", &control, model, model_size,
+        gate_offset, up_offset, down_offset, gate_expert_bytes,
+        gate_row_bytes, down_expert_bytes, down_row_bytes,
+        n_total_expert, x, selected, weights, false);
+
+    configure_tail_cull_test_env(true);
+    ds4_gpu_set_streaming_expert_cache_budget(cache_budget);
+    const int candidate_ok = run_once(
+        "tail-cull-candidate", &candidate, model, model_size,
+        gate_offset, up_offset, down_offset, gate_expert_bytes,
+        gate_row_bytes, down_expert_bytes, down_row_bytes,
+        n_total_expert, x, selected, weights, false);
+
+    configure_tail_cull_test_env(true);
+    ds4_gpu_set_streaming_expert_cache_budget(cache_budget);
+    const int repeat_ok = run_once(
+        "tail-cull-repeat", &repeat, model, model_size,
+        gate_offset, up_offset, down_offset, gate_expert_bytes,
+        gate_row_bytes, down_expert_bytes, down_row_bytes,
+        n_total_expert, x, selected, weights, false);
+
+    ok = control_ok && candidate_ok && repeat_ok;
+    if (ok) {
+        const int candidate_exact = compare_results_bit_exact(
+            "candidate-control", &candidate, &control);
+        const int repeat_exact = compare_results_bit_exact(
+            "repeat-control", &repeat, &control);
+        ok = candidate_exact && repeat_exact;
+    }
+    fprintf(stderr,
+            "IQ2_XXS/Q2_K Metal SSD address-MM tail-cull bit-exact %s\n",
+            ok ? "PASS" : "FAIL");
+
+cleanup:
+    clear_tail_cull_test_env();
+    result_free(&control);
+    result_free(&candidate);
+    result_free(&repeat);
+    return ok;
+}
+
+static int run_pair(
+        const char *name,
+        uint32_t tokens,
+        const void *model,
+        uint64_t model_size,
+        uint64_t gate_offset,
+        uint64_t up_offset,
+        uint64_t down_offset,
+        uint64_t gate_expert_bytes,
+        uint64_t gate_row_bytes,
+        uint64_t down_expert_bytes,
+        uint64_t down_row_bytes,
+        uint32_t n_total_expert,
+        uint32_t cache_budget,
+        const float *x,
+        const int32_t *selected,
+        const float *weights) {
+    run_result control;
+    run_result candidate;
+    memset(&control, 0, sizeof(control));
+    memset(&candidate, 0, sizeof(candidate));
+    if (!result_alloc(&control, tokens) || !result_alloc(&candidate, tokens)) {
+        result_free(&control);
+        result_free(&candidate);
+        return 0;
+    }
+
+    unsetenv("DS4_METAL_ENABLE_IQ2_XXS_SSD_PREFILL_MM");
+    unsetenv("DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM");
+    setenv("DS4_METAL_DISABLE_IQ2_XXS_SSD_PREFILL_MM", "1", 1);
+    ds4_gpu_set_streaming_expert_cache_budget(cache_budget);
+    char control_name[64];
+    char candidate_name[64];
+    snprintf(control_name, sizeof(control_name), "%s-control", name);
+    snprintf(candidate_name, sizeof(candidate_name), "%s-candidate", name);
+    int ok = run_once(control_name, &control,
+                      model, model_size, gate_offset, up_offset,
+                      down_offset, gate_expert_bytes, gate_row_bytes,
+                      down_expert_bytes, down_row_bytes, n_total_expert,
+                      x, selected, weights, false);
+
+    unsetenv("DS4_METAL_DISABLE_IQ2_XXS_SSD_PREFILL_MM");
+    unsetenv("DS4_METAL_ENABLE_IQ2_XXS_SSD_PREFILL_MM");
+    unsetenv("DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM");
+    ds4_gpu_set_streaming_expert_cache_budget(cache_budget);
+    ok = ok && run_once(candidate_name, &candidate,
+                        model, model_size, gate_offset, up_offset,
+                        down_offset, gate_expert_bytes, gate_row_bytes,
+                        down_expert_bytes, down_row_bytes, n_total_expert,
+                        x, selected, weights, false);
+    if (ok) ok = compare_results(name, &candidate, &control);
+
+    result_free(&control);
+    result_free(&candidate);
+    return ok;
+}
+
+static int read_mm_stats(mm_stats_snapshot *s) {
+    memset(s, 0, sizeof(*s));
+    return ds4_gpu_test_iq2_stream_addr_mm_stats(
+        &s->candidate_calls, &s->calls, &s->tokens, &s->rows,
+        &s->require_failures, &s->min_tokens, &s->max_tokens);
+}
+
+static int check_mm_stats_delta(const char *name,
+                                const mm_stats_snapshot *before,
+                                const mm_stats_snapshot *after,
+                                uint64_t expected_tokens) {
+    const int monotonic =
+        after->candidate_calls >= before->candidate_calls &&
+        after->calls >= before->calls &&
+        after->tokens >= before->tokens &&
+        after->rows >= before->rows &&
+        after->require_failures >= before->require_failures;
+    const uint64_t candidates = monotonic ?
+        after->candidate_calls - before->candidate_calls : UINT64_MAX;
+    const uint64_t calls = monotonic ?
+        after->calls - before->calls : UINT64_MAX;
+    const uint64_t tokens = monotonic ?
+        after->tokens - before->tokens : UINT64_MAX;
+    const uint64_t rows = monotonic ?
+        after->rows - before->rows : UINT64_MAX;
+    const uint64_t failures = monotonic ?
+        after->require_failures - before->require_failures : UINT64_MAX;
+    const int ok = monotonic && candidates == 2u && calls == 1u &&
+        tokens == expected_tokens &&
+        rows == expected_tokens * N_EXPERT && failures == 0u;
+    fprintf(stderr,
+            "IQ2_XXS SSD grouped-MM %-15s coverage %s candidates=%llu "
+            "calls=%llu tokens=%llu rows=%llu require_failures=%llu\n",
+            name, ok ? "PASS" : "FAIL",
+            (unsigned long long)candidates,
+            (unsigned long long)calls,
+            (unsigned long long)tokens,
+            (unsigned long long)rows,
+            (unsigned long long)failures);
+    return ok;
+}
+
+static int check_implicit_require_fault(
+        const void *model,
+        uint64_t model_size,
+        uint64_t gate_offset,
+        uint64_t up_offset,
+        uint64_t down_offset,
+        uint64_t gate_expert_bytes,
+        uint64_t gate_row_bytes,
+        uint64_t down_expert_bytes,
+        uint64_t down_row_bytes,
+        const float *x,
+        const int32_t *selected,
+        const float *weights) {
+    run_result result;
+    run_result tail_result;
+    memset(&result, 0, sizeof(result));
+    memset(&tail_result, 0, sizeof(tail_result));
+    if (!result_alloc(&result, 32u) || !result_alloc(&tail_result, 31u)) {
+        result_free(&result);
+        result_free(&tail_result);
+        return 0;
+    }
+
+    mm_stats_snapshot before;
+    mm_stats_snapshot after;
+    int ok = read_mm_stats(&before);
+    ds4_gpu_test_set_flags(
+        DS4_GPU_TEST_IQ2_SSD_GROUPED_PIPELINE_FAILURE);
+
+    unsetenv("DS4_METAL_ENABLE_IQ2_XXS_SSD_PREFILL_MM");
+    unsetenv("DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM");
+    unsetenv("DS4_METAL_DISABLE_IQ2_XXS_SSD_PREFILL_MM");
+    ds4_gpu_set_streaming_expert_cache_budget(N_TOTAL_EXPERT);
+    const int default_failed = !run_once(
+        "default-fail-closed", &result, model, model_size,
+        gate_offset, up_offset, down_offset, gate_expert_bytes,
+        gate_row_bytes, down_expert_bytes, down_row_bytes,
+        N_TOTAL_EXPERT, x, selected, weights, false);
+
+    setenv("DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM", "0", 1);
+    ds4_gpu_set_streaming_expert_cache_budget(N_TOTAL_EXPERT);
+    const int require_zero_fallback = run_once(
+        "require-zero-fallback", &result, model, model_size,
+        gate_offset, up_offset, down_offset, gate_expert_bytes,
+        gate_row_bytes, down_expert_bytes, down_row_bytes,
+        N_TOTAL_EXPERT, x, selected, weights, false);
+
+    unsetenv("DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM");
+    setenv("DS4_METAL_DISABLE_IQ2_XXS_SSD_PREFILL_MM", "1", 1);
+    ds4_gpu_set_streaming_expert_cache_budget(N_TOTAL_EXPERT);
+    const int disable_fallback = run_once(
+        "disable-fallback", &result, model, model_size,
+        gate_offset, up_offset, down_offset, gate_expert_bytes,
+        gate_row_bytes, down_expert_bytes, down_row_bytes,
+        N_TOTAL_EXPERT, x, selected, weights, false);
+
+    setenv("DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM", "1", 1);
+    ds4_gpu_set_streaming_expert_cache_budget(N_TOTAL_EXPERT);
+    const int require_disable_failed = !run_once(
+        "require-disable-fail", &result, model, model_size,
+        gate_offset, up_offset, down_offset, gate_expert_bytes,
+        gate_row_bytes, down_expert_bytes, down_row_bytes,
+        N_TOTAL_EXPERT, x, selected, weights, false);
+
+    /* The automatic fail-closed arm must not turn a cache that cannot hold
+     * the complete expert domain into an error. Explicit REQUIRE remains
+     * strong for exactly that same materially-ineligible configuration. */
+    unsetenv("DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM");
+    unsetenv("DS4_METAL_DISABLE_IQ2_XXS_SSD_PREFILL_MM");
+    ds4_gpu_set_streaming_expert_cache_budget(N_TOTAL_EXPERT - 1u);
+    const int small_cache_fallback = run_once(
+        "small-cache-fallback", &result, model, model_size,
+        gate_offset, up_offset, down_offset, gate_expert_bytes,
+        gate_row_bytes, down_expert_bytes, down_row_bytes,
+        N_TOTAL_EXPERT, x, selected, weights, true);
+
+    setenv("DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM", "1", 1);
+    ds4_gpu_set_streaming_expert_cache_budget(N_TOTAL_EXPERT - 1u);
+    const int small_cache_require_failed = !run_once(
+        "small-cache-require-fail", &result, model, model_size,
+        gate_offset, up_offset, down_offset, gate_expert_bytes,
+        gate_row_bytes, down_expert_bytes, down_row_bytes,
+        N_TOTAL_EXPERT, x, selected, weights, false);
+
+    /* A short final chunk is outside the grouped-MM candidate contract even
+     * with a full cache and the injected missing pipeline. It must retain the
+     * established path, including for the contradictory explicit controls. */
+    unsetenv("DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM");
+    ds4_gpu_set_streaming_expert_cache_budget(N_TOTAL_EXPERT);
+    const int short_tail_fallback = run_once(
+        "short-tail-fallback", &tail_result, model, model_size,
+        gate_offset, up_offset, down_offset, gate_expert_bytes,
+        gate_row_bytes, down_expert_bytes, down_row_bytes,
+        N_TOTAL_EXPERT, x, selected, weights, false);
+
+    setenv("DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM", "1", 1);
+    setenv("DS4_METAL_DISABLE_IQ2_XXS_SSD_PREFILL_MM", "1", 1);
+    ds4_gpu_set_streaming_expert_cache_budget(N_TOTAL_EXPERT);
+    const int short_tail_conflict_fallback = run_once(
+        "short-tail-conflict-fallback", &tail_result, model, model_size,
+        gate_offset, up_offset, down_offset, gate_expert_bytes,
+        gate_row_bytes, down_expert_bytes, down_row_bytes,
+        N_TOTAL_EXPERT, x, selected, weights, false);
+
+    ds4_gpu_test_set_flags(0);
+    unsetenv("DS4_METAL_ENABLE_IQ2_XXS_SSD_PREFILL_MM");
+    unsetenv("DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM");
+    unsetenv("DS4_METAL_DISABLE_IQ2_XXS_SSD_PREFILL_MM");
+    ok = read_mm_stats(&after) && ok;
+
+    const int monotonic =
+        after.candidate_calls >= before.candidate_calls &&
+        after.calls >= before.calls &&
+        after.require_failures >= before.require_failures;
+    const uint64_t candidate_delta = monotonic ?
+        after.candidate_calls - before.candidate_calls : UINT64_MAX;
+    const uint64_t call_delta = monotonic ?
+        after.calls - before.calls : UINT64_MAX;
+    const uint64_t failure_delta = monotonic ?
+        after.require_failures - before.require_failures : UINT64_MAX;
+    const int coverage_ok = monotonic && candidate_delta == 6u &&
+        call_delta == 0u && failure_delta == 3u;
+    ok = default_failed && require_zero_fallback && disable_fallback &&
+        require_disable_failed && small_cache_fallback &&
+        small_cache_require_failed && short_tail_fallback &&
+        short_tail_conflict_fallback && coverage_ok && ok;
+    fprintf(stderr,
+            "IQ2 grouped-MM implicit REQUIRE integration %s "
+            "default_fail=%d require0_fallback=%d disable_fallback=%d "
+            "require_disable_fail=%d small_fallback=%d "
+            "small_require_fail=%d tail_fallback=%d tail_conflict=%d "
+            "candidates=%llu calls=%llu require_failures=%llu\n",
+            ok ? "PASS" : "FAIL", default_failed, require_zero_fallback,
+            disable_fallback, require_disable_failed, small_cache_fallback,
+            small_cache_require_failed, short_tail_fallback,
+            short_tail_conflict_fallback,
+            (unsigned long long)candidate_delta,
+            (unsigned long long)call_delta,
+            (unsigned long long)failure_delta);
+    result_free(&result);
+    result_free(&tail_result);
+    return ok;
+}
+
+static int full_result_alloc(full_run_result *r) {
+    memset(r, 0, sizeof(*r));
+    r->pair_count =
+        (uint64_t)FULL_TOKENS * N_EXPERT * FULL_MID_DIM;
+    r->out_count = (uint64_t)FULL_TOKENS * FULL_OUT_DIM;
+    r->gate = calloc((size_t)r->pair_count, sizeof(float));
+    r->up = calloc((size_t)r->pair_count, sizeof(float));
+    r->mid = calloc((size_t)r->pair_count, sizeof(float));
+    r->out = calloc((size_t)r->out_count, sizeof(float));
+    return r->gate && r->up && r->mid && r->out;
+}
+
+static void full_result_free(full_run_result *r) {
+    free(r->gate);
+    free(r->up);
+    free(r->mid);
+    free(r->out);
+    memset(r, 0, sizeof(*r));
+}
+
+static uint32_t *full_guarded_payload(uint64_t payload_words) {
+    if (payload_words > (SIZE_MAX / sizeof(uint32_t)) - FULL_GUARD_WORDS) {
+        return NULL;
+    }
+    uint32_t *words = malloc(
+        (size_t)(payload_words + FULL_GUARD_WORDS) * sizeof(uint32_t));
+    if (!words) return NULL;
+
+    uint32_t sentinel_bits = 0;
+    const float sentinel = SENTINEL;
+    memcpy(&sentinel_bits, &sentinel, sizeof(sentinel_bits));
+    for (uint64_t i = 0; i < payload_words; i++) words[i] = sentinel_bits;
+    for (uint32_t i = 0; i < FULL_GUARD_WORDS; i++) {
+        words[payload_words + i] = FULL_GUARD_BITS;
+    }
+    return words;
+}
+
+static int full_check_guard(const char *run_name,
+                            const char *tensor_name,
+                            const ds4_gpu_tensor *tensor,
+                            uint64_t payload_words,
+                            uint64_t *mismatches) {
+    uint32_t words[FULL_GUARD_WORDS];
+    if (!ds4_gpu_tensor_read(tensor,
+                             payload_words * sizeof(uint32_t),
+                             words,
+                             sizeof(words))) {
+        fprintf(stderr,
+                "IQ2_XXS SSD grouped-MM %s %s guard readback FAIL\n",
+                run_name, tensor_name);
+        return 0;
+    }
+    uint64_t local = 0;
+    for (uint32_t i = 0; i < FULL_GUARD_WORDS; i++) {
+        if (words[i] != FULL_GUARD_BITS) local++;
+    }
+    *mismatches += local;
+    fprintf(stderr,
+            "IQ2_XXS SSD grouped-MM %-24s %-7s guard_%s=%llu\n",
+            run_name, tensor_name, local == 0 ? "PASS" : "FAIL",
+            (unsigned long long)local);
+    return local == 0;
+}
+
+static int run_full_once(
+        const char *run_name,
+        full_run_result *result,
+        const void *model,
+        uint64_t model_size,
+        uint64_t gate_offset,
+        uint64_t up_offset,
+        uint64_t down_offset,
+        uint64_t gate_expert_bytes,
+        uint64_t gate_row_bytes,
+        uint64_t down_expert_bytes,
+        uint64_t down_row_bytes,
+        const float *x,
+        const int32_t *selected,
+        const float *weights) {
+    const uint64_t x_count = (uint64_t)FULL_TOKENS * FULL_IN_DIM;
+    const uint64_t route_count = (uint64_t)FULL_TOKENS * N_EXPERT;
+    const uint64_t expert_count =
+        (uint64_t)FULL_TOKENS * N_EXPERT * FULL_OUT_DIM;
+    const uint64_t pair_bytes = result->pair_count * sizeof(float);
+    const uint64_t expert_bytes = expert_count * sizeof(float);
+    const uint64_t out_bytes = result->out_count * sizeof(float);
+    const uint64_t guard_bytes =
+        (uint64_t)FULL_GUARD_WORDS * sizeof(uint32_t);
+
+    uint32_t *pair_init = full_guarded_payload(result->pair_count);
+    uint32_t *expert_init = full_guarded_payload(expert_count);
+    uint32_t *out_init = full_guarded_payload(result->out_count);
+    ds4_gpu_tensor *x_t = NULL;
+    ds4_gpu_tensor *selected_t = NULL;
+    ds4_gpu_tensor *weights_t = NULL;
+    ds4_gpu_tensor *gate_t = NULL;
+    ds4_gpu_tensor *up_t = NULL;
+    ds4_gpu_tensor *mid_t = NULL;
+    ds4_gpu_tensor *experts_t = NULL;
+    ds4_gpu_tensor *out_t = NULL;
+    int ok = pair_init && expert_init && out_init;
+
+    if (ok) x_t = ds4_gpu_tensor_alloc(x_count * sizeof(float));
+    if (ok) selected_t =
+        ds4_gpu_tensor_alloc(route_count * sizeof(int32_t));
+    if (ok) weights_t = ds4_gpu_tensor_alloc(route_count * sizeof(float));
+    if (ok) gate_t = ds4_gpu_tensor_alloc(pair_bytes + guard_bytes);
+    if (ok) up_t = ds4_gpu_tensor_alloc(pair_bytes + guard_bytes);
+    if (ok) mid_t = ds4_gpu_tensor_alloc(pair_bytes + guard_bytes);
+    if (ok) experts_t = ds4_gpu_tensor_alloc(expert_bytes + guard_bytes);
+    if (ok) out_t = ds4_gpu_tensor_alloc(out_bytes + guard_bytes);
+    ok = ok && x_t && selected_t && weights_t && gate_t && up_t && mid_t &&
+         experts_t && out_t;
+    ok = ok && ds4_gpu_tensor_write(x_t, 0, x,
+                                     x_count * sizeof(float));
+    ok = ok && ds4_gpu_tensor_write(selected_t, 0, selected,
+                                     route_count * sizeof(int32_t));
+    ok = ok && ds4_gpu_tensor_write(weights_t, 0, weights,
+                                     route_count * sizeof(float));
+    ok = ok && ds4_gpu_tensor_write(gate_t, 0, pair_init,
+                                     pair_bytes + guard_bytes);
+    ok = ok && ds4_gpu_tensor_write(up_t, 0, pair_init,
+                                     pair_bytes + guard_bytes);
+    ok = ok && ds4_gpu_tensor_write(mid_t, 0, pair_init,
+                                     pair_bytes + guard_bytes);
+    ok = ok && ds4_gpu_tensor_write(experts_t, 0, expert_init,
+                                     expert_bytes + guard_bytes);
+    ok = ok && ds4_gpu_tensor_write(out_t, 0, out_init,
+                                     out_bytes + guard_bytes);
+
+    bool mid_is_f16 = true;
+    if (ok) {
+        ok = ds4_gpu_routed_moe_batch_tensor(
+            out_t, gate_t, up_t, mid_t, experts_t, model, model_size,
+            gate_offset, up_offset, down_offset, IQ2_XXS_TYPE, Q2_K_TYPE,
+            gate_expert_bytes, gate_row_bytes, down_expert_bytes,
+            down_row_bytes, FULL_IN_DIM, FULL_MID_DIM, FULL_OUT_DIM,
+            selected_t, weights_t, N_TOTAL_EXPERT, N_EXPERT, CLAMP, x_t,
+            0u, FULL_TOKENS, &mid_is_f16, false);
+    }
+    if (ok && mid_is_f16) {
+        fprintf(stderr,
+                "IQ2_XXS SSD full-shape %s unexpectedly selected f16 mid\n",
+                run_name);
+        ok = 0;
+    }
+    if (ok) {
+        ok = ds4_gpu_tensor_read(gate_t, 0, result->gate, pair_bytes) &&
+             ds4_gpu_tensor_read(up_t, 0, result->up, pair_bytes) &&
+             ds4_gpu_tensor_read(mid_t, 0, result->mid, pair_bytes) &&
+             ds4_gpu_tensor_read(out_t, 0, result->out, out_bytes);
+    }
+
+    result->guard_mismatches = 0;
+    if (gate_t) {
+        const int guard_ok = full_check_guard(
+            run_name, "gate", gate_t, result->pair_count,
+            &result->guard_mismatches);
+        ok = guard_ok && ok;
+    }
+    if (up_t) {
+        const int guard_ok = full_check_guard(
+            run_name, "up", up_t, result->pair_count,
+            &result->guard_mismatches);
+        ok = guard_ok && ok;
+    }
+    if (mid_t) {
+        const int guard_ok = full_check_guard(
+            run_name, "mid", mid_t, result->pair_count,
+            &result->guard_mismatches);
+        ok = guard_ok && ok;
+    }
+    if (experts_t) {
+        const int guard_ok = full_check_guard(
+            run_name, "experts", experts_t, expert_count,
+            &result->guard_mismatches);
+        ok = guard_ok && ok;
+    }
+    if (out_t) {
+        const int guard_ok = full_check_guard(
+            run_name, "out", out_t, result->out_count,
+            &result->guard_mismatches);
+        ok = guard_ok && ok;
+    }
+
+    ds4_gpu_tensor_free(x_t);
+    ds4_gpu_tensor_free(selected_t);
+    ds4_gpu_tensor_free(weights_t);
+    ds4_gpu_tensor_free(gate_t);
+    ds4_gpu_tensor_free(up_t);
+    ds4_gpu_tensor_free(mid_t);
+    ds4_gpu_tensor_free(experts_t);
+    ds4_gpu_tensor_free(out_t);
+    free(pair_init);
+    free(expert_init);
+    free(out_init);
+    return ok;
+}
+
+static int compare_full_results(const full_run_result *candidate,
+                                const full_run_result *control) {
+    if (candidate->pair_count != control->pair_count ||
+        candidate->out_count != control->out_count ||
+        candidate->guard_mismatches != 0 ||
+        control->guard_mismatches != 0) {
+        return 0;
+    }
+
+    /* The first M1 production-shape run measured max errors below 0.0036 for
+     * every tensor (gate/up RMS below 0.00057, mid below 0.00018, out below
+     * 0.0016).  The absolute bounds below retain roughly 3x max headroom and
+     * 2.5x-or-better RMS headroom for compiler/device variation while still
+     * rejecting a material change in accumulation or intermediate precision. */
+    int ok = compare_array("full-4096", "gate", candidate->gate,
+                           control->gate, candidate->pair_count,
+                           0.012, 0.002);
+    ok = compare_array("full-4096", "up", candidate->up,
+                       control->up, candidate->pair_count,
+                       0.012, 0.002) && ok;
+    ok = compare_array("full-4096", "mid", candidate->mid,
+                       control->mid, candidate->pair_count,
+                       0.012, 0.001) && ok;
+    ok = compare_array("full-4096", "out", candidate->out,
+                       control->out, candidate->out_count,
+                       0.012, 0.004) && ok;
+    return ok;
+}
+
+static int run_full_pair(
+        const void *model,
+        uint64_t model_size,
+        uint64_t gate_offset,
+        uint64_t up_offset,
+        uint64_t down_offset,
+        uint64_t gate_expert_bytes,
+        uint64_t gate_row_bytes,
+        uint64_t down_expert_bytes,
+        uint64_t down_row_bytes,
+        const float *x,
+        const int32_t *selected,
+        const float *weights) {
+    full_run_result control;
+    full_run_result candidate;
+    memset(&control, 0, sizeof(control));
+    memset(&candidate, 0, sizeof(candidate));
+    if (!full_result_alloc(&control) || !full_result_alloc(&candidate)) {
+        full_result_free(&control);
+        full_result_free(&candidate);
+        return 0;
+    }
+
+    unsetenv("DS4_METAL_ENABLE_IQ2_XXS_SSD_PREFILL_MM");
+    unsetenv("DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM");
+    setenv("DS4_METAL_DISABLE_IQ2_XXS_SSD_PREFILL_MM", "1", 1);
+    ds4_gpu_set_streaming_expert_cache_budget(N_TOTAL_EXPERT);
+    const int control_ok = run_full_once(
+        "control", &control, model, model_size, gate_offset, up_offset,
+        down_offset, gate_expert_bytes, gate_row_bytes, down_expert_bytes,
+        down_row_bytes, x, selected, weights);
+
+    unsetenv("DS4_METAL_DISABLE_IQ2_XXS_SSD_PREFILL_MM");
+    setenv("DS4_METAL_ENABLE_IQ2_XXS_SSD_PREFILL_MM", "1", 1);
+    setenv("DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM", "1", 1);
+    ds4_gpu_set_streaming_expert_cache_budget(N_TOTAL_EXPERT);
+    const int candidate_ok = run_full_once(
+        "candidate", &candidate, model, model_size, gate_offset, up_offset,
+        down_offset, gate_expert_bytes, gate_row_bytes, down_expert_bytes,
+        down_row_bytes, x, selected, weights);
+
+    int ok = control_ok && candidate_ok;
+    if (control_ok && candidate_ok) {
+        ok = compare_full_results(&candidate, &control);
+    }
+    full_result_free(&control);
+    full_result_free(&candidate);
+    return ok;
+}
+
+static int run_full_shape_oracle(const void *restore_model,
+                                 uint64_t restore_model_size,
+                                 int restore_model_fd,
+                                 uint64_t restore_cache_expert_bytes) {
+    const uint64_t page = (uint64_t)getpagesize();
+    const uint64_t gate_row_bytes =
+        (FULL_IN_DIM / QK_K) * sizeof(block_iq2_xxs);
+    const uint64_t gate_expert_bytes =
+        (uint64_t)FULL_MID_DIM * gate_row_bytes;
+    const uint64_t gate_tensor_bytes =
+        (uint64_t)N_TOTAL_EXPERT * gate_expert_bytes;
+    const uint64_t down_row_bytes =
+        (FULL_MID_DIM / QK_K) * sizeof(block_q2_K);
+    const uint64_t down_expert_bytes =
+        (uint64_t)FULL_OUT_DIM * down_row_bytes;
+    const uint64_t down_tensor_bytes =
+        (uint64_t)N_TOTAL_EXPERT * down_expert_bytes;
+    const uint64_t gate_offset = 0;
+    const uint64_t up_offset = align_up(gate_tensor_bytes, page);
+    const uint64_t down_offset =
+        align_up(up_offset + gate_tensor_bytes, page);
+    const uint64_t full_model_size =
+        align_up(down_offset + down_tensor_bytes, page);
+
+    if (gate_row_bytes != 1056u || gate_expert_bytes != 2162688u ||
+        down_row_bytes != 672u || down_expert_bytes != 2752512u) {
+        fprintf(stderr,
+                "IQ2_XXS SSD full-shape layout FAIL gate_row=%llu "
+                "gate_expert=%llu down_row=%llu down_expert=%llu\n",
+                (unsigned long long)gate_row_bytes,
+                (unsigned long long)gate_expert_bytes,
+                (unsigned long long)down_row_bytes,
+                (unsigned long long)down_expert_bytes);
+        return 0;
+    }
+    fprintf(stderr,
+            "IQ2_XXS SSD full-shape layout PASS in=%u mid=%u out=%u "
+            "tokens=%u topk=%u experts=%u gate_row=%llu down_row=%llu "
+            "model_bytes=%llu\n",
+            FULL_IN_DIM, FULL_MID_DIM, FULL_OUT_DIM, FULL_TOKENS,
+            N_EXPERT, N_TOTAL_EXPERT,
+            (unsigned long long)gate_row_bytes,
+            (unsigned long long)down_row_bytes,
+            (unsigned long long)full_model_size);
+
+    int ok = 1;
+    int backend_switched = 0;
+    void *full_model = NULL;
+    float *x = NULL;
+    int32_t *selected = NULL;
+    float *weights = NULL;
+    char tmp_path[] = "/tmp/ds4-iq2-ssd-full-mm.XXXXXX";
+    int full_fd = -1;
+
+    if (posix_memalign(&full_model, (size_t)page,
+                       (size_t)full_model_size) != 0) {
+        return 0;
+    }
+    memset(full_model, 0, (size_t)full_model_size);
+    block_iq2_xxs *gate =
+        (block_iq2_xxs *)((uint8_t *)full_model + gate_offset);
+    block_iq2_xxs *up =
+        (block_iq2_xxs *)((uint8_t *)full_model + up_offset);
+    block_q2_K *down =
+        (block_q2_K *)((uint8_t *)full_model + down_offset);
+    fill_iq2_full(gate, 19u);
+    fill_iq2_full(up, 47u);
+    fill_q2_full(down);
+
+    const uint64_t x_count = (uint64_t)FULL_TOKENS * FULL_IN_DIM;
+    const uint64_t route_count = (uint64_t)FULL_TOKENS * N_EXPERT;
+    x = calloc((size_t)x_count, sizeof(float));
+    selected = calloc((size_t)route_count, sizeof(int32_t));
+    weights = calloc((size_t)route_count, sizeof(float));
+    if (!x || !selected || !weights) {
+        ok = 0;
+        goto cleanup;
+    }
+    for (uint32_t token = 0; token < FULL_TOKENS; token++) {
+        for (uint32_t k = 0; k < FULL_IN_DIM; k++) {
+            const int32_t v =
+                (int32_t)((token * 43u + k * 29u +
+                           (k >> 3u) * 17u) % 251u) - 125;
+            x[(uint64_t)token * FULL_IN_DIM + k] = (float)v / 256.0f;
+        }
+        for (uint32_t slot = 0; slot < N_EXPERT; slot++) {
+            const uint64_t route = (uint64_t)token * N_EXPERT + slot;
+            selected[route] =
+                (int32_t)((token * 3u + slot) % N_TOTAL_EXPERT);
+            weights[route] = (float)(slot + 1u) / 21.0f;
+        }
+    }
+
+    full_fd = mkstemp(tmp_path);
+    if (full_fd < 0 || ftruncate(full_fd, (off_t)full_model_size) != 0) {
+        fprintf(stderr, "IQ2_XXS SSD full-shape fixture creation FAIL\n");
+        ok = 0;
+        goto cleanup;
+    }
+    uint64_t written = 0;
+    while (written < full_model_size) {
+        const size_t chunk = full_model_size - written > (1u << 20) ?
+            (1u << 20) : (size_t)(full_model_size - written);
+        const ssize_t n = pwrite(full_fd,
+                                 (const uint8_t *)full_model + written,
+                                 chunk,
+                                 (off_t)written);
+        if (n <= 0) {
+            fprintf(stderr, "IQ2_XXS SSD full-shape fixture write FAIL\n");
+            ok = 0;
+            goto cleanup;
+        }
+        written += (uint64_t)n;
+    }
+
+    /* The streaming cache deliberately freezes one expert-size slab class.
+     * Re-seed it for the production 6.75 MiB expert after clearing the compact
+     * fixture, then restore the compact class before returning. */
+    ds4_gpu_set_streaming_expert_cache_budget(N_TOTAL_EXPERT);
+    ds4_gpu_set_streaming_expert_cache_expert_bytes(
+        2u * gate_expert_bytes + down_expert_bytes);
+    if (!ds4_gpu_set_model_fd(full_fd)) {
+        ok = 0;
+        goto cleanup;
+    }
+    backend_switched = 1;
+    if (!ds4_gpu_set_model_map(full_model, full_model_size)) {
+        ok = 0;
+        goto cleanup;
+    }
+
+    uint64_t before_candidates = 0;
+    uint64_t before_calls = 0;
+    uint64_t before_tokens = 0;
+    uint64_t before_rows = 0;
+    uint64_t before_failures = 0;
+    uint32_t before_min = 0;
+    uint32_t before_max = 0;
+    if (!ds4_gpu_test_iq2_stream_addr_mm_stats(
+            &before_candidates, &before_calls, &before_tokens, &before_rows,
+            &before_failures, &before_min, &before_max)) {
+        ok = 0;
+        goto cleanup;
+    }
+
+    const int pair_ok = run_full_pair(
+        full_model, full_model_size, gate_offset, up_offset, down_offset,
+        gate_expert_bytes, gate_row_bytes, down_expert_bytes, down_row_bytes,
+        x, selected, weights);
+
+    uint64_t after_candidates = 0;
+    uint64_t after_calls = 0;
+    uint64_t after_tokens = 0;
+    uint64_t after_rows = 0;
+    uint64_t after_failures = 0;
+    uint32_t after_min = 0;
+    uint32_t after_max = 0;
+    const int stats_ok = ds4_gpu_test_iq2_stream_addr_mm_stats(
+        &after_candidates, &after_calls, &after_tokens, &after_rows,
+        &after_failures, &after_min, &after_max);
+    const int monotonic = stats_ok &&
+        after_candidates >= before_candidates &&
+        after_calls >= before_calls &&
+        after_tokens >= before_tokens &&
+        after_rows >= before_rows &&
+        after_failures >= before_failures;
+    const uint64_t delta_candidates = monotonic ?
+        after_candidates - before_candidates : UINT64_MAX;
+    const uint64_t delta_calls = monotonic ?
+        after_calls - before_calls : UINT64_MAX;
+    const uint64_t delta_tokens = monotonic ?
+        after_tokens - before_tokens : UINT64_MAX;
+    const uint64_t delta_rows = monotonic ?
+        after_rows - before_rows : UINT64_MAX;
+    const uint64_t delta_failures = monotonic ?
+        after_failures - before_failures : UINT64_MAX;
+    const int coverage_ok = monotonic &&
+        delta_candidates == 2u &&
+        delta_calls == 1u &&
+        delta_tokens == FULL_TOKENS &&
+        delta_rows == (uint64_t)FULL_TOKENS * N_EXPERT &&
+        delta_failures == 0u;
+    fprintf(stderr,
+            "IQ2_XXS SSD full-shape coverage %s candidates=%llu calls=%llu "
+            "tokens=%llu rows=%llu require_failures=%llu "
+            "global_min=%u->%u global_max=%u->%u\n",
+            coverage_ok ? "PASS" : "FAIL",
+            (unsigned long long)delta_candidates,
+            (unsigned long long)delta_calls,
+            (unsigned long long)delta_tokens,
+            (unsigned long long)delta_rows,
+            (unsigned long long)delta_failures,
+            before_min, after_min, before_max, after_max);
+    ok = pair_ok && coverage_ok && ok;
+
+cleanup:
+    unsetenv("DS4_METAL_ENABLE_IQ2_XXS_SSD_PREFILL_MM");
+    unsetenv("DS4_METAL_REQUIRE_IQ2_XXS_SSD_PREFILL_MM");
+    unsetenv("DS4_METAL_DISABLE_IQ2_XXS_SSD_PREFILL_MM");
+    if (backend_switched) {
+        ds4_gpu_set_streaming_expert_cache_budget(N_TOTAL_EXPERT);
+        ds4_gpu_set_streaming_expert_cache_expert_bytes(
+            restore_cache_expert_bytes);
+        const int fd_ok = ds4_gpu_set_model_fd(restore_model_fd);
+        const int map_ok = ds4_gpu_set_model_map(
+            restore_model, restore_model_size);
+        if (!fd_ok || !map_ok) {
+            fprintf(stderr,
+                    "IQ2_XXS SSD full-shape backend restoration FAIL\n");
+            ok = 0;
+        }
+    }
+    if (full_fd >= 0) close(full_fd);
+    if (full_fd >= 0) unlink(tmp_path);
+    free(x);
+    free(selected);
+    free(weights);
+    free(full_model);
+    fprintf(stderr,
+            "IQ2_XXS/Q2_K Metal SSD full production-shape oracle %s\n",
+            ok ? "PASS" : "FAIL");
+    return ok;
+}
+
+static int run_256_expert_oracle(
+        const void *restore_model,
+        uint64_t restore_model_size,
+        int restore_model_fd,
+        uint64_t restore_gate_expert_bytes,
+        uint64_t restore_gate_row_bytes,
+        uint64_t restore_down_expert_bytes,
+        uint64_t restore_down_row_bytes) {
+    const uint64_t page = (uint64_t)getpagesize();
+    const uint64_t gate_row_bytes = sizeof(block_iq2_xxs);
+    const uint64_t gate_expert_bytes = MID_DIM * gate_row_bytes;
+    const uint64_t gate_tensor_bytes =
+        (uint64_t)N_TOTAL_EXPERT_256 * gate_expert_bytes;
+    const uint64_t down_row_bytes = sizeof(block_q2_K);
+    const uint64_t down_expert_bytes = OUT_DIM * down_row_bytes;
+    const uint64_t down_tensor_bytes =
+        (uint64_t)N_TOTAL_EXPERT_256 * down_expert_bytes;
+    const uint64_t gate_offset = 0;
+    const uint64_t up_offset = align_up(gate_tensor_bytes, page);
+    const uint64_t down_offset =
+        align_up(up_offset + gate_tensor_bytes, page);
+    const uint64_t model_size =
+        align_up(down_offset + down_tensor_bytes, page);
+
+    int ok = 1;
+    int backend_switched = 0;
+    void *model = NULL;
+    float *x = NULL;
+    int32_t *selected = NULL;
+    float *weights = NULL;
+    char tmp_path[] = "/tmp/ds4-iq2-ssd-256-mm.XXXXXX";
+    int model_fd = -1;
+
+    if (posix_memalign(&model, (size_t)page, (size_t)model_size) != 0) {
+        return 0;
+    }
+    memset(model, 0, (size_t)model_size);
+    block_iq2_xxs *gate =
+        (block_iq2_xxs *)((uint8_t *)model + gate_offset);
+    block_iq2_xxs *up =
+        (block_iq2_xxs *)((uint8_t *)model + up_offset);
+    block_q2_K *down =
+        (block_q2_K *)((uint8_t *)model + down_offset);
+    fill_iq2(gate, 23u, N_TOTAL_EXPERT_256);
+    fill_iq2(up, 53u, N_TOTAL_EXPERT_256);
+    fill_q2(down, N_TOTAL_EXPERT_256);
+
+    const uint32_t tokens = 33u;
+    const uint64_t x_count = (uint64_t)tokens * IN_DIM;
+    const uint64_t route_count = (uint64_t)tokens * N_EXPERT;
+    x = calloc((size_t)x_count, sizeof(float));
+    selected = calloc((size_t)route_count, sizeof(int32_t));
+    weights = calloc((size_t)route_count, sizeof(float));
+    if (!x || !selected || !weights) {
+        ok = 0;
+        goto cleanup;
+    }
+    for (uint32_t token = 0; token < tokens; token++) {
+        for (uint32_t k = 0; k < IN_DIM; k++) {
+            const int32_t v =
+                (int32_t)((token * 31u + k * 17u + (k >> 2u) * 7u) %
+                          127u) - 63;
+            x[(uint64_t)token * IN_DIM + k] = (float)v / 96.0f;
+        }
+        for (uint32_t slot = 0; slot < N_EXPERT; slot++) {
+            const uint64_t route = (uint64_t)token * N_EXPERT + slot;
+            if (slot == 0u) {
+                /* Exactly 33 routes: one full tile plus a one-row tail. */
+                selected[route] = (int32_t)HIGH_EXPERT_ID;
+            } else if (slot == 1u && token < 15u) {
+                selected[route] = 0;
+            } else if (slot == 2u && token == 0u) {
+                /* Duplicate expert zero within token zero. Together with the
+                 * 15 routes above this makes the critical nr1 == 16 tile. */
+                selected[route] = 0;
+            } else if (slot == 1u && token < 32u) {
+                /* Exactly 17 routes exercise the first non-culled row half. */
+                selected[route] = 1;
+            } else {
+                /* Keep all remaining routes away from 0, 1, and 255 so the
+                 * three boundary counts remain exact. */
+                selected[route] =
+                    (int32_t)(2u + (token * 17u + slot * 29u) % 252u);
+            }
+            weights[route] = (float)(slot + 1u) / 21.0f;
+        }
+    }
+    uint32_t routes_16 = 0;
+    uint32_t routes_17 = 0;
+    uint32_t high_id_routes = 0;
+    uint32_t duplicate_routes = 0;
+    for (uint32_t token = 0; token < tokens; token++) {
+        for (uint32_t slot = 0; slot < N_EXPERT; slot++) {
+            const int32_t id = selected[(uint64_t)token * N_EXPERT + slot];
+            if (id == 0) routes_16++;
+            if (id == 1) routes_17++;
+            if (id == (int32_t)HIGH_EXPERT_ID) high_id_routes++;
+            for (uint32_t prior = 0; prior < slot; prior++) {
+                if (id == selected[(uint64_t)token * N_EXPERT + prior]) {
+                    duplicate_routes++;
+                    break;
+                }
+            }
+        }
+    }
+    const uint32_t high_id_work_items = (high_id_routes + 31u) / 32u;
+    if (routes_16 != 16u || routes_17 != 17u || high_id_routes != 33u ||
+        duplicate_routes == 0u || high_id_work_items != 2u) {
+        fprintf(stderr,
+                "IQ2_XXS SSD tail-cull route construction FAIL "
+                "rows16=%u rows17=%u id255_rows=%u duplicates=%u "
+                "work_items=%u\n",
+                routes_16, routes_17, high_id_routes, duplicate_routes,
+                high_id_work_items);
+        ok = 0;
+        goto cleanup;
+    }
+    fprintf(stderr,
+            "IQ2_XXS SSD tail-cull routes PASS model_bytes=%llu "
+            "rows16=%u rows17=%u id255_rows=%u duplicates=%u "
+            "work_items=%u second_tile_r1=32\n",
+            (unsigned long long)model_size, routes_16, routes_17,
+            high_id_routes, duplicate_routes,
+            high_id_work_items);
+
+    model_fd = mkstemp(tmp_path);
+    if (model_fd < 0 || ftruncate(model_fd, (off_t)model_size) != 0) {
+        ok = 0;
+        goto cleanup;
+    }
+    uint64_t written = 0;
+    while (written < model_size) {
+        const size_t chunk = model_size - written > (1u << 20) ?
+            (1u << 20) : (size_t)(model_size - written);
+        const ssize_t n = pwrite(model_fd,
+                                 (const uint8_t *)model + written,
+                                 chunk,
+                                 (off_t)written);
+        if (n <= 0) {
+            ok = 0;
+            goto cleanup;
+        }
+        written += (uint64_t)n;
+    }
+
+    ds4_gpu_set_streaming_expert_cache_budget(N_TOTAL_EXPERT_256);
+    ds4_gpu_set_streaming_expert_cache_expert_bytes(
+        2u * gate_expert_bytes + down_expert_bytes);
+    if (!ds4_gpu_set_model_fd(model_fd)) {
+        ok = 0;
+        goto cleanup;
+    }
+    backend_switched = 1;
+    if (!ds4_gpu_set_model_map(model, model_size)) {
+        ok = 0;
+        goto cleanup;
+    }
+
+    mm_stats_snapshot before;
+    mm_stats_snapshot after;
+    if (!read_mm_stats(&before)) {
+        ok = 0;
+        goto cleanup;
+    }
+    const int pair_ok = run_pair(
+        "id255-hot-33", tokens, model, model_size,
+        gate_offset, up_offset, down_offset,
+        gate_expert_bytes, gate_row_bytes,
+        down_expert_bytes, down_row_bytes,
+        N_TOTAL_EXPERT_256, N_TOTAL_EXPERT_256,
+        x, selected, weights);
+    const int stats_ok = read_mm_stats(&after) &&
+        check_mm_stats_delta("id255-hot-33", &before, &after, tokens);
+    const int tail_cull_ok = run_tail_cull_bit_exact(
+        model, model_size, gate_offset, up_offset, down_offset,
+        gate_expert_bytes, gate_row_bytes, down_expert_bytes,
+        down_row_bytes, N_TOTAL_EXPERT_256, N_TOTAL_EXPERT_256,
+        x, selected, weights);
+    ok = pair_ok && stats_ok && tail_cull_ok && ok;
+
+cleanup:
+    clear_tail_cull_test_env();
+    if (backend_switched) {
+        ds4_gpu_set_streaming_expert_cache_budget(N_TOTAL_EXPERT);
+        ds4_gpu_set_streaming_expert_cache_expert_bytes(
+            2u * restore_gate_expert_bytes + restore_down_expert_bytes);
+        const int fd_ok = ds4_gpu_set_model_fd(restore_model_fd);
+        const int map_ok = ds4_gpu_set_model_map(
+            restore_model, restore_model_size);
+        if (!fd_ok || !map_ok ||
+            restore_gate_row_bytes != sizeof(block_iq2_xxs) ||
+            restore_down_row_bytes != sizeof(block_q2_K)) {
+            fprintf(stderr,
+                    "IQ2_XXS SSD 256-expert backend restoration FAIL\n");
+            ok = 0;
+        }
+    }
+    if (model_fd >= 0) close(model_fd);
+    if (model_fd >= 0) unlink(tmp_path);
+    free(x);
+    free(selected);
+    free(weights);
+    free(model);
+    fprintf(stderr,
+            "IQ2_XXS/Q2_K Metal compact 256-expert ID255 oracle %s\n",
+            ok ? "PASS" : "FAIL");
+    return ok;
+}
+
+int main(void) {
+    if (sizeof(block_iq2_xxs) != 66u || sizeof(block_q2_K) != 84u) {
+        fprintf(stderr,
+                "IQ2_XXS SSD grouped-MM unexpected block sizes iq2=%zu q2=%zu\n",
+                sizeof(block_iq2_xxs), sizeof(block_q2_K));
+        return 1;
+    }
+
+    int ok = check_grouped_mm_policy();
+
+    const uint64_t gate_row_bytes = sizeof(block_iq2_xxs);
+    const uint64_t gate_expert_bytes = MID_DIM * gate_row_bytes;
+    const uint64_t gate_tensor_bytes = N_TOTAL_EXPERT * gate_expert_bytes;
+    const uint64_t down_row_bytes = sizeof(block_q2_K);
+    const uint64_t down_expert_bytes = OUT_DIM * down_row_bytes;
+    const uint64_t down_tensor_bytes = N_TOTAL_EXPERT * down_expert_bytes;
+    const uint64_t page = (uint64_t)getpagesize();
+    const uint64_t gate_offset = 0;
+    const uint64_t up_offset = align_up(gate_tensor_bytes, page);
+    const uint64_t down_offset = align_up(up_offset + gate_tensor_bytes, page);
+    const uint64_t model_size = align_up(down_offset + down_tensor_bytes, page);
+
+    void *model = NULL;
+    if (posix_memalign(&model, (size_t)page, (size_t)model_size) != 0) {
+        return 1;
+    }
+    memset(model, 0, (size_t)model_size);
+    block_iq2_xxs *gate =
+        (block_iq2_xxs *)((uint8_t *)model + gate_offset);
+    block_iq2_xxs *up =
+        (block_iq2_xxs *)((uint8_t *)model + up_offset);
+    block_q2_K *down = (block_q2_K *)((uint8_t *)model + down_offset);
+    fill_iq2(gate, 3u, N_TOTAL_EXPERT);
+    fill_iq2(up, 11u, N_TOTAL_EXPERT);
+    fill_q2(down, N_TOTAL_EXPERT);
+
+    float *x = calloc((size_t)MAX_TOKENS * IN_DIM, sizeof(float));
+    int32_t *selected =
+        calloc((size_t)MAX_TOKENS * N_EXPERT, sizeof(int32_t));
+    int32_t *selected_duplicate =
+        calloc((size_t)MAX_TOKENS * N_EXPERT, sizeof(int32_t));
+    int32_t *selected_hot =
+        calloc((size_t)MAX_TOKENS * N_EXPERT, sizeof(int32_t));
+    float *weights =
+        calloc((size_t)MAX_TOKENS * N_EXPERT, sizeof(float));
+    ok = x && selected && selected_duplicate && selected_hot && weights && ok;
+    for (uint32_t token = 0; ok && token < MAX_TOKENS; token++) {
+        for (uint32_t k = 0; k < IN_DIM; k++) {
+            const int32_t v = (int32_t)((token * 17u + k * 11u +
+                                         (token ^ k) * 3u) % 97u) - 48;
+            x[(uint64_t)token * IN_DIM + k] = (float)v / 64.0f;
+        }
+        for (uint32_t slot = 0; slot < N_EXPERT; slot++) {
+            const uint64_t route = (uint64_t)token * N_EXPERT + slot;
+            selected[route] = (int32_t)((token + slot) % N_TOTAL_EXPERT);
+            selected_duplicate[route] = selected[route];
+            /* Expert zero receives exactly one route per token. At 33 tokens
+             * this forces work items at r1=0 and r1=32 without relying on
+             * duplicate IDs inside one token. */
+            selected_hot[route] = slot == 0u ? 0 :
+                (int32_t)(1u + ((token + slot - 1u) %
+                                (N_TOTAL_EXPERT - 1u)));
+            weights[route] = 1.0f / N_EXPERT;
+        }
+        selected_duplicate[(uint64_t)token * N_EXPERT + N_EXPERT - 1u] =
+            selected_duplicate[(uint64_t)token * N_EXPERT];
+    }
+
+    char tmp_path[] = "/tmp/ds4-iq2-ssd-grouped-mm.XXXXXX";
+    int model_fd = ok ? mkstemp(tmp_path) : -1;
+    if (model_fd < 0 || ftruncate(model_fd, (off_t)model_size) != 0) {
+        fprintf(stderr, "IQ2_XXS SSD grouped-MM could not create fixture\n");
+        ok = 0;
+    }
+    uint64_t written = 0;
+    while (ok && written < model_size) {
+        const size_t chunk = model_size - written > (1u << 20) ?
+            (1u << 20) : (size_t)(model_size - written);
+        const ssize_t n = pwrite(model_fd, (const uint8_t *)model + written,
+                                 chunk, (off_t)written);
+        if (n <= 0) {
+            fprintf(stderr, "IQ2_XXS SSD grouped-MM fixture write failed\n");
+            ok = 0;
+        } else {
+            written += (uint64_t)n;
+        }
+    }
+
+    unsetenv("DS4_METAL_DISABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR");
+    unsetenv("DS4_METAL_DISABLE_STREAMING_EXPERT_ADDR_TABLE");
+    unsetenv("DS4_METAL_DISABLE_ROUTED_PAIR_SWIGLU_FUSION");
+    unsetenv("DS4_METAL_MOE_WRITE_CLAMPED_ACT");
+    unsetenv("DS4_METAL_GRAPH_DUMP_PREFIX");
+    clear_tail_cull_test_env();
+    setenv("DS4_METAL_ENABLE_STREAMING_PREFILL_BATCH_SELECTED_ADDR", "1", 1);
+    setenv("DS4_METAL_ENABLE_STREAMING_EXPERT_ADDR_TABLE", "1", 1);
+    setenv("DS4_METAL_IQ2_XXS_SSD_PREFILL_MM_STATS", "1", 1);
+
+    ok = ok && ds4_gpu_init() && ds4_gpu_set_model_map(model, model_size);
+    ds4_gpu_set_quality(false);
+    ds4_gpu_set_ssd_streaming(true);
+    ds4_gpu_set_streaming_expert_cache_budget(N_TOTAL_EXPERT);
+    ok = ok && ds4_gpu_set_model_fd(model_fd);
+    const int backend_ready = ok;
+
+    if (backend_ready) {
+        const int case_ok = run_pair("tile-32", 32u, model, model_size,
+                                     gate_offset, up_offset, down_offset,
+                                     gate_expert_bytes, gate_row_bytes,
+                                     down_expert_bytes, down_row_bytes,
+                                     N_TOTAL_EXPERT, N_TOTAL_EXPERT,
+                                     x, selected, weights);
+        ok = case_ok && ok;
+    }
+    if (backend_ready) {
+        const int case_ok = run_pair("tail-33", 33u, model, model_size,
+                                     gate_offset, up_offset, down_offset,
+                                     gate_expert_bytes, gate_row_bytes,
+                                     down_expert_bytes, down_row_bytes,
+                                     N_TOTAL_EXPERT, N_TOTAL_EXPERT,
+                                     x, selected, weights);
+        ok = case_ok && ok;
+    }
+    if (backend_ready) {
+        const int case_ok = run_pair("duplicate-32", 32u, model, model_size,
+                                     gate_offset, up_offset, down_offset,
+                                     gate_expert_bytes, gate_row_bytes,
+                                     down_expert_bytes, down_row_bytes,
+                                     N_TOTAL_EXPERT, N_TOTAL_EXPERT,
+                                     x, selected_duplicate, weights);
+        ok = case_ok && ok;
+    }
+
+    uint64_t candidate_calls = 0;
+    uint64_t calls = 0;
+    uint64_t tokens = 0;
+    uint64_t rows = 0;
+    uint64_t require_failures = 0;
+    uint32_t min_tokens = 0;
+    uint32_t max_tokens = 0;
+    const int stats_ok = ds4_gpu_test_iq2_stream_addr_mm_stats(
+        &candidate_calls, &calls, &tokens, &rows, &require_failures,
+        &min_tokens, &max_tokens);
+    const int counters_ok =
+        candidate_calls == 6u && calls == 3u && tokens == 97u &&
+        rows == 582u && require_failures == 0u &&
+        min_tokens == 32u && max_tokens == 33u;
+    fprintf(stderr,
+            "IQ2_XXS SSD grouped-MM coverage %s candidates=%llu calls=%llu "
+            "tokens=%llu rows=%llu require_failures=%llu min=%u max=%u\n",
+            counters_ok ? "PASS" : "FAIL",
+            (unsigned long long)candidate_calls,
+            (unsigned long long)calls,
+            (unsigned long long)tokens,
+            (unsigned long long)rows,
+            (unsigned long long)require_failures,
+            min_tokens, max_tokens);
+    ok = stats_ok && counters_ok && ok;
+
+    if (backend_ready) {
+        const int policy_fault_ok = check_implicit_require_fault(
+            model, model_size, gate_offset, up_offset, down_offset,
+            gate_expert_bytes, gate_row_bytes, down_expert_bytes,
+            down_row_bytes, x, selected, weights);
+        ok = policy_fault_ok && ok;
+    }
+
+    if (backend_ready) {
+        uint32_t hot_rows = 0;
+        for (uint32_t token = 0; token < 33u; token++) {
+            for (uint32_t slot = 0; slot < N_EXPERT; slot++) {
+                if (selected_hot[(uint64_t)token * N_EXPERT + slot] == 0) {
+                    hot_rows++;
+                }
+            }
+        }
+        const uint32_t hot_work_items = (hot_rows + 31u) / 32u;
+        const int route_shape_ok = hot_rows >= 33u && hot_work_items >= 2u;
+        fprintf(stderr,
+                "IQ2_XXS SSD compact hot-route %s expert=0 rows=%u "
+                "work_items=%u second_tile_r1=32\n",
+                route_shape_ok ? "PASS" : "FAIL",
+                hot_rows, hot_work_items);
+        mm_stats_snapshot before;
+        mm_stats_snapshot after;
+        const int before_ok = read_mm_stats(&before);
+        const int pair_ok = route_shape_ok && run_pair(
+            "hot-r1-32", 33u, model, model_size,
+            gate_offset, up_offset, down_offset,
+            gate_expert_bytes, gate_row_bytes,
+            down_expert_bytes, down_row_bytes,
+            N_TOTAL_EXPERT, N_TOTAL_EXPERT,
+            x, selected_hot, weights);
+        const int delta_ok = before_ok && read_mm_stats(&after) &&
+            check_mm_stats_delta("hot-r1-32", &before, &after, 33u);
+        ok = pair_ok && delta_ok && ok;
+    }
+
+    if (backend_ready) {
+        const int id255_ok = run_256_expert_oracle(
+            model, model_size, model_fd,
+            gate_expert_bytes, gate_row_bytes,
+            down_expert_bytes, down_row_bytes);
+        ok = id255_ok && ok;
+    }
+
+    /* Keep the full production geometry as a distinct oracle and report its
+     * counter deltas independently from the compact 32/33 smoke cases. */
+    if (backend_ready) {
+        const int full_ok =
+            run_full_shape_oracle(
+                model, model_size, model_fd,
+                2u * gate_expert_bytes + down_expert_bytes);
+        ok = full_ok && ok;
+    }
+
+    clear_tail_cull_test_env();
+    ds4_gpu_set_model_fd(-1);
+    ds4_gpu_set_ssd_streaming(false);
+    ds4_gpu_cleanup();
+    if (model_fd >= 0) close(model_fd);
+    if (tmp_path[0]) unlink(tmp_path);
+    free(x);
+    free(selected);
+    free(selected_duplicate);
+    free(selected_hot);
+    free(weights);
+    free(model);
+
+    fprintf(stderr, "IQ2_XXS/Q2_K Metal SSD grouped address-MM %s\n",
+            ok ? "PASS" : "FAIL");
+    return ok ? 0 : 1;
+}
+
+#else
+
+int main(void) {
+    fprintf(stderr,
+            "test_metal_iq2_ssd_grouped_mm: skipped (Metal requires macOS)\n");
+    return 0;
+}
+
+#endif
diff --git a/tests/test_metal_q4_attn_exactn b/tests/test_metal_q4_attn_exactn
new file mode 100755
index 0000000000..871ef14d54
Binary files /dev/null and b/tests/test_metal_q4_attn_exactn differ
diff --git a/tests/test_metal_q4_attn_exactn.c b/tests/test_metal_q4_attn_exactn.c
new file mode 100644
index 0000000000..c176883bd1
--- /dev/null
+++ b/tests/test_metal_q4_attn_exactn.c
@@ -0,0 +1,740 @@
+#define _DARWIN_C_SOURCE
+
+/* Synthetic, GGUF-free bitwise oracle for the M1--M4 SSD-prefill Q4_K
+ * attention-output token-tiled kernel. */
+
+#include "ds4_gpu.h"
+
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+bool ds4_log_is_tty(FILE *fp) {
+    (void)fp;
+    return false;
+}
+
+#ifdef __APPLE__
+
+enum {
+    Q4_K_TYPE = 12u,
+    QK_K = 256u,
+    /* Multiblock geometry: attention-A spans 16 Q4_K blocks per row and
+     * output-B spans four, exercising every ix lane and repeated ib steps. */
+    GROUP_DIM = 4096u,
+    RANK = 512u,
+    N_GROUPS = 2u,
+    LOW_DIM = N_GROUPS * RANK,
+    OUT_DIM = 67u,
+    MAX_ROWS = 31u,
+    ALLOC_ROWS = 33u,
+};
+
+typedef struct {
+    uint16_t d;
+    uint16_t dmin;
+    uint8_t scales[12];
+    uint8_t qs[QK_K / 2u];
+} block_q4_K;
+
+static const char *k_enable =
+    "DS4_METAL_ENABLE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN";
+static const char *k_disable =
+    "DS4_METAL_DISABLE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN";
+static const char *k_require =
+    "DS4_METAL_REQUIRE_Q4_SSD_PREFILL_ATTN_OUT_EXACTN";
+static const char *k_disable_scale_meta =
+    "DS4_METAL_DISABLE_Q4_SSD_PREFILL_ATTN_OUT_SCALE_META";
+static const char *k_require_scale_meta =
+    "DS4_METAL_REQUIRE_Q4_SSD_PREFILL_ATTN_OUT_SCALE_META";
+static const char *k_disable_classic = "DS4_METAL_DISABLE_Q4_MV_CLASSIC";
+static const char *k_disable_f16_rhs =
+    "DS4_METAL_DISABLE_Q4_ATTN_OUT_B_F16_RHS";
+static const char *k_require_f16_rhs =
+    "DS4_METAL_REQUIRE_Q4_ATTN_OUT_B_F16_RHS";
+
+static void fail(const char *what) {
+    fprintf(stderr, "Metal Q4 SSD-prefill exact-N oracle FAIL: %s\n", what);
+    exit(1);
+}
+
+#define CHECK(expr, what) do { if (!(expr)) fail(what); } while (0)
+
+static uint64_t align_up(uint64_t value, uint64_t alignment) {
+    return (value + alignment - 1u) & ~(alignment - 1u);
+}
+
+static void pack_scales(uint8_t packed[12],
+                        const uint8_t scale[8],
+                        const uint8_t minimum[8]) {
+    memset(packed, 0, 12u);
+    for (uint32_t group = 0; group < 4u; group++) {
+        packed[group] = scale[group] & 63u;
+        packed[group + 4u] = minimum[group] & 63u;
+    }
+    for (uint32_t group = 4u; group < 8u; group++) {
+        packed[group + 4u] = (scale[group] & 15u) |
+                             ((minimum[group] & 15u) << 4u);
+        packed[group - 4u] |= (scale[group] >> 4u) << 6u;
+        packed[group] |= (minimum[group] >> 4u) << 6u;
+    }
+}
+
+static void fill_q4_matrix(void *raw,
+                           uint32_t in_dim,
+                           uint32_t rows,
+                           uint32_t salt) {
+    CHECK(sizeof(block_q4_K) == 144u, "unexpected Q4_K block size");
+    CHECK((in_dim % QK_K) == 0u, "unaligned Q4_K fixture");
+    const uint32_t blocks_per_row = in_dim / QK_K;
+    block_q4_K *matrix = raw;
+    for (uint32_t row = 0; row < rows; row++) {
+        for (uint32_t block = 0; block < blocks_per_row; block++) {
+            block_q4_K *b = matrix +
+                (uint64_t)row * blocks_per_row + block;
+            const uint32_t key =
+                salt + row * 1009u + block * 313u +
+                (row ^ (block * 17u));
+            uint8_t scale[8];
+            uint8_t minimum[8];
+            for (uint32_t group = 0; group < 8u; group++) {
+                scale[group] = (uint8_t)((key + group * 7u) % 64u);
+                minimum[group] =
+                    (uint8_t)((key / 3u + group * 5u) % 64u);
+            }
+            pack_scales(b->scales, scale, minimum);
+            for (uint32_t i = 0; i < QK_K / 2u; i++) {
+                b->qs[i] =
+                    (uint8_t)(key + i * 37u + (i >> 2u) * 11u);
+            }
+            /* Exact binary scales: 2^-5 and 2^-7. */
+            b->d = 0x2800u;
+            b->dmin = 0x2000u;
+        }
+    }
+}
+
+static void poison(float *values, uint64_t count, uint32_t base) {
+    for (uint64_t i = 0; i < count; i++) {
+        const uint32_t bits = base + (uint32_t)(i & 0xffffu);
+        memcpy(&values[i], &bits, sizeof(bits));
+    }
+}
+
+static uint64_t count_bit_mismatches(const float *reference,
+                                     const float *actual,
+                                     uint64_t count,
+                                     uint64_t *first) {
+    uint64_t mismatches = 0;
+    *first = UINT64_MAX;
+    for (uint64_t i = 0; i < count; i++) {
+        if (memcmp(&reference[i], &actual[i], sizeof(float)) != 0) {
+            if (*first == UINT64_MAX) *first = i;
+            mismatches++;
+        }
+    }
+    return mismatches;
+}
+
+static uint64_t count_poison_mismatches(const float *actual,
+                                        uint64_t begin,
+                                        uint64_t end,
+                                        uint32_t base) {
+    uint64_t mismatches = 0;
+    for (uint64_t i = begin; i < end; i++) {
+        uint32_t bits = 0;
+        memcpy(&bits, &actual[i], sizeof(bits));
+        if (bits != base + (uint32_t)(i & 0xffffu)) mismatches++;
+    }
+    return mismatches;
+}
+
+int main(void) {
+    static const uint32_t exact_rows[] = {6u, 8u, 9u, 16u, 21u, 30u, 31u};
+    const uint64_t page = (uint64_t)getpagesize();
+    const uint64_t row_a_bytes = (GROUP_DIM / QK_K) * sizeof(block_q4_K);
+    const uint64_t out_a_bytes =
+        (uint64_t)N_GROUPS * RANK * row_a_bytes;
+    const uint64_t out_b_offset = align_up(out_a_bytes, page);
+    const uint64_t row_b_bytes = (LOW_DIM / QK_K) * sizeof(block_q4_K);
+    const uint64_t out_b_bytes = (uint64_t)OUT_DIM * row_b_bytes;
+    const uint64_t model_bytes =
+        align_up(out_b_offset + out_b_bytes, page);
+    const uint64_t heads_row_bytes =
+        (uint64_t)N_GROUPS * GROUP_DIM * sizeof(float);
+    const uint64_t low_row_bytes = (uint64_t)LOW_DIM * sizeof(float);
+    const uint64_t out_row_bytes = (uint64_t)OUT_DIM * sizeof(float);
+    const uint64_t heads_count = (uint64_t)ALLOC_ROWS * N_GROUPS * GROUP_DIM;
+    const uint64_t low_count = (uint64_t)ALLOC_ROWS * LOW_DIM;
+    const uint64_t out_count = (uint64_t)ALLOC_ROWS * OUT_DIM;
+
+    CHECK(unsetenv(k_enable) == 0, "clear enable env");
+    CHECK(unsetenv(k_disable) == 0, "clear disable env");
+    CHECK(unsetenv(k_require) == 0, "clear require env");
+    CHECK(unsetenv(k_disable_scale_meta) == 0,
+          "clear scale-meta disable env");
+    CHECK(unsetenv(k_require_scale_meta) == 0,
+          "clear scale-meta require env");
+    CHECK(unsetenv(k_disable_classic) == 0, "clear classic kill env");
+    CHECK(unsetenv(k_disable_f16_rhs) == 0, "clear F16 RHS disable env");
+    CHECK(unsetenv(k_require_f16_rhs) == 0, "clear F16 RHS require env");
+
+    CHECK(ds4_gpu_init() != 0, "Metal init");
+    if (!ds4_gpu_device_is_pre_m5_apple_silicon()) {
+        fprintf(stderr,
+                "Metal Q4 SSD-prefill exact-N oracle SKIP: requires Apple M1--M4\n");
+        ds4_gpu_cleanup();
+        return 0;
+    }
+
+    void *model = NULL;
+    CHECK(posix_memalign(&model, (size_t)page, (size_t)model_bytes) == 0,
+          "model allocation");
+    memset(model, 0, (size_t)model_bytes);
+    fill_q4_matrix(model, GROUP_DIM, N_GROUPS * RANK, 211u);
+    fill_q4_matrix((uint8_t *)model + out_b_offset,
+                   LOW_DIM, OUT_DIM, 307u);
+
+    float *heads_host = malloc((size_t)heads_count * sizeof(float));
+    float *reference_low_host = calloc((size_t)low_count, sizeof(float));
+    float *reference_out_host = calloc((size_t)out_count, sizeof(float));
+    float *candidate_low_host = malloc((size_t)low_count * sizeof(float));
+    float *candidate_out_host = malloc((size_t)out_count * sizeof(float));
+    CHECK(heads_host && reference_low_host && reference_out_host &&
+          candidate_low_host && candidate_out_host, "host tensors");
+
+    for (uint32_t row = 0; row < ALLOC_ROWS; row++) {
+        for (uint32_t i = 0; i < N_GROUPS * GROUP_DIM; i++) {
+            const uint32_t key =
+                i * 41u + row * 271u + ((i >> 2u) ^ (row * 19u));
+            heads_host[(uint64_t)row * N_GROUPS * GROUP_DIM + i] =
+                (float)((int)(key % 257u) - 128) / 137.0f;
+        }
+    }
+
+    ds4_gpu_tensor *heads =
+        ds4_gpu_tensor_alloc(heads_count * sizeof(float));
+    ds4_gpu_tensor *reference_low =
+        ds4_gpu_tensor_alloc(low_count * sizeof(float));
+    ds4_gpu_tensor *reference_out =
+        ds4_gpu_tensor_alloc(out_count * sizeof(float));
+    ds4_gpu_tensor *candidate_low =
+        ds4_gpu_tensor_alloc(low_count * sizeof(float));
+    ds4_gpu_tensor *candidate_out =
+        ds4_gpu_tensor_alloc(out_count * sizeof(float));
+    CHECK(heads && reference_low && reference_out && candidate_low &&
+          candidate_out, "Metal tensors");
+    CHECK(ds4_gpu_tensor_write(heads, 0, heads_host,
+                               heads_count * sizeof(float)) != 0,
+          "heads upload");
+    CHECK(ds4_gpu_tensor_write(reference_low, 0, reference_low_host,
+                               low_count * sizeof(float)) != 0,
+          "reference low clear");
+    CHECK(ds4_gpu_tensor_write(reference_out, 0, reference_out_host,
+                               out_count * sizeof(float)) != 0,
+          "reference out clear");
+    CHECK(ds4_gpu_set_model_map(model, model_bytes) != 0, "model map");
+    ds4_gpu_set_quality(false);
+    ds4_gpu_set_ssd_streaming(false);
+
+    /* Canonical oracle: one row at a time through the established classic
+     * Q4_K attention-A slice and output matvec entry points. */
+    for (uint32_t row = 0; row < MAX_ROWS; row++) {
+        ds4_gpu_tensor *heads_row = ds4_gpu_tensor_view(
+            heads, (uint64_t)row * heads_row_bytes, heads_row_bytes);
+        ds4_gpu_tensor *low_row = ds4_gpu_tensor_view(
+            reference_low, (uint64_t)row * low_row_bytes, low_row_bytes);
+        ds4_gpu_tensor *out_row = ds4_gpu_tensor_view(
+            reference_out, (uint64_t)row * out_row_bytes, out_row_bytes);
+        CHECK(heads_row && low_row && out_row, "reference views");
+        CHECK(ds4_gpu_attention_output_low_q4_K_slice_tensor(
+                  low_row, model, model_bytes, 0,
+                  GROUP_DIM, RANK, 0, N_GROUPS, heads_row, 0) != 0,
+              "reference low projection");
+        CHECK(ds4_gpu_matmul_quant_tensor(
+                  out_row, model, model_bytes, out_b_offset, Q4_K_TYPE,
+                  LOW_DIM, OUT_DIM, low_row, 1u) != 0,
+              "reference output projection");
+        ds4_gpu_tensor_free(out_row);
+        ds4_gpu_tensor_free(low_row);
+        ds4_gpu_tensor_free(heads_row);
+    }
+    CHECK(ds4_gpu_tensor_read(reference_low, 0, reference_low_host,
+                              low_count * sizeof(float)) != 0,
+          "reference low read");
+    CHECK(ds4_gpu_tensor_read(reference_out, 0, reference_out_host,
+                              out_count * sizeof(float)) != 0,
+          "reference out read");
+
+    ds4_gpu_set_ssd_streaming(true);
+
+    /* Default-off is observable through the production wrapper. */
+    CHECK(ds4_gpu_attention_output_q4_K_batch_tensor(
+              candidate_out, candidate_low, NULL, NULL, model, model_bytes,
+              0, out_b_offset, Q4_K_TYPE, GROUP_DIM, RANK,
+              N_GROUPS, OUT_DIM, heads, 6u) == 0,
+          "default-off gate");
+    CHECK(setenv(k_enable, "1", 1) == 0, "enable candidate");
+
+    for (uint32_t scale_variant = 0; scale_variant < 2u; scale_variant++) {
+        const bool use_scale_meta = scale_variant == 0u;
+        if (use_scale_meta) {
+            CHECK(unsetenv(k_disable_scale_meta) == 0,
+                  "enable shared scale metadata");
+            CHECK(setenv(k_require_scale_meta, "1", 1) == 0,
+                  "require shared scale metadata");
+        } else {
+            CHECK(unsetenv(k_require_scale_meta) == 0,
+                  "clear shared scale metadata requirement");
+            CHECK(setenv(k_disable_scale_meta, "1", 1) == 0,
+                  "select legacy scale unpack");
+        }
+        for (uint32_t case_i = 0;
+             case_i < sizeof(exact_rows) / sizeof(exact_rows[0]);
+             case_i++) {
+            const uint32_t n_rows = exact_rows[case_i];
+            poison(candidate_low_host, low_count, 0x7fc10000u);
+            poison(candidate_out_host, out_count, 0x7fc20000u);
+            CHECK(ds4_gpu_tensor_write(candidate_low, 0, candidate_low_host,
+                                       low_count * sizeof(float)) != 0,
+                  "candidate low poison");
+            CHECK(ds4_gpu_tensor_write(candidate_out, 0, candidate_out_host,
+                                       out_count * sizeof(float)) != 0,
+                  "candidate out poison");
+
+            /* Exercise the production wrapper delegation, not only the direct
+             * test entry point. Scratch arguments are unused by this path. */
+            CHECK(ds4_gpu_attention_output_q4_K_batch_tensor(
+                      candidate_out, candidate_low, NULL, NULL,
+                      model, model_bytes, 0, out_b_offset, Q4_K_TYPE,
+                      GROUP_DIM, RANK, N_GROUPS, OUT_DIM, heads, n_rows) == 1,
+                  "candidate dispatch");
+            CHECK(ds4_gpu_tensor_read(candidate_low, 0, candidate_low_host,
+                                      low_count * sizeof(float)) != 0,
+                  "candidate low read");
+            CHECK(ds4_gpu_tensor_read(candidate_out, 0, candidate_out_host,
+                                      out_count * sizeof(float)) != 0,
+                  "candidate out read");
+
+            uint64_t first_low = UINT64_MAX;
+            uint64_t first_out = UINT64_MAX;
+            const uint64_t compared_low = (uint64_t)n_rows * LOW_DIM;
+            const uint64_t compared_out = (uint64_t)n_rows * OUT_DIM;
+            const uint64_t low_mismatch = count_bit_mismatches(
+                reference_low_host, candidate_low_host,
+                compared_low, &first_low);
+            const uint64_t out_mismatch = count_bit_mismatches(
+                reference_out_host, candidate_out_host,
+                compared_out, &first_out);
+            const uint64_t low_canary = count_poison_mismatches(
+                candidate_low_host, compared_low, low_count, 0x7fc10000u);
+            const uint64_t out_canary = count_poison_mismatches(
+                candidate_out_host, compared_out, out_count, 0x7fc20000u);
+            fprintf(stderr,
+                    "Metal Q4 SSD-prefill exact-N=%u scale_meta=%s "
+                    "low=%llu/%llu "
+                    "out=%llu/%llu low_canary=%llu out_canary=%llu\n",
+                    n_rows,
+                    use_scale_meta ? "shared" : "legacy",
+                    (unsigned long long)low_mismatch,
+                    (unsigned long long)compared_low,
+                    (unsigned long long)out_mismatch,
+                    (unsigned long long)compared_out,
+                    (unsigned long long)low_canary,
+                    (unsigned long long)out_canary);
+            if (low_mismatch != 0) {
+                uint32_t reference_bits = 0;
+                uint32_t candidate_bits = 0;
+                memcpy(&reference_bits, &reference_low_host[first_low],
+                       sizeof(reference_bits));
+                memcpy(&candidate_bits, &candidate_low_host[first_low],
+                       sizeof(candidate_bits));
+                fprintf(stderr,
+                        "  first low mismatch index=%llu "
+                        "reference=%a (0x%08x) candidate=%a (0x%08x)\n",
+                        (unsigned long long)first_low,
+                        reference_low_host[first_low], reference_bits,
+                        candidate_low_host[first_low], candidate_bits);
+            }
+            if (out_mismatch != 0) {
+                fprintf(stderr, "  first out mismatch index=%llu\n",
+                        (unsigned long long)first_out);
+            }
+            CHECK(low_mismatch == 0, "low projection bitwise mismatch");
+            CHECK(out_mismatch == 0, "output projection bitwise mismatch");
+            CHECK(low_canary == 0, "low tail canary");
+            CHECK(out_canary == 0, "output tail canary");
+        }
+    }
+
+    /* At N=32 both output-B variants execute the same legacy M64xN32xK32
+     * schedule.  The candidate differs only by materializing the staging cast
+     * once into scratch, so the complete wrapper output must remain bitwise
+     * identical.  Keep one extra output row and guards around the F16 scratch
+     * to catch either output or conversion overruns. */
+    ds4_gpu_set_ssd_streaming(false);
+    ds4_gpu_set_quality(true);
+    enum { F16_RHS_ROWS = 32u, F16_RHS_GUARD = 64u };
+    const uint64_t f16_rhs_payload =
+        (uint64_t)F16_RHS_ROWS * LOW_DIM;
+    const uint64_t f16_rhs_storage =
+        F16_RHS_GUARD + f16_rhs_payload + F16_RHS_GUARD;
+    uint16_t *f16_rhs_host = malloc(
+        (size_t)f16_rhs_storage * sizeof(uint16_t));
+    ds4_gpu_tensor *f16_rhs_base = ds4_gpu_tensor_alloc(
+        f16_rhs_storage * sizeof(uint16_t));
+    ds4_gpu_tensor *f16_rhs = ds4_gpu_tensor_view(
+        f16_rhs_base,
+        F16_RHS_GUARD * sizeof(uint16_t),
+        f16_rhs_payload * sizeof(uint16_t));
+    CHECK(f16_rhs_host && f16_rhs_base && f16_rhs,
+          "F16 RHS scratch allocation");
+    for (uint64_t i = 0; i < f16_rhs_storage; i++) {
+        f16_rhs_host[i] =
+            (uint16_t)(0x7e00u | (uint16_t)(i & 0x1ffu));
+    }
+    CHECK(ds4_gpu_tensor_write(
+              f16_rhs_base, 0, f16_rhs_host,
+              f16_rhs_storage * sizeof(uint16_t)) != 0,
+          "F16 RHS scratch poison");
+
+    poison(reference_low_host, low_count, 0x7fc30000u);
+    poison(reference_out_host, out_count, 0x7fc40000u);
+    poison(candidate_low_host, low_count, 0x7fc30000u);
+    poison(candidate_out_host, out_count, 0x7fc40000u);
+    CHECK(ds4_gpu_tensor_write(reference_low, 0, reference_low_host,
+                               low_count * sizeof(float)) != 0,
+          "F16 RHS baseline low poison");
+    CHECK(ds4_gpu_tensor_write(reference_out, 0, reference_out_host,
+                               out_count * sizeof(float)) != 0,
+          "F16 RHS baseline out poison");
+    CHECK(ds4_gpu_tensor_write(candidate_low, 0, candidate_low_host,
+                               low_count * sizeof(float)) != 0,
+          "F16 RHS candidate low poison");
+    CHECK(ds4_gpu_tensor_write(candidate_out, 0, candidate_out_host,
+                               out_count * sizeof(float)) != 0,
+          "F16 RHS candidate out poison");
+
+    CHECK(ds4_gpu_attention_output_q4_K_batch_tensor(
+              reference_out, reference_low, NULL, NULL,
+              model, model_bytes, 0, out_b_offset, Q4_K_TYPE,
+              GROUP_DIM, RANK, N_GROUPS, OUT_DIM, heads,
+              F16_RHS_ROWS) == 1,
+          "F16 RHS baseline dispatch");
+    CHECK(ds4_gpu_attention_output_q4_K_batch_tensor(
+              candidate_out, candidate_low, f16_rhs, NULL,
+              model, model_bytes, 0, out_b_offset, Q4_K_TYPE,
+              GROUP_DIM, RANK, N_GROUPS, OUT_DIM, heads,
+              F16_RHS_ROWS) == 1,
+          "F16 RHS default-on candidate dispatch");
+    CHECK(ds4_gpu_tensor_read(reference_low, 0, reference_low_host,
+                              low_count * sizeof(float)) != 0,
+          "F16 RHS baseline low read");
+    CHECK(ds4_gpu_tensor_read(reference_out, 0, reference_out_host,
+                              out_count * sizeof(float)) != 0,
+          "F16 RHS baseline out read");
+    CHECK(ds4_gpu_tensor_read(candidate_low, 0, candidate_low_host,
+                              low_count * sizeof(float)) != 0,
+          "F16 RHS candidate low read");
+    CHECK(ds4_gpu_tensor_read(candidate_out, 0, candidate_out_host,
+                              out_count * sizeof(float)) != 0,
+          "F16 RHS candidate out read");
+    CHECK(ds4_gpu_tensor_read(
+              f16_rhs_base, 0, f16_rhs_host,
+              f16_rhs_storage * sizeof(uint16_t)) != 0,
+          "F16 RHS scratch read");
+
+    uint64_t first_low = UINT64_MAX;
+    uint64_t first_out = UINT64_MAX;
+    const uint64_t f16_low_mismatch = count_bit_mismatches(
+        reference_low_host, candidate_low_host,
+        (uint64_t)F16_RHS_ROWS * LOW_DIM, &first_low);
+    const uint64_t f16_out_mismatch = count_bit_mismatches(
+        reference_out_host, candidate_out_host,
+        (uint64_t)F16_RHS_ROWS * OUT_DIM, &first_out);
+    const uint64_t baseline_low_tail = count_poison_mismatches(
+        reference_low_host, (uint64_t)F16_RHS_ROWS * LOW_DIM,
+        low_count, 0x7fc30000u);
+    const uint64_t candidate_low_tail = count_poison_mismatches(
+        candidate_low_host, (uint64_t)F16_RHS_ROWS * LOW_DIM,
+        low_count, 0x7fc30000u);
+    const uint64_t baseline_out_tail = count_poison_mismatches(
+        reference_out_host, (uint64_t)F16_RHS_ROWS * OUT_DIM,
+        out_count, 0x7fc40000u);
+    const uint64_t candidate_out_tail = count_poison_mismatches(
+        candidate_out_host, (uint64_t)F16_RHS_ROWS * OUT_DIM,
+        out_count, 0x7fc40000u);
+    uint64_t f16_guard_mismatch = 0;
+    uint64_t f16_payload_poison = 0;
+    for (uint64_t i = 0; i < F16_RHS_GUARD; i++) {
+        const uint16_t expected =
+            (uint16_t)(0x7e00u | (uint16_t)(i & 0x1ffu));
+        if (f16_rhs_host[i] != expected) f16_guard_mismatch++;
+    }
+    for (uint64_t i = F16_RHS_GUARD;
+         i < F16_RHS_GUARD + f16_rhs_payload; i++) {
+        const uint16_t poison_value =
+            (uint16_t)(0x7e00u | (uint16_t)(i & 0x1ffu));
+        if (f16_rhs_host[i] == poison_value) f16_payload_poison++;
+    }
+    for (uint64_t i = F16_RHS_GUARD + f16_rhs_payload;
+         i < f16_rhs_storage; i++) {
+        const uint16_t expected =
+            (uint16_t)(0x7e00u | (uint16_t)(i & 0x1ffu));
+        if (f16_rhs_host[i] != expected) f16_guard_mismatch++;
+    }
+    fprintf(stderr,
+            "Metal Q4 attention output-B F16 RHS N=32 "
+            "low=%llu out=%llu low_tail=%llu/%llu "
+            "out_tail=%llu/%llu scratch_guard=%llu payload_poison=%llu\n",
+            (unsigned long long)f16_low_mismatch,
+            (unsigned long long)f16_out_mismatch,
+            (unsigned long long)baseline_low_tail,
+            (unsigned long long)candidate_low_tail,
+            (unsigned long long)baseline_out_tail,
+            (unsigned long long)candidate_out_tail,
+            (unsigned long long)f16_guard_mismatch,
+            (unsigned long long)f16_payload_poison);
+    CHECK(f16_low_mismatch == 0, "F16 RHS low bitwise mismatch");
+    CHECK(f16_out_mismatch == 0, "F16 RHS output bitwise mismatch");
+    CHECK(baseline_low_tail == 0 && candidate_low_tail == 0,
+          "F16 RHS low tail canary");
+    CHECK(baseline_out_tail == 0 && candidate_out_tail == 0,
+          "F16 RHS output tail canary");
+    CHECK(f16_guard_mismatch == 0, "F16 RHS scratch canary");
+    CHECK(f16_payload_poison == 0, "F16 RHS default-on materialization");
+
+    /* Flash uses an output dimension divisible by 64, which selects the
+     * direct full-tile store and its smaller threadgroup allocation.  Reuse
+     * the first 64 rows of the fixture so both legacy specializations remain
+     * covered without growing the model allocation. */
+    enum { F16_RHS_FULL_TILE_OUT_DIM = 64u };
+    const uint64_t f16_full_tile_count =
+        (uint64_t)F16_RHS_ROWS * F16_RHS_FULL_TILE_OUT_DIM;
+    poison(reference_out_host, out_count, 0x7fc50000u);
+    poison(candidate_out_host, out_count, 0x7fc50000u);
+    CHECK(ds4_gpu_tensor_write(reference_out, 0, reference_out_host,
+                               out_count * sizeof(float)) != 0,
+          "F16 RHS full-tile baseline poison");
+    CHECK(ds4_gpu_tensor_write(candidate_out, 0, candidate_out_host,
+                               out_count * sizeof(float)) != 0,
+          "F16 RHS full-tile candidate poison");
+    CHECK(ds4_gpu_attention_output_q4_K_batch_tensor(
+              reference_out, reference_low, NULL, NULL,
+              model, model_bytes, 0, out_b_offset, Q4_K_TYPE,
+              GROUP_DIM, RANK, N_GROUPS, F16_RHS_FULL_TILE_OUT_DIM, heads,
+              F16_RHS_ROWS) == 1,
+          "F16 RHS full-tile baseline dispatch");
+    CHECK(ds4_gpu_attention_output_q4_K_batch_tensor(
+              candidate_out, candidate_low, f16_rhs, NULL,
+              model, model_bytes, 0, out_b_offset, Q4_K_TYPE,
+              GROUP_DIM, RANK, N_GROUPS, F16_RHS_FULL_TILE_OUT_DIM, heads,
+              F16_RHS_ROWS) == 1,
+          "F16 RHS full-tile candidate dispatch");
+    CHECK(ds4_gpu_tensor_read(reference_out, 0, reference_out_host,
+                              out_count * sizeof(float)) != 0,
+          "F16 RHS full-tile baseline read");
+    CHECK(ds4_gpu_tensor_read(candidate_out, 0, candidate_out_host,
+                              out_count * sizeof(float)) != 0,
+          "F16 RHS full-tile candidate read");
+    uint64_t first_full_tile = UINT64_MAX;
+    const uint64_t f16_full_tile_mismatch = count_bit_mismatches(
+        reference_out_host, candidate_out_host,
+        f16_full_tile_count, &first_full_tile);
+    const uint64_t baseline_full_tile_tail = count_poison_mismatches(
+        reference_out_host, f16_full_tile_count,
+        out_count, 0x7fc50000u);
+    const uint64_t candidate_full_tile_tail = count_poison_mismatches(
+        candidate_out_host, f16_full_tile_count,
+        out_count, 0x7fc50000u);
+    fprintf(stderr,
+            "Metal Q4 attention output-B F16 RHS full tile N=32 "
+            "out=%llu tail=%llu/%llu\n",
+            (unsigned long long)f16_full_tile_mismatch,
+            (unsigned long long)baseline_full_tile_tail,
+            (unsigned long long)candidate_full_tile_tail);
+    CHECK(f16_full_tile_mismatch == 0,
+          "F16 RHS full-tile output bitwise mismatch");
+    CHECK(baseline_full_tile_tail == 0 && candidate_full_tile_tail == 0,
+          "F16 RHS full-tile output tail canary");
+
+    /* Restore the boundary-specialized reference consumed by the SSD
+     * fallback comparison below. */
+    poison(reference_out_host, out_count, 0x7fc40000u);
+    CHECK(ds4_gpu_tensor_write(reference_out, 0, reference_out_host,
+                               out_count * sizeof(float)) != 0,
+          "F16 RHS boundary reference poison restore");
+    CHECK(ds4_gpu_attention_output_q4_K_batch_tensor(
+              reference_out, reference_low, NULL, NULL,
+              model, model_bytes, 0, out_b_offset, Q4_K_TYPE,
+              GROUP_DIM, RANK, N_GROUPS, OUT_DIM, heads,
+              F16_RHS_ROWS) == 1,
+          "F16 RHS boundary reference restore dispatch");
+    CHECK(ds4_gpu_tensor_read(reference_out, 0, reference_out_host,
+                              out_count * sizeof(float)) != 0,
+          "F16 RHS boundary reference restore read");
+
+    CHECK(setenv(k_require_f16_rhs, "1", 1) == 0,
+          "require F16 RHS candidate");
+    CHECK(ds4_gpu_attention_output_q4_K_batch_tensor(
+              candidate_out, candidate_low, f16_rhs, NULL,
+              model, model_bytes, 0, out_b_offset, Q4_K_TYPE,
+              GROUP_DIM, RANK, N_GROUPS, OUT_DIM, heads,
+              F16_RHS_ROWS) == 1,
+          "required F16 RHS candidate dispatch");
+    CHECK(ds4_gpu_attention_output_q4_K_batch_tensor(
+              candidate_out, candidate_low, f16_rhs, NULL,
+              model, model_bytes, 0, out_b_offset, Q4_K_TYPE,
+              GROUP_DIM, RANK, N_GROUPS, OUT_DIM, heads, 31u) == -1,
+          "required F16 RHS rejects N below one MM tile");
+    CHECK(setenv(k_disable_f16_rhs, "1", 1) == 0,
+          "disable required F16 RHS candidate");
+    CHECK(ds4_gpu_attention_output_q4_K_batch_tensor(
+              candidate_out, candidate_low, f16_rhs, NULL,
+              model, model_bytes, 0, out_b_offset, Q4_K_TYPE,
+              GROUP_DIM, RANK, N_GROUPS, OUT_DIM, heads,
+              F16_RHS_ROWS) == -1,
+          "F16 RHS disable wins over REQUIRE");
+    CHECK(unsetenv(k_disable_f16_rhs) == 0,
+          "clear F16 RHS disable env");
+    ds4_gpu_set_ssd_streaming(true);
+    CHECK(ds4_gpu_attention_output_q4_K_batch_tensor(
+              candidate_out, candidate_low, f16_rhs, NULL,
+              model, model_bytes, 0, out_b_offset, Q4_K_TYPE,
+              GROUP_DIM, RANK, N_GROUPS, OUT_DIM, heads,
+              F16_RHS_ROWS) == -1,
+          "required F16 RHS rejects SSD streaming");
+    CHECK(unsetenv(k_require_f16_rhs) == 0,
+          "clear F16 RHS require env");
+
+    /* In ordinary SSD mode the wrapper must use output-B's established F32
+     * path, return success, and leave the F16 scratch completely untouched. */
+    for (uint64_t i = 0; i < f16_rhs_storage; i++) {
+        f16_rhs_host[i] =
+            (uint16_t)(0x7e00u | (uint16_t)(i & 0x1ffu));
+    }
+    poison(candidate_low_host, low_count, 0x7fc30000u);
+    poison(candidate_out_host, out_count, 0x7fc40000u);
+    CHECK(ds4_gpu_tensor_write(
+              f16_rhs_base, 0, f16_rhs_host,
+              f16_rhs_storage * sizeof(uint16_t)) != 0,
+          "SSD fallback scratch poison");
+    CHECK(ds4_gpu_tensor_write(candidate_low, 0, candidate_low_host,
+                               low_count * sizeof(float)) != 0,
+          "SSD fallback low poison");
+    CHECK(ds4_gpu_tensor_write(candidate_out, 0, candidate_out_host,
+                               out_count * sizeof(float)) != 0,
+          "SSD fallback out poison");
+    CHECK(ds4_gpu_attention_output_q4_K_batch_tensor(
+              candidate_out, candidate_low, f16_rhs, NULL,
+              model, model_bytes, 0, out_b_offset, Q4_K_TYPE,
+              GROUP_DIM, RANK, N_GROUPS, OUT_DIM, heads,
+              F16_RHS_ROWS) == 1,
+          "SSD fallback F32 dispatch");
+    CHECK(ds4_gpu_tensor_read(candidate_low, 0, candidate_low_host,
+                              low_count * sizeof(float)) != 0,
+          "SSD fallback low read");
+    CHECK(ds4_gpu_tensor_read(candidate_out, 0, candidate_out_host,
+                              out_count * sizeof(float)) != 0,
+          "SSD fallback out read");
+    CHECK(ds4_gpu_tensor_read(
+              f16_rhs_base, 0, f16_rhs_host,
+              f16_rhs_storage * sizeof(uint16_t)) != 0,
+          "SSD fallback scratch read");
+    const uint64_t ssd_low_mismatch = count_bit_mismatches(
+        reference_low_host, candidate_low_host,
+        (uint64_t)F16_RHS_ROWS * LOW_DIM, &first_low);
+    const uint64_t ssd_out_mismatch = count_bit_mismatches(
+        reference_out_host, candidate_out_host,
+        (uint64_t)F16_RHS_ROWS * OUT_DIM, &first_out);
+    uint64_t ssd_scratch_mismatch = 0;
+    for (uint64_t i = 0; i < f16_rhs_storage; i++) {
+        const uint16_t expected =
+            (uint16_t)(0x7e00u | (uint16_t)(i & 0x1ffu));
+        if (f16_rhs_host[i] != expected) ssd_scratch_mismatch++;
+    }
+    fprintf(stderr,
+            "Metal Q4 attention output-B SSD fallback N=32 "
+            "low=%llu out=%llu scratch=%llu\n",
+            (unsigned long long)ssd_low_mismatch,
+            (unsigned long long)ssd_out_mismatch,
+            (unsigned long long)ssd_scratch_mismatch);
+    CHECK(ssd_low_mismatch == 0, "SSD fallback low bitwise mismatch");
+    CHECK(ssd_out_mismatch == 0, "SSD fallback output bitwise mismatch");
+    CHECK(ssd_scratch_mismatch == 0, "SSD fallback touched F16 scratch");
+    ds4_gpu_set_quality(false);
+    ds4_gpu_tensor_free(f16_rhs);
+    ds4_gpu_tensor_free(f16_rhs_base);
+    free(f16_rhs_host);
+
+    CHECK(unsetenv(k_disable_scale_meta) == 0,
+          "restore shared scale metadata");
+    CHECK(unsetenv(k_require_scale_meta) == 0,
+          "clear shared scale metadata requirement");
+    CHECK(setenv(k_require_scale_meta, "1", 1) == 0,
+          "set scale-meta REQUIRE for kill-switch check");
+    CHECK(setenv(k_disable_scale_meta, "1", 1) == 0,
+          "set scale-meta disable for kill-switch check");
+    CHECK(ds4_gpu_attention_output_q4_K_batch_tensor(
+              candidate_out, candidate_low, NULL, NULL, model, model_bytes,
+              0, out_b_offset, Q4_K_TYPE, GROUP_DIM, RANK,
+              N_GROUPS, OUT_DIM, heads, 21u) == -1,
+          "scale-meta disable wins over REQUIRE");
+    CHECK(unsetenv(k_disable_scale_meta) == 0,
+          "clear scale-meta disable after kill-switch check");
+    CHECK(unsetenv(k_require_scale_meta) == 0,
+          "clear scale-meta REQUIRE after kill-switch check");
+
+    /* REQUIRE implies enable.  Both explicit kill switches win and must
+     * return -1 instead of allowing a false-green row fallback. */
+    CHECK(unsetenv(k_enable) == 0, "clear enable before REQUIRE");
+    CHECK(setenv(k_require, "1", 1) == 0, "set REQUIRE");
+    CHECK(ds4_gpu_attention_output_q4_K_batch_tensor(
+              candidate_out, candidate_low, NULL, NULL, model, model_bytes,
+              0, out_b_offset, Q4_K_TYPE, GROUP_DIM, RANK,
+              N_GROUPS, OUT_DIM, heads, 21u) == 1,
+          "REQUIRE implies enable");
+    CHECK(setenv(k_disable, "1", 1) == 0, "set exact-N disable");
+    CHECK(ds4_gpu_attention_output_q4_K_batch_tensor(
+              candidate_out, candidate_low, NULL, NULL, model, model_bytes,
+              0, out_b_offset, Q4_K_TYPE, GROUP_DIM, RANK,
+              N_GROUPS, OUT_DIM, heads, 21u) == -1,
+          "disable wins over REQUIRE");
+    CHECK(unsetenv(k_disable) == 0, "clear exact-N disable");
+    CHECK(setenv(k_disable_classic, "1", 1) == 0, "set classic disable");
+    CHECK(ds4_gpu_attention_output_q4_K_batch_tensor(
+              candidate_out, candidate_low, NULL, NULL, model, model_bytes,
+              0, out_b_offset, Q4_K_TYPE, GROUP_DIM, RANK,
+              N_GROUPS, OUT_DIM, heads, 21u) == -1,
+          "classic kill wins over REQUIRE");
+
+    ds4_gpu_set_ssd_streaming(false);
+    ds4_gpu_tensor_free(candidate_out);
+    ds4_gpu_tensor_free(candidate_low);
+    ds4_gpu_tensor_free(reference_out);
+    ds4_gpu_tensor_free(reference_low);
+    ds4_gpu_tensor_free(heads);
+    ds4_gpu_cleanup();
+    free(candidate_out_host);
+    free(candidate_low_host);
+    free(reference_out_host);
+    free(reference_low_host);
+    free(heads_host);
+    free(model);
+    fprintf(stderr,
+            "Metal Q4 SSD-prefill exact-N oracle PASS rows=6,8,9,16,21,30,31 "
+            "scale_meta=shared,legacy bitwise=1 canary=1 gates=1\n");
+    return 0;
+}
+
+#else
+
+int main(void) {
+    fprintf(stderr, "Metal Q4 SSD-prefill exact-N oracle SKIP: non-Apple host\n");
+    return 0;
+}
+
+#endif
diff --git a/tests/test_metal_q4_attn_out_a_direct b/tests/test_metal_q4_attn_out_a_direct
new file mode 100755
index 0000000000..8f2a776840
Binary files /dev/null and b/tests/test_metal_q4_attn_out_a_direct differ
diff --git a/tests/test_metal_q4_attn_out_a_direct.c b/tests/test_metal_q4_attn_out_a_direct.c
new file mode 100644
index 0000000000..50d3aabc11
--- /dev/null
+++ b/tests/test_metal_q4_attn_out_a_direct.c
@@ -0,0 +1,539 @@
+#define _DARWIN_C_SOURCE
+
+/* Production-shape, GGUF-free oracle for the pre-M5 Metal Q4_K attention
+ * output-A fixed-route specialization.  The established routed kernel is the
+ * bitwise baseline; output-B deliberately stays small to keep this focused
+ * test's resident memory footprint prudent. */
+
+#include "ds4_gpu.h"
+
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+bool ds4_log_is_tty(FILE *fp) {
+    (void)fp;
+    return false;
+}
+
+#ifdef __APPLE__
+
+enum {
+    Q4_K_TYPE = 12u,
+    Q8_0_TYPE = 8u,
+    QK_K = 256u,
+    QK8_0 = 32u,
+    GROUP_DIM = 4096u,
+    RANK = 1024u,
+    N_GROUPS = 8u,
+    LOW_DIM = N_GROUPS * RANK,
+    OUT_DIM = 64u,
+    ACTIVE_ROWS = 513u,
+    REJECT_ROWS = 511u,
+    ALLOC_ROWS = ACTIVE_ROWS + 1u,
+    GUARD_ELEMENTS = 64u,
+};
+
+typedef struct {
+    uint16_t d;
+    uint16_t dmin;
+    uint8_t scales[12];
+    uint8_t qs[QK_K / 2u];
+} block_q4_K;
+
+typedef struct {
+    uint16_t d;
+    int8_t qs[QK8_0];
+} block_q8_0;
+
+static const char *k_disable =
+    "DS4_METAL_DISABLE_Q4_ATTN_OUT_A_DIRECT";
+static const char *k_require =
+    "DS4_METAL_REQUIRE_Q4_ATTN_OUT_A_DIRECT";
+static const char *k_disable_f16_rhs =
+    "DS4_METAL_DISABLE_Q4_ATTN_OUT_B_F16_RHS";
+static const char *k_require_f16_rhs =
+    "DS4_METAL_REQUIRE_Q4_ATTN_OUT_B_F16_RHS";
+
+static void fail(const char *what) {
+    fprintf(stderr, "Metal Q4 output-A direct oracle FAIL: %s\n", what);
+    exit(1);
+}
+
+#define CHECK(expr, what) do { if (!(expr)) fail(what); } while (0)
+
+static uint64_t align_up(uint64_t value, uint64_t alignment) {
+    return (value + alignment - 1u) & ~(alignment - 1u);
+}
+
+static uint64_t hash_bytes(const void *raw, uint64_t bytes) {
+    const uint8_t *p = raw;
+    uint64_t hash = UINT64_C(1469598103934665603);
+    for (uint64_t i = 0; i < bytes; i++) {
+        hash ^= p[i];
+        hash *= UINT64_C(1099511628211);
+    }
+    return hash;
+}
+
+static void pack_scales(uint8_t packed[12],
+                        const uint8_t scale[8],
+                        const uint8_t minimum[8]) {
+    memset(packed, 0, 12u);
+    for (uint32_t group = 0; group < 4u; group++) {
+        packed[group] = scale[group] & 63u;
+        packed[group + 4u] = minimum[group] & 63u;
+    }
+    for (uint32_t group = 4u; group < 8u; group++) {
+        packed[group + 4u] = (scale[group] & 15u) |
+                             ((minimum[group] & 15u) << 4u);
+        packed[group - 4u] |= (scale[group] >> 4u) << 6u;
+        packed[group] |= (minimum[group] >> 4u) << 6u;
+    }
+}
+
+static void fill_q4_matrix(void *raw, uint32_t in_dim,
+                           uint32_t rows, uint32_t salt) {
+    CHECK(sizeof(block_q4_K) == 144u, "unexpected Q4_K block size");
+    CHECK((in_dim % QK_K) == 0u, "unaligned Q4_K fixture");
+    const uint32_t blocks_per_row = in_dim / QK_K;
+    block_q4_K *matrix = raw;
+    for (uint32_t row = 0; row < rows; row++) {
+        for (uint32_t block = 0; block < blocks_per_row; block++) {
+            block_q4_K *b = matrix +
+                (uint64_t)row * blocks_per_row + block;
+            const uint32_t key = salt + row * 1009u + block * 313u +
+                                 (row ^ (block * 17u));
+            uint8_t scale[8];
+            uint8_t minimum[8];
+            for (uint32_t group = 0; group < 8u; group++) {
+                scale[group] = (uint8_t)((key + group * 7u) % 64u);
+                minimum[group] =
+                    (uint8_t)((key / 3u + group * 5u) % 64u);
+            }
+            pack_scales(b->scales, scale, minimum);
+            for (uint32_t i = 0; i < QK_K / 2u; i++) {
+                b->qs[i] =
+                    (uint8_t)(key + i * 37u + (i >> 2u) * 11u);
+            }
+            /* Exact binary half scales keep the fixture deterministic. */
+            b->d = 0x2400u;
+            b->dmin = 0x1c00u;
+        }
+    }
+}
+
+static void fill_q8_matrix(void *raw, uint32_t in_dim,
+                           uint32_t rows, uint32_t salt) {
+    CHECK(sizeof(block_q8_0) == 34u, "unexpected Q8_0 block size");
+    CHECK((in_dim % QK8_0) == 0u, "unaligned Q8_0 fixture");
+    const uint32_t blocks_per_row = in_dim / QK8_0;
+    block_q8_0 *matrix = raw;
+    for (uint32_t row = 0; row < rows; row++) {
+        for (uint32_t block = 0; block < blocks_per_row; block++) {
+            block_q8_0 *b = matrix +
+                (uint64_t)row * blocks_per_row + block;
+            const uint32_t key = salt + row * 977u + block * 101u;
+            b->d = 0x2000u;
+            for (uint32_t i = 0; i < QK8_0; i++) {
+                b->qs[i] = (int8_t)((int)((key + i * 29u) % 127u) - 63);
+            }
+        }
+    }
+}
+
+static void fill_heads(float *heads) {
+    for (uint32_t row = 0; row < ALLOC_ROWS; row++) {
+        for (uint32_t i = 0; i < N_GROUPS * GROUP_DIM; i++) {
+            const uint32_t key =
+                i * 41u + row * 271u + ((i >> 2u) ^ (row * 19u));
+            heads[(uint64_t)row * N_GROUPS * GROUP_DIM + i] =
+                (float)((int)(key % 255u) - 127) / 512.0f;
+        }
+    }
+}
+
+static void poison(float *values, uint64_t count, uint32_t base) {
+    for (uint64_t i = 0; i < count; i++) {
+        const uint32_t bits = base + (uint32_t)(i & 0xffffu);
+        memcpy(&values[i], &bits, sizeof(bits));
+    }
+}
+
+static uint64_t count_poison_mismatches(const float *actual,
+                                        uint64_t begin,
+                                        uint64_t end,
+                                        uint32_t base) {
+    uint64_t mismatches = 0;
+    for (uint64_t i = begin; i < end; i++) {
+        uint32_t bits = 0;
+        memcpy(&bits, &actual[i], sizeof(bits));
+        if (bits != base + (uint32_t)(i & 0xffffu)) mismatches++;
+    }
+    return mismatches;
+}
+
+static uint64_t count_bit_mismatches(const float *reference,
+                                     const float *actual,
+                                     uint64_t count,
+                                     uint64_t *first) {
+    uint64_t mismatches = 0;
+    *first = UINT64_MAX;
+    for (uint64_t i = 0; i < count; i++) {
+        if (memcmp(&reference[i], &actual[i], sizeof(float)) != 0) {
+            if (*first == UINT64_MAX) *first = i;
+            mismatches++;
+        }
+    }
+    return mismatches;
+}
+
+static void upload_poison(ds4_gpu_tensor *low,
+                          ds4_gpu_tensor *out,
+                          float *low_host,
+                          float *out_host,
+                          uint64_t low_count,
+                          uint64_t out_count,
+                          uint32_t low_poison,
+                          uint32_t out_poison) {
+    poison(low_host, low_count, low_poison);
+    poison(out_host, out_count, out_poison);
+    CHECK(ds4_gpu_tensor_write(low, 0, low_host,
+                               low_count * sizeof(float)) != 0,
+          "low poison upload");
+    CHECK(ds4_gpu_tensor_write(out, 0, out_host,
+                               out_count * sizeof(float)) != 0,
+          "out poison upload");
+}
+
+static void read_outputs(ds4_gpu_tensor *low,
+                         ds4_gpu_tensor *out,
+                         float *low_host,
+                         float *out_host,
+                         uint64_t low_count,
+                         uint64_t out_count) {
+    CHECK(ds4_gpu_tensor_read(low, 0, low_host,
+                              low_count * sizeof(float)) != 0,
+          "low read");
+    CHECK(ds4_gpu_tensor_read(out, 0, out_host,
+                              out_count * sizeof(float)) != 0,
+          "out read");
+}
+
+static void check_inputs_immutable(ds4_gpu_tensor *heads,
+                                   float *heads_host,
+                                   uint64_t heads_bytes,
+                                   uint64_t heads_hash,
+                                   const void *model,
+                                   uint64_t model_bytes,
+                                   uint64_t model_hash) {
+    CHECK(ds4_gpu_tensor_read(heads, 0, heads_host, heads_bytes) != 0,
+          "heads immutability read");
+    CHECK(hash_bytes(heads_host, heads_bytes) == heads_hash,
+          "heads were modified");
+    CHECK(hash_bytes(model, model_bytes) == model_hash,
+          "model weights were modified");
+}
+
+int main(void) {
+    const uint64_t page = (uint64_t)getpagesize();
+    const uint64_t row_a_bytes =
+        (GROUP_DIM / QK_K) * sizeof(block_q4_K);
+    const uint64_t out_a_bytes = (uint64_t)LOW_DIM * row_a_bytes;
+    const uint64_t out_b_offset = align_up(out_a_bytes, page);
+    const uint64_t row_b_bytes =
+        (LOW_DIM / QK8_0) * sizeof(block_q8_0);
+    const uint64_t out_b_bytes = (uint64_t)OUT_DIM * row_b_bytes;
+    const uint64_t model_bytes =
+        align_up(out_b_offset + out_b_bytes, page);
+    const uint64_t heads_payload_count =
+        (uint64_t)ALLOC_ROWS * N_GROUPS * GROUP_DIM;
+    const uint64_t low_payload_count = (uint64_t)ALLOC_ROWS * LOW_DIM;
+    const uint64_t out_payload_count = (uint64_t)ALLOC_ROWS * OUT_DIM;
+    const uint64_t heads_storage_count =
+        GUARD_ELEMENTS + heads_payload_count + GUARD_ELEMENTS;
+    const uint64_t low_storage_count =
+        GUARD_ELEMENTS + low_payload_count + GUARD_ELEMENTS;
+    const uint64_t out_storage_count =
+        GUARD_ELEMENTS + out_payload_count + GUARD_ELEMENTS;
+    const uint64_t active_low_count = (uint64_t)ACTIVE_ROWS * LOW_DIM;
+    const uint64_t active_out_count = (uint64_t)ACTIVE_ROWS * OUT_DIM;
+    const uint64_t heads_payload_bytes =
+        heads_payload_count * sizeof(float);
+    const uint64_t heads_storage_bytes =
+        heads_storage_count * sizeof(float);
+
+    CHECK(unsetenv(k_disable) == 0, "clear direct disable env");
+    CHECK(unsetenv(k_require) == 0, "clear direct require env");
+    CHECK(unsetenv(k_disable_f16_rhs) == 0,
+          "clear output-B F16 disable env");
+    CHECK(unsetenv(k_require_f16_rhs) == 0,
+          "clear output-B F16 require env");
+
+    CHECK(ds4_gpu_init() != 0, "Metal init");
+    if (!ds4_gpu_device_is_pre_m5_apple_silicon()) {
+        fprintf(stderr,
+                "Metal Q4 output-A direct oracle SKIP: requires Apple M1--M4\n");
+        ds4_gpu_cleanup();
+        return 0;
+    }
+
+    void *model = NULL;
+    CHECK(posix_memalign(&model, (size_t)page, (size_t)model_bytes) == 0,
+          "model allocation");
+    memset(model, 0, (size_t)model_bytes);
+    fill_q4_matrix(model, GROUP_DIM, LOW_DIM, 211u);
+    fill_q8_matrix((uint8_t *)model + out_b_offset,
+                   LOW_DIM, OUT_DIM, 307u);
+    const uint64_t model_hash = hash_bytes(model, model_bytes);
+
+    float *heads_host = malloc((size_t)heads_storage_bytes);
+    float *low_host = malloc((size_t)low_storage_count * sizeof(float));
+    float *out_host = malloc((size_t)out_storage_count * sizeof(float));
+    float *baseline_low = malloc((size_t)active_low_count * sizeof(float));
+    float *baseline_out = malloc((size_t)active_out_count * sizeof(float));
+    CHECK(heads_host && low_host && out_host && baseline_low && baseline_out,
+          "host tensor allocation");
+    poison(heads_host, heads_storage_count, 0x7fc00000u);
+    fill_heads(heads_host + GUARD_ELEMENTS);
+    const uint64_t heads_hash =
+        hash_bytes(heads_host, heads_storage_bytes);
+
+    ds4_gpu_tensor *heads_base =
+        ds4_gpu_tensor_alloc(heads_storage_bytes);
+    ds4_gpu_tensor *low_base =
+        ds4_gpu_tensor_alloc(low_storage_count * sizeof(float));
+    ds4_gpu_tensor *out_base =
+        ds4_gpu_tensor_alloc(out_storage_count * sizeof(float));
+    CHECK(heads_base && low_base && out_base,
+          "guarded Metal base allocation");
+    ds4_gpu_tensor *heads = ds4_gpu_tensor_view(
+        heads_base, GUARD_ELEMENTS * sizeof(float), heads_payload_bytes);
+    ds4_gpu_tensor *low = ds4_gpu_tensor_view(
+        low_base, GUARD_ELEMENTS * sizeof(float),
+        low_payload_count * sizeof(float));
+    ds4_gpu_tensor *out = ds4_gpu_tensor_view(
+        out_base, GUARD_ELEMENTS * sizeof(float),
+        out_payload_count * sizeof(float));
+    CHECK(heads && low && out, "guarded Metal tensor views");
+    CHECK(ds4_gpu_tensor_write(heads_base, 0, heads_host,
+                               heads_storage_bytes) != 0,
+          "heads upload");
+
+    /* Quality mode also disables the Metal4 cooperative path.  SSD mode is
+     * intentional: this is the exact production dispatch policy under test. */
+    ds4_gpu_set_quality(true);
+    ds4_gpu_set_ssd_streaming(true);
+    CHECK(ds4_gpu_set_model_map(model, model_bytes) != 0, "model map");
+
+    const uint32_t low_poison = 0x7fc10000u;
+    const uint32_t out_poison = 0x7fc20000u;
+    upload_poison(low_base, out_base, low_host, out_host,
+                  low_storage_count, out_storage_count,
+                  low_poison, out_poison);
+    CHECK(setenv(k_disable, "1", 1) == 0, "select routed baseline");
+    CHECK(setenv(k_require, "1", 1) == 0,
+          "require direct for kill-switch preflight");
+    CHECK(ds4_gpu_attention_output_q4_K_batch_tensor(
+              out, low, NULL, NULL, model, model_bytes,
+              0, out_b_offset, Q8_0_TYPE, GROUP_DIM, RANK,
+              N_GROUPS, OUT_DIM, heads, ACTIVE_ROWS) == -1,
+          "direct disable must win over REQUIRE");
+    read_outputs(low_base, out_base, low_host, out_host,
+                 low_storage_count, out_storage_count);
+    CHECK(count_poison_mismatches(low_host, 0, low_storage_count,
+                                  low_poison) == 0,
+          "direct kill-switch preflight modified low");
+    CHECK(count_poison_mismatches(out_host, 0, out_storage_count,
+                                  out_poison) == 0,
+          "direct kill-switch preflight modified out");
+    CHECK(unsetenv(k_require) == 0, "clear REQUIRE for baseline");
+    CHECK(ds4_gpu_attention_output_q4_K_batch_tensor(
+              out, low, NULL, NULL, model, model_bytes,
+              0, out_b_offset, Q8_0_TYPE, GROUP_DIM, RANK,
+              N_GROUPS, OUT_DIM, heads, ACTIVE_ROWS) == 1,
+          "routed baseline dispatch");
+    read_outputs(low_base, out_base, low_host, out_host,
+                 low_storage_count, out_storage_count);
+    CHECK(count_poison_mismatches(
+              low_host, GUARD_ELEMENTS,
+              GUARD_ELEMENTS + active_low_count, low_poison) ==
+              active_low_count,
+          "baseline did not overwrite every active low value");
+    CHECK(count_poison_mismatches(
+              out_host, GUARD_ELEMENTS,
+              GUARD_ELEMENTS + active_out_count, out_poison) ==
+              active_out_count,
+          "baseline did not overwrite every active out value");
+    CHECK(count_poison_mismatches(low_host, 0, GUARD_ELEMENTS,
+                                  low_poison) == 0,
+          "baseline low prefix canary");
+    CHECK(count_poison_mismatches(
+              low_host, GUARD_ELEMENTS + active_low_count,
+              GUARD_ELEMENTS + low_payload_count,
+              low_poison) == 0,
+          "baseline low tail canary");
+    CHECK(count_poison_mismatches(
+              low_host, GUARD_ELEMENTS + low_payload_count,
+              low_storage_count, low_poison) == 0,
+          "baseline low suffix canary");
+    CHECK(count_poison_mismatches(out_host, 0, GUARD_ELEMENTS,
+                                  out_poison) == 0,
+          "baseline out prefix canary");
+    CHECK(count_poison_mismatches(
+              out_host, GUARD_ELEMENTS + active_out_count,
+              GUARD_ELEMENTS + out_payload_count,
+              out_poison) == 0,
+          "baseline out tail canary");
+    CHECK(count_poison_mismatches(
+              out_host, GUARD_ELEMENTS + out_payload_count,
+              out_storage_count, out_poison) == 0,
+          "baseline out suffix canary");
+    memcpy(baseline_low, low_host + GUARD_ELEMENTS,
+           (size_t)active_low_count * sizeof(float));
+    memcpy(baseline_out, out_host + GUARD_ELEMENTS,
+           (size_t)active_out_count * sizeof(float));
+    check_inputs_immutable(heads_base, heads_host, heads_storage_bytes,
+                           heads_hash,
+                           model, model_bytes, model_hash);
+
+    upload_poison(low_base, out_base, low_host, out_host,
+                  low_storage_count, out_storage_count,
+                  low_poison, out_poison);
+    CHECK(unsetenv(k_disable) == 0, "enable direct candidate");
+    CHECK(setenv(k_require, "1", 1) == 0, "require direct candidate");
+    CHECK(ds4_gpu_attention_output_q4_K_batch_tensor(
+              out, low, NULL, NULL, model, model_bytes,
+              0, out_b_offset, Q8_0_TYPE, GROUP_DIM, RANK,
+              N_GROUPS, OUT_DIM, heads, ACTIVE_ROWS) == 1,
+          "direct candidate dispatch");
+    read_outputs(low_base, out_base, low_host, out_host,
+                 low_storage_count, out_storage_count);
+    CHECK(count_poison_mismatches(
+              low_host, GUARD_ELEMENTS,
+              GUARD_ELEMENTS + active_low_count, low_poison) ==
+              active_low_count,
+          "direct candidate did not overwrite every active low value");
+    CHECK(count_poison_mismatches(
+              out_host, GUARD_ELEMENTS,
+              GUARD_ELEMENTS + active_out_count, out_poison) ==
+              active_out_count,
+          "direct candidate did not overwrite every active out value");
+
+    uint64_t first_low = UINT64_MAX;
+    uint64_t first_out = UINT64_MAX;
+    const uint64_t low_mismatch = count_bit_mismatches(
+        baseline_low, low_host + GUARD_ELEMENTS,
+        active_low_count, &first_low);
+    const uint64_t out_mismatch = count_bit_mismatches(
+        baseline_out, out_host + GUARD_ELEMENTS,
+        active_out_count, &first_out);
+    const uint64_t low_prefix_mismatch = count_poison_mismatches(
+        low_host, 0, GUARD_ELEMENTS, low_poison);
+    const uint64_t low_tail_mismatch = count_poison_mismatches(
+        low_host, GUARD_ELEMENTS + active_low_count,
+        GUARD_ELEMENTS + low_payload_count, low_poison);
+    const uint64_t low_suffix_mismatch = count_poison_mismatches(
+        low_host, GUARD_ELEMENTS + low_payload_count,
+        low_storage_count, low_poison);
+    const uint64_t out_prefix_mismatch = count_poison_mismatches(
+        out_host, 0, GUARD_ELEMENTS, out_poison);
+    const uint64_t out_tail_mismatch = count_poison_mismatches(
+        out_host, GUARD_ELEMENTS + active_out_count,
+        GUARD_ELEMENTS + out_payload_count, out_poison);
+    const uint64_t out_suffix_mismatch = count_poison_mismatches(
+        out_host, GUARD_ELEMENTS + out_payload_count,
+        out_storage_count, out_poison);
+    fprintf(stderr,
+            "Metal Q4 output-A direct N=%u low=%llu/%llu out=%llu/%llu "
+            "low_guard=%llu/%llu/%llu out_guard=%llu/%llu/%llu\n",
+            ACTIVE_ROWS,
+            (unsigned long long)low_mismatch,
+            (unsigned long long)active_low_count,
+            (unsigned long long)out_mismatch,
+            (unsigned long long)active_out_count,
+            (unsigned long long)low_prefix_mismatch,
+            (unsigned long long)low_tail_mismatch,
+            (unsigned long long)low_suffix_mismatch,
+            (unsigned long long)out_prefix_mismatch,
+            (unsigned long long)out_tail_mismatch,
+            (unsigned long long)out_suffix_mismatch);
+    if (low_mismatch != 0) {
+        fprintf(stderr, "  first low mismatch index=%llu\n",
+                (unsigned long long)first_low);
+    }
+    if (out_mismatch != 0) {
+        fprintf(stderr, "  first out mismatch index=%llu\n",
+                (unsigned long long)first_out);
+    }
+    CHECK(low_mismatch == 0, "direct low bitwise mismatch");
+    CHECK(out_mismatch == 0, "direct final output bitwise mismatch");
+    CHECK(low_prefix_mismatch == 0, "direct low prefix canary");
+    CHECK(low_tail_mismatch == 0, "direct low tail canary");
+    CHECK(low_suffix_mismatch == 0, "direct low suffix canary");
+    CHECK(out_prefix_mismatch == 0, "direct out prefix canary");
+    CHECK(out_tail_mismatch == 0, "direct out tail canary");
+    CHECK(out_suffix_mismatch == 0, "direct out suffix canary");
+    check_inputs_immutable(heads_base, heads_host, heads_storage_bytes,
+                           heads_hash,
+                           model, model_bytes, model_hash);
+
+    /* REQUIRE must fail closed before encoding anything just below the
+     * production threshold.  The entire low/out allocations are poison here,
+     * so this also detects partial work from an ineligible dispatch. */
+    const uint32_t reject_low_poison = 0x7fc30000u;
+    const uint32_t reject_out_poison = 0x7fc40000u;
+    upload_poison(low_base, out_base, low_host, out_host,
+                  low_storage_count, out_storage_count,
+                  reject_low_poison, reject_out_poison);
+    CHECK(ds4_gpu_attention_output_q4_K_batch_tensor(
+              out, low, NULL, NULL, model, model_bytes,
+              0, out_b_offset, Q8_0_TYPE, GROUP_DIM, RANK,
+              N_GROUPS, OUT_DIM, heads, REJECT_ROWS) == -1,
+          "N=511 REQUIRE must fail closed");
+    read_outputs(low_base, out_base, low_host, out_host,
+                 low_storage_count, out_storage_count);
+    CHECK(count_poison_mismatches(low_host, 0, low_storage_count,
+                                  reject_low_poison) == 0,
+          "N=511 REQUIRE modified low");
+    CHECK(count_poison_mismatches(out_host, 0, out_storage_count,
+                                  reject_out_poison) == 0,
+          "N=511 REQUIRE modified out");
+    check_inputs_immutable(heads_base, heads_host, heads_storage_bytes,
+                           heads_hash,
+                           model, model_bytes, model_hash);
+
+    CHECK(unsetenv(k_require) == 0, "clear direct REQUIRE");
+    CHECK(unsetenv(k_disable) == 0, "clear direct disable");
+    ds4_gpu_set_ssd_streaming(false);
+    ds4_gpu_set_quality(false);
+    ds4_gpu_tensor_free(out);
+    ds4_gpu_tensor_free(low);
+    ds4_gpu_tensor_free(heads);
+    ds4_gpu_tensor_free(out_base);
+    ds4_gpu_tensor_free(low_base);
+    ds4_gpu_tensor_free(heads_base);
+    ds4_gpu_cleanup();
+    free(baseline_out);
+    free(baseline_low);
+    free(out_host);
+    free(low_host);
+    free(heads_host);
+    free(model);
+    fprintf(stderr,
+            "Metal Q4 output-A direct oracle PASS N=513 bitwise=1 "
+            "tail=1 immutable=1 reject_N511=1\n");
+    return 0;
+}
+
+#else
+
+int main(void) {
+    fprintf(stderr, "Metal Q4 output-A direct oracle SKIP: non-Apple host\n");
+    return 0;
+}
+
+#endif
diff --git a/tests/test_metal_q4_prefill_pair.c b/tests/test_metal_q4_prefill_pair.c
new file mode 100644
index 0000000000..3612f509e6
--- /dev/null
+++ b/tests/test_metal_q4_prefill_pair.c
@@ -0,0 +1,491 @@
+#define _DARWIN_C_SOURCE
+
+/* Runtime oracle for the Metal Q4_K q_a/KV prefill pair.
+ *
+ * The model is a page-aligned synthetic mapping containing only the two
+ * production projection matrices.  It deliberately installs two disjoint
+ * model spans while SSD mode is enabled, so the test exercises the same model
+ * view contract as streaming inference without reading a GGUF or SSD.
+ */
+
+#include "ds4_gpu.h"
+
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#ifdef __APPLE__
+
+/* Standalone Metal tests link the backend without ds4.c. */
+bool ds4_log_is_tty(FILE *fp) {
+    (void)fp;
+    return false;
+}
+
+enum {
+    Q4_K_TYPE = 12u,
+    QK_K = 256u,
+    IN_DIM = 4096u,
+    OUT0_DIM = 1024u,
+    OUT1_DIM = 512u,
+    MAX_TOKENS = 128u,
+    TEST_STREAMS = 2u,
+};
+
+static const uint32_t k_tokens[] = {32u, 64u, 96u, 128u};
+static const float k_poison = -12345.25f;
+
+typedef struct {
+    uint16_t d;
+    uint16_t dmin;
+    uint8_t scales[12];
+    uint8_t qs[QK_K / 2u];
+} block_q4_K;
+
+_Static_assert(sizeof(block_q4_K) == 144u, "Q4_K ABI");
+
+typedef struct {
+    void *model;
+    uint64_t model_size;
+    uint64_t row_bytes;
+    uint64_t weight_offset[2];
+    uint64_t weight_bytes[2];
+    ds4_gpu_tensor *x[TEST_STREAMS];
+    ds4_gpu_tensor *baseline[TEST_STREAMS][2];
+    ds4_gpu_tensor *candidate[TEST_STREAMS][2];
+    float *x_host[TEST_STREAMS];
+} fixture;
+
+static void fail(const char *what) {
+    fprintf(stderr, "Metal Q4 prefill pair runtime FAIL: %s\n", what);
+    exit(1);
+}
+
+static void set_flag(const char *name, bool enabled) {
+    const int rc = enabled ? setenv(name, "1", 1) : unsetenv(name);
+    if (rc != 0) fail("environment update");
+}
+
+static void set_pair_controls(bool enabled, bool disabled, bool required) {
+    set_flag("DS4_METAL_ENABLE_Q4_PREFILL_PAIR_F16_RHS", enabled);
+    set_flag("DS4_METAL_DISABLE_Q4_PREFILL_PAIR_F16_RHS", disabled);
+    set_flag("DS4_METAL_REQUIRE_Q4_PREFILL_PAIR_F16_RHS", required);
+}
+
+static uint64_t align_up(uint64_t value, uint64_t alignment) {
+    return (value + alignment - 1u) / alignment * alignment;
+}
+
+static uint32_t lcg(uint32_t *state) {
+    *state = *state * 1664525u + 1013904223u;
+    return *state;
+}
+
+static void fill_q4(block_q4_K *blocks, uint64_t count, uint32_t seed) {
+    for (uint64_t i = 0; i < count; i++) {
+        blocks[i].d = (uint16_t)(0x2400u | (lcg(&seed) & 0x03ffu));
+        blocks[i].dmin =
+            (uint16_t)(0x1c00u | (lcg(&seed) & 0x03ffu));
+        for (uint32_t j = 0; j < sizeof(blocks[i].scales); j++) {
+            blocks[i].scales[j] = (uint8_t)(lcg(&seed) >> 24u);
+        }
+        for (uint32_t j = 0; j < sizeof(blocks[i].qs); j++) {
+            blocks[i].qs[j] = (uint8_t)(lcg(&seed) >> 24u);
+        }
+    }
+}
+
+static void fill_x(float *x, uint32_t stream) {
+    for (uint64_t i = 0; i < (uint64_t)MAX_TOKENS * IN_DIM; i++) {
+        const int32_t v = (int32_t)(
+            (i * 29u + stream * 47u + ((uint64_t)stream ^ i) * 3u) %
+            255u) - 127;
+        x[i] = (float)v / 128.0f;
+    }
+}
+
+static uint64_t checksum(const void *ptr, uint64_t bytes) {
+    const uint8_t *p = ptr;
+    uint64_t hash = UINT64_C(1469598103934665603);
+    for (uint64_t i = 0; i < bytes; i++) {
+        hash ^= p[i];
+        hash *= UINT64_C(1099511628211);
+    }
+    return hash;
+}
+
+static uint64_t output_count(uint32_t output) {
+    return (uint64_t)MAX_TOKENS *
+           (output == 0u ? OUT0_DIM : OUT1_DIM);
+}
+
+static uint64_t active_output_count(uint32_t output, uint32_t n_tokens) {
+    return (uint64_t)n_tokens *
+           (output == 0u ? OUT0_DIM : OUT1_DIM);
+}
+
+static void fixture_init(fixture *f) {
+    memset(f, 0, sizeof(*f));
+    f->row_bytes = (IN_DIM / QK_K) * sizeof(block_q4_K);
+    const uint64_t page = (uint64_t)getpagesize();
+    f->weight_offset[0] = 0u;
+    f->weight_bytes[0] = (uint64_t)OUT0_DIM * f->row_bytes;
+    f->weight_bytes[1] = (uint64_t)OUT1_DIM * f->row_bytes;
+    f->weight_offset[1] = align_up(f->weight_bytes[0] + page, page);
+
+    f->model_size = align_up(f->weight_offset[1] + f->weight_bytes[1],
+                             page);
+    if (posix_memalign(&f->model, (size_t)page, (size_t)f->model_size) != 0) {
+        fail("synthetic model allocation");
+    }
+    memset(f->model, 0, (size_t)f->model_size);
+    fill_q4((block_q4_K *)((uint8_t *)f->model + f->weight_offset[0]),
+            f->weight_bytes[0] / sizeof(block_q4_K), 0x41c64e6du);
+    fill_q4((block_q4_K *)((uint8_t *)f->model + f->weight_offset[1]),
+            f->weight_bytes[1] / sizeof(block_q4_K), 0x9e3779b9u);
+
+    ds4_gpu_set_quality(false);
+    ds4_gpu_set_ssd_streaming(true);
+    const uint64_t offsets[2] = {
+        f->weight_offset[0], f->weight_offset[1],
+    };
+    const uint64_t sizes[2] = {
+        f->weight_bytes[0], f->weight_bytes[1],
+    };
+    if (!ds4_gpu_set_model_map_spans(f->model, f->model_size,
+                                     offsets, sizes, 2u,
+                                     f->weight_bytes[0])) {
+        fail("SSD-style model spans");
+    }
+
+    const uint64_t x_count = (uint64_t)MAX_TOKENS * IN_DIM;
+    for (uint32_t stream = 0; stream < TEST_STREAMS; stream++) {
+        f->x_host[stream] = malloc((size_t)x_count * sizeof(float));
+        if (!f->x_host[stream]) fail("host activation allocation");
+        fill_x(f->x_host[stream], stream);
+        f->x[stream] = ds4_gpu_tensor_alloc(x_count * sizeof(float));
+        if (!f->x[stream] ||
+            !ds4_gpu_tensor_write(f->x[stream], 0u, f->x_host[stream],
+                                  x_count * sizeof(float))) {
+            fail("activation tensor");
+        }
+        for (uint32_t output = 0; output < 2u; output++) {
+            const uint64_t bytes = output_count(output) * sizeof(float);
+            f->baseline[stream][output] = ds4_gpu_tensor_alloc(bytes);
+            f->candidate[stream][output] = ds4_gpu_tensor_alloc(bytes);
+            if (!f->baseline[stream][output] ||
+                !f->candidate[stream][output]) {
+                fail("output tensor allocation");
+            }
+        }
+    }
+    fprintf(stderr,
+            "Metal Q4 prefill pair runtime: synthetic=%.2f MiB spans=2 "
+            "ssd=1 max_tokens=%u\n",
+            (double)f->model_size / 1048576.0, MAX_TOKENS);
+}
+
+static void fixture_destroy(fixture *f) {
+    ds4_gpu_set_stream(0);
+    (void)ds4_gpu_synchronize();
+    for (uint32_t stream = 0; stream < TEST_STREAMS; stream++) {
+        for (uint32_t output = 0; output < 2u; output++) {
+            ds4_gpu_tensor_free(f->candidate[stream][output]);
+            ds4_gpu_tensor_free(f->baseline[stream][output]);
+        }
+        ds4_gpu_tensor_free(f->x[stream]);
+        free(f->x_host[stream]);
+    }
+    ds4_gpu_cleanup();
+    free(f->model);
+    memset(f, 0, sizeof(*f));
+}
+
+static void poison_outputs(fixture *f, uint32_t stream) {
+    for (uint32_t output = 0; output < 2u; output++) {
+        const uint64_t count = output_count(output);
+        if (!ds4_gpu_tensor_fill_f32(f->baseline[stream][output],
+                                     k_poison, count) ||
+            !ds4_gpu_tensor_fill_f32(f->candidate[stream][output],
+                                     k_poison, count)) {
+            fail("output poison");
+        }
+    }
+}
+
+static int run_baseline(fixture *f, uint32_t stream, uint32_t n_tokens) {
+    ds4_gpu_set_stream((int)stream);
+    if (!ds4_gpu_begin_commands()) return 0;
+    int ok = ds4_gpu_matmul_quant_tensor(
+        f->baseline[stream][0], f->model, f->model_size,
+        f->weight_offset[0], Q4_K_TYPE, IN_DIM, OUT0_DIM,
+        f->x[stream], n_tokens);
+    if (ok) {
+        ok = ds4_gpu_matmul_quant_tensor(
+            f->baseline[stream][1], f->model, f->model_size,
+            f->weight_offset[1], Q4_K_TYPE, IN_DIM, OUT1_DIM,
+            f->x[stream], n_tokens);
+    }
+    const int ended = ds4_gpu_end_commands();
+    return ok && ended;
+}
+
+static int run_candidate(fixture *f, uint32_t stream, uint32_t n_tokens) {
+    ds4_gpu_set_stream((int)stream);
+    if (!ds4_gpu_begin_commands()) return 0;
+    const int rc = ds4_gpu_matmul_q4_K_pair_tensor(
+        f->candidate[stream][0], f->candidate[stream][1],
+        f->model, f->model_size,
+        f->weight_offset[0], f->weight_offset[1],
+        IN_DIM, OUT0_DIM, OUT1_DIM, f->x[stream], n_tokens);
+    const int ended = ds4_gpu_end_commands();
+    return rc == 1 && ended;
+}
+
+static void check_poison_suffix(const float *values, uint64_t begin,
+                                uint64_t count, const char *label) {
+    for (uint64_t i = begin; i < count; i++) {
+        if (memcmp(&values[i], &k_poison, sizeof(k_poison)) != 0) {
+            fprintf(stderr,
+                    "Metal Q4 prefill pair suffix write label=%s index=%llu\n",
+                    label, (unsigned long long)i);
+            fail("output suffix canary");
+        }
+    }
+}
+
+static void check_tensor_all_poison(const ds4_gpu_tensor *tensor,
+                                    uint64_t count, const char *label) {
+    float *values = malloc((size_t)count * sizeof(float));
+    if (!values ||
+        !ds4_gpu_tensor_read(tensor, 0u, values, count * sizeof(float))) {
+        fail("poison readback");
+    }
+    check_poison_suffix(values, 0u, count, label);
+    free(values);
+}
+
+static void check_outputs(fixture *f, uint32_t stream, uint32_t n_tokens) {
+    for (uint32_t output = 0; output < 2u; output++) {
+        const uint64_t count = output_count(output);
+        const uint64_t active = active_output_count(output, n_tokens);
+        float *base = malloc((size_t)count * sizeof(float));
+        float *candidate = malloc((size_t)count * sizeof(float));
+        if (!base || !candidate) fail("host output allocation");
+        if (!ds4_gpu_tensor_read(f->baseline[stream][output], 0u, base,
+                                 count * sizeof(float)) ||
+            !ds4_gpu_tensor_read(f->candidate[stream][output], 0u, candidate,
+                                 count * sizeof(float))) {
+            fail("output readback");
+        }
+        if (memcmp(base, candidate, active * sizeof(float)) != 0) {
+            uint64_t first = 0u;
+            while (first < active &&
+                   memcmp(&base[first], &candidate[first],
+                          sizeof(float)) == 0) {
+                first++;
+            }
+            fprintf(stderr,
+                    "Metal Q4 prefill pair mismatch N=%u stream=%u output=%u "
+                    "first=%llu base=%g candidate=%g\n",
+                    n_tokens, stream, output, (unsigned long long)first,
+                    first < active ? base[first] : 0.0f,
+                    first < active ? candidate[first] : 0.0f);
+            fail("bitwise parity");
+        }
+        check_poison_suffix(base, active, count, "baseline");
+        check_poison_suffix(candidate, active, count, "candidate");
+        fprintf(stderr,
+                "Metal Q4 prefill pair N=%u stream=%u output=%u "
+                "checksum=%016llx bitwise=1 canary=1\n",
+                n_tokens, stream, output,
+                (unsigned long long)checksum(base, active * sizeof(float)));
+        free(candidate);
+        free(base);
+    }
+}
+
+static void check_x_unchanged(fixture *f, uint32_t stream) {
+    const uint64_t count = (uint64_t)MAX_TOKENS * IN_DIM;
+    float *actual = malloc((size_t)count * sizeof(float));
+    if (!actual) fail("activation readback allocation");
+    if (!ds4_gpu_tensor_read(f->x[stream], 0u, actual,
+                             count * sizeof(float)) ||
+        memcmp(actual, f->x_host[stream], count * sizeof(float)) != 0) {
+        fail("activation modified");
+    }
+    free(actual);
+}
+
+static void test_shapes(fixture *f) {
+    set_pair_controls(true, false, true);
+    for (uint32_t i = 0; i < sizeof(k_tokens) / sizeof(k_tokens[0]); i++) {
+        const uint32_t n_tokens = k_tokens[i];
+        poison_outputs(f, 0u);
+        if (!run_baseline(f, 0u, n_tokens)) fail("baseline projection");
+        if (!run_candidate(f, 0u, n_tokens)) fail("required pair projection");
+        check_outputs(f, 0u, n_tokens);
+        check_x_unchanged(f, 0u);
+    }
+}
+
+static void test_alias_rejection(fixture *f) {
+    const uint32_t n_tokens = 32u;
+    const uint64_t alias_bytes =
+        active_output_count(0u, n_tokens) * sizeof(float);
+    ds4_gpu_tensor *alias =
+        ds4_gpu_tensor_view(f->x[0], 0u, alias_bytes);
+    if (!alias) fail("alias tensor view");
+    poison_outputs(f, 0u);
+    set_pair_controls(true, false, false);
+    const int rc = ds4_gpu_matmul_q4_K_pair_tensor(
+        alias, f->candidate[0][1], f->model, f->model_size,
+        f->weight_offset[0], f->weight_offset[1],
+        IN_DIM, OUT0_DIM, OUT1_DIM, f->x[0], n_tokens);
+    ds4_gpu_tensor_free(alias);
+    if (rc != 0) fail("alias was not rejected with fallback status");
+    check_x_unchanged(f, 0u);
+
+    check_tensor_all_poison(f->candidate[0][1], output_count(1u),
+                            "alias-reject");
+    fprintf(stderr, "Metal Q4 prefill pair alias rejection: PASS\n");
+}
+
+static void test_required_failure_is_local(fixture *f) {
+    const uint32_t n_tokens = 32u;
+    const uint64_t active = active_output_count(0u, n_tokens);
+    const uint64_t count = output_count(0u);
+    float *reference = malloc((size_t)active * sizeof(float));
+    float *actual = malloc((size_t)count * sizeof(float));
+    if (!reference || !actual) fail("REQUIRE oracle allocation");
+
+    set_pair_controls(false, false, false);
+    if (!ds4_gpu_matmul_quant_tensor(
+            f->baseline[0][0], f->model, f->model_size,
+            f->weight_offset[0], Q4_K_TYPE, IN_DIM, OUT0_DIM,
+            f->x[0], n_tokens) ||
+        !ds4_gpu_tensor_read(f->baseline[0][0], 0u, reference,
+                             active * sizeof(float))) {
+        fail("REQUIRE reference matmul");
+    }
+
+    poison_outputs(f, 0u);
+    set_pair_controls(true, true, true);
+    const int rc = ds4_gpu_matmul_q4_K_pair_tensor(
+        f->candidate[0][0], f->candidate[0][1],
+        f->model, f->model_size,
+        f->weight_offset[0], f->weight_offset[1],
+        IN_DIM, OUT0_DIM, OUT1_DIM, f->x[0], n_tokens);
+    if (rc != -1) {
+        fprintf(stderr,
+                "Metal Q4 prefill pair REQUIRE negative returned %d, expected -1\n",
+                rc);
+        fail("REQUIRE negative status");
+    }
+    check_tensor_all_poison(f->candidate[0][0], output_count(0u),
+                            "require-negative-out0");
+    check_tensor_all_poison(f->candidate[0][1], output_count(1u),
+                            "require-negative-out1");
+
+    set_pair_controls(false, false, false);
+    if (!ds4_gpu_matmul_quant_tensor(
+            f->baseline[0][0], f->model, f->model_size,
+            f->weight_offset[0], Q4_K_TYPE, IN_DIM, OUT0_DIM,
+            f->x[0], n_tokens)) {
+        fail("REQUIRE failure contaminated next quant matmul");
+    }
+    if (!ds4_gpu_tensor_read(f->baseline[0][0], 0u, actual,
+                             count * sizeof(float)) ||
+        memcmp(actual, reference, active * sizeof(float)) != 0) {
+        fail("REQUIRE failure changed next quant matmul");
+    }
+    check_poison_suffix(actual, active, count, "require-next-matmul");
+    free(actual);
+    free(reference);
+    fprintf(stderr,
+            "Metal Q4 prefill pair REQUIRE negative: PASS "
+            "rc=-1 next_matmul_bitwise=1\n");
+}
+
+static void test_two_stream_async(fixture *f) {
+    static const uint32_t stream_tokens[TEST_STREAMS] = {96u, 128u};
+    set_pair_controls(true, false, true);
+    for (uint32_t stream = 0; stream < TEST_STREAMS; stream++) {
+        poison_outputs(f, stream);
+        if (!run_baseline(f, stream, stream_tokens[stream])) {
+            fail("async baseline");
+        }
+    }
+
+    uint32_t submitted = 0u;
+    for (uint32_t stream = 0; stream < TEST_STREAMS; stream++) {
+        ds4_gpu_set_stream((int)stream);
+        if (!ds4_gpu_begin_commands()) fail("async begin");
+        const int rc = ds4_gpu_matmul_q4_K_pair_tensor(
+            f->candidate[stream][0], f->candidate[stream][1],
+            f->model, f->model_size,
+            f->weight_offset[0], f->weight_offset[1],
+            IN_DIM, OUT0_DIM, OUT1_DIM, f->x[stream],
+            stream_tokens[stream]);
+        if (rc != 1 || !ds4_gpu_end_commands_async()) {
+            fail("async pair submission");
+        }
+        submitted++;
+    }
+    for (uint32_t stream = 0; stream < submitted; stream++) {
+        if (!ds4_gpu_wait_stream((int)stream)) fail("async stream wait");
+    }
+    ds4_gpu_set_stream(0);
+    for (uint32_t stream = 0; stream < TEST_STREAMS; stream++) {
+        check_outputs(f, stream, stream_tokens[stream]);
+        check_x_unchanged(f, stream);
+    }
+    fprintf(stderr,
+            "Metal Q4 prefill pair async streams: PASS streams=2 "
+            "tokens=96,128 scratch_isolation=bitwise\n");
+}
+
+int main(void) {
+    set_pair_controls(false, false, false);
+    set_flag("DS4_METAL_DISABLE_Q4_DENSE_PAIR", false);
+    set_flag("DS4_METAL_DISABLE_CONTIG_F32_F16_COPY", false);
+
+    if (!ds4_gpu_init()) {
+        fprintf(stderr,
+                "test_metal_q4_prefill_pair: SKIP: no Metal device\n");
+        return 0;
+    }
+    if (!ds4_gpu_device_is_pre_m5_apple_silicon()) {
+        fprintf(stderr,
+                "test_metal_q4_prefill_pair: SKIP: requires Apple M1-M4\n");
+        ds4_gpu_cleanup();
+        return 0;
+    }
+
+    fixture f;
+    fixture_init(&f);
+    test_shapes(&f);
+    test_alias_rejection(&f);
+    test_required_failure_is_local(&f);
+    test_two_stream_async(&f);
+    fixture_destroy(&f);
+    fprintf(stderr,
+            "test_metal_q4_prefill_pair PASS tokens=32,64,96,128 "
+            "bitwise=1 require=1 alias=1 ssd_spans=1 streams=2 "
+            "unretained=%s\n",
+            getenv("DS4_METAL_UNRETAINED_COMMAND_BUFFERS") ? "on" : "off");
+    return 0;
+}
+
+#else
+
+int main(void) {
+    fprintf(stderr,
+            "test_metal_q4_prefill_pair: SKIP: Metal requires macOS\n");
+    return 0;
+}
+
+#endif
diff --git a/tests/test_metal_q4_qb_f16_cache.c b/tests/test_metal_q4_qb_f16_cache.c
new file mode 100644
index 0000000000..5f7e4f5828
--- /dev/null
+++ b/tests/test_metal_q4_qb_f16_cache.c
@@ -0,0 +1,2564 @@
+#define _DARWIN_C_SOURCE
+
+/* GGUF-free production-shape oracle for the resident pre-M5 Metal Q4_K
+ * attn_q_b F16 weight sidecar.  The candidate is compared bit-for-bit with
+ * the established Q4_K matmul followed by the exact same head norm/RoPE
+ * entry point used by the production fallback. */
+
+#include "ds4_gpu.h"
+
+#include <errno.h>
+#include <math.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+#include <unistd.h>
+
+bool ds4_log_is_tty(FILE *fp) {
+    (void)fp;
+    return false;
+}
+
+#ifdef __APPLE__
+
+enum {
+    Q4_K_TYPE = 12u,
+    QK_K = 256u,
+    IN_DIM = 1024u,
+    OUT_DIM = 32768u,
+    N_HEAD = 64u,
+    HEAD_DIM = 512u,
+    N_ROT = 64u,
+    MAX_TOKENS = 64u,
+    BLOCKS_PER_ROW = IN_DIM / QK_K,
+    GROUPS_PER_BLOCK = 8u,
+    GROUP_SIZE = 32u,
+    GUARD_FLOATS = 256u,
+    GUARD_HALFS = 256u,
+    TIMING_SAMPLES = 8u,
+    SSD_SOURCE_LEADING = 128u,
+};
+
+typedef struct {
+    uint16_t d;
+    uint16_t dmin;
+    uint8_t scales[12];
+    uint8_t qs[QK_K / 2u];
+} block_q4_K;
+
+static const char *k_disable =
+    "DS4_METAL_DISABLE_Q4_ATTN_Q_B_F16_CACHE";
+static const char *k_enable_ssd_streaming =
+    "DS4_METAL_ENABLE_Q4_ATTN_Q_B_F16_CACHE_WITH_SSD_STREAMING";
+static const char *k_disable_f16_rhs =
+    "DS4_METAL_DISABLE_Q4_ATTN_Q_B_F16_RHS";
+static const char *k_disable_transient_f16 =
+    "DS4_METAL_DISABLE_Q4_ATTN_Q_B_TRANSIENT_F16";
+static const char *k_require =
+    "DS4_METAL_REQUIRE_Q4_ATTN_Q_B_F16_CACHE";
+static const char *k_min_tokens =
+    "DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MIN_TOKENS";
+static const char *k_transient_min_tokens =
+    "DS4_METAL_Q4_ATTN_Q_B_TRANSIENT_F16_MIN_TOKENS";
+static const char *k_cache_mb =
+    "DS4_METAL_Q4_ATTN_Q_B_F16_CACHE_MB";
+static const char *k_timing =
+    "DS4_TEST_METAL_Q4_QB_F16_CACHE_TIMING";
+
+static const uint32_t k_reference_poison = 0x7fc10000u;
+static const uint32_t k_candidate_poison = 0x7fc30000u;
+
+static void fail(const char *what) {
+    fprintf(stderr, "Metal Q4 attn_q_b F16 cache oracle FAIL: %s\n", what);
+    exit(1);
+}
+
+#define CHECK(expr, what) do { if (!(expr)) fail(what); } while (0)
+
+static uint64_t align_up(uint64_t value, uint64_t alignment) {
+    return (value + alignment - 1u) / alignment * alignment;
+}
+
+static void pack_scales(uint8_t packed[12],
+                        const uint8_t scales[GROUPS_PER_BLOCK],
+                        const uint8_t minima[GROUPS_PER_BLOCK]) {
+    memset(packed, 0, 12u);
+    for (uint32_t group = 0; group < 4u; group++) {
+        packed[group] = scales[group] & 63u;
+        packed[group + 4u] = minima[group] & 63u;
+    }
+    for (uint32_t group = 4u; group < GROUPS_PER_BLOCK; group++) {
+        packed[group + 4u] = (scales[group] & 15u) |
+                             ((minima[group] & 15u) << 4u);
+        packed[group - 4u] |= (scales[group] >> 4u) << 6u;
+        packed[group] |= (minima[group] >> 4u) << 6u;
+    }
+}
+
+static void fill_q4_matrix(block_q4_K *matrix) {
+    CHECK(sizeof(block_q4_K) == 144u, "unexpected Q4_K block size");
+
+    for (uint32_t row = 0; row < OUT_DIM; row++) {
+        for (uint32_t block = 0; block < BLOCKS_PER_ROW; block++) {
+            block_q4_K *b = matrix +
+                (uint64_t)row * BLOCKS_PER_ROW + block;
+            const uint32_t key =
+                row * 1009u + block * 313u +
+                (row ^ (block * 17u)) + 29u;
+            uint8_t scales[GROUPS_PER_BLOCK];
+            uint8_t minima[GROUPS_PER_BLOCK];
+
+            for (uint32_t group = 0; group < GROUPS_PER_BLOCK; group++) {
+                scales[group] =
+                    (uint8_t)((key + group * 37u) & 63u);
+                minima[group] =
+                    (uint8_t)((key / 3u + group * 29u) & 63u);
+            }
+            pack_scales(b->scales, scales, minima);
+            for (uint32_t i = 0; i < QK_K / 2u; i++) {
+                b->qs[i] = (uint8_t)(
+                    key + i * 37u + (i >> 2u) * 11u);
+            }
+
+            /* Exercise non-power-of-two half scales and all packed 6-bit
+             * scale/minimum lanes, including the high bits of groups 4--7. */
+            b->d = (uint16_t)(0x1801u + (key & 0x01ffu));
+            b->dmin = (uint16_t)(0x1403u + ((key >> 3u) & 0x01ffu));
+        }
+    }
+}
+
+static uint32_t poison_bits(uint32_t base, uint64_t index) {
+    return base + (uint32_t)(index & 0xffffu);
+}
+
+static void poison_f32(float *values, uint64_t count, uint32_t base) {
+    for (uint64_t i = 0; i < count; i++) {
+        const uint32_t bits = poison_bits(base, i);
+        memcpy(&values[i], &bits, sizeof(bits));
+    }
+}
+
+static uint64_t count_poison_f32_mismatches(const float *values,
+                                            uint64_t begin,
+                                            uint64_t end,
+                                            uint32_t base) {
+    uint64_t mismatches = 0;
+    for (uint64_t i = begin; i < end; i++) {
+        uint32_t bits = 0;
+        memcpy(&bits, &values[i], sizeof(bits));
+        if (bits != poison_bits(base, i)) mismatches++;
+    }
+    return mismatches;
+}
+
+static uint16_t half_poison_bits(uint64_t index) {
+    return (uint16_t)(0x7e00u | (uint16_t)(index & 0x01ffu));
+}
+
+static void poison_f16(uint16_t *values, uint64_t count) {
+    for (uint64_t i = 0; i < count; i++) {
+        values[i] = half_poison_bits(i);
+    }
+}
+
+static uint64_t count_poison_f16_mismatches_range(
+        const uint16_t *values,
+        uint64_t        begin,
+        uint64_t        end) {
+    uint64_t mismatches = 0;
+    for (uint64_t i = begin; i < end; i++) {
+        if (values[i] != half_poison_bits(i)) mismatches++;
+    }
+    return mismatches;
+}
+
+static uint64_t count_poison_f16_mismatches(const uint16_t *values,
+                                            uint64_t count) {
+    return count_poison_f16_mismatches_range(values, 0, count);
+}
+
+static void fill_inputs(float *values) {
+    for (uint32_t token = 0; token < MAX_TOKENS; token++) {
+        for (uint32_t col = 0; col < IN_DIM; col++) {
+            const uint32_t key = token * 131u + col * 17u +
+                                 ((col >> 3u) ^ (token * 29u));
+            /* Exercise values that are not exactly representable as half;
+             * the F16-RHS path must match the legacy per-tile narrowing. */
+            values[(uint64_t)token * IN_DIM + col] =
+                (float)((int)(key % 129u) - 64) / 509.0f;
+        }
+    }
+}
+
+static uint64_t count_bit_mismatches(const float *reference,
+                                     const float *candidate,
+                                     uint64_t count,
+                                     uint64_t *first) {
+    uint64_t mismatches = 0;
+    *first = UINT64_MAX;
+    for (uint64_t i = 0; i < count; i++) {
+        if (memcmp(&reference[i], &candidate[i], sizeof(float)) != 0) {
+            if (*first == UINT64_MAX) *first = i;
+            mismatches++;
+        }
+    }
+    return mismatches;
+}
+
+static int run_reference_at(ds4_gpu_tensor *out,
+                            const void *model,
+                            uint64_t model_bytes,
+                            uint64_t weight_offset,
+                            const ds4_gpu_tensor *x,
+                            uint32_t n_tok) {
+    if (!ds4_gpu_matmul_quant_tensor(
+            out, model, model_bytes, weight_offset, Q4_K_TYPE,
+            IN_DIM, OUT_DIM, x, n_tok)) {
+        return 0;
+    }
+    return ds4_gpu_head_rms_norm_rope_tail_tensor(
+        out, n_tok, N_HEAD, HEAD_DIM, N_ROT,
+        17u, 0u, false,
+        10000.0f, 1.0f, 0.0f, 1.0f, 32.0f, 1.0f, 1.0e-6f);
+}
+
+static int run_reference(ds4_gpu_tensor *out,
+                         const void *model,
+                         uint64_t model_bytes,
+                         const ds4_gpu_tensor *x,
+                         uint32_t n_tok) {
+    return run_reference_at(
+        out, model, model_bytes, 0u, x, n_tok);
+}
+
+static int run_candidate_at(ds4_gpu_tensor *out,
+                            ds4_gpu_tensor *q_half,
+                            const void *model,
+                            uint64_t model_bytes,
+                            uint64_t weight_offset,
+                            const ds4_gpu_tensor *x,
+                            uint32_t n_tok) {
+    return ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor(
+        out, q_half, model, model_bytes, weight_offset, Q4_K_TYPE,
+        IN_DIM, OUT_DIM, x, n_tok, N_HEAD, HEAD_DIM, N_ROT,
+        17u, 0u, false,
+        10000.0f, 1.0f, 0.0f, 1.0f, 32.0f, 1.0f, 1.0e-6f);
+}
+
+static int run_candidate(ds4_gpu_tensor *out,
+                         ds4_gpu_tensor *q_half,
+                         const void *model,
+                         uint64_t model_bytes,
+                         const ds4_gpu_tensor *x,
+                         uint32_t n_tok) {
+    return run_candidate_at(
+        out, q_half, model, model_bytes, 0u, x, n_tok);
+}
+
+static const char *mm_arm_name(ds4_gpu_test_q4_qb_mm_arm arm) {
+    switch (arm) {
+    case DS4_GPU_TEST_Q4_QB_MM_Q4_F32: return "Q4/F32";
+    case DS4_GPU_TEST_Q4_QB_MM_Q4_F16: return "Q4/F16";
+    case DS4_GPU_TEST_Q4_QB_MM_F16_F32: return "F16/F32";
+    case DS4_GPU_TEST_Q4_QB_MM_F16_F16: return "F16/F16";
+    case DS4_GPU_TEST_Q4_QB_MM_Q4_TRANSIENT_F16_F16:
+        return "Q4 transient/F16/F16";
+    default: return "invalid";
+    }
+}
+
+static bool mm_arm_uses_f16_rhs(ds4_gpu_test_q4_qb_mm_arm arm) {
+    switch (arm) {
+    case DS4_GPU_TEST_Q4_QB_MM_Q4_F16:
+    case DS4_GPU_TEST_Q4_QB_MM_F16_F16:
+    case DS4_GPU_TEST_Q4_QB_MM_Q4_TRANSIENT_F16_F16:
+        return true;
+    default:
+        return false;
+    }
+}
+
+static bool mm_arm_is_transient(ds4_gpu_test_q4_qb_mm_arm arm) {
+    return arm == DS4_GPU_TEST_Q4_QB_MM_Q4_TRANSIENT_F16_F16;
+}
+
+static bool mm_arm_is_prepacked_experiment(
+        ds4_gpu_test_q4_qb_mm_arm arm) {
+    return mm_arm_is_transient(arm);
+}
+
+static int run_mm_arm_projection(
+        ds4_gpu_tensor             *out,
+        ds4_gpu_tensor             *rhs_f16,
+        const void                 *model,
+        uint64_t                    model_bytes,
+        const ds4_gpu_tensor       *x,
+        uint32_t                    n_tok,
+        ds4_gpu_test_q4_qb_mm_arm   arm,
+        bool                        materialize_rhs) {
+    return ds4_gpu_test_q4_attn_q_b_mm_variant_tensor(
+        out, rhs_f16, model, model_bytes, 0u, IN_DIM, OUT_DIM,
+        x, n_tok, arm, materialize_rhs);
+}
+
+static int run_mm_arm_with_tail(
+        ds4_gpu_tensor             *out,
+        ds4_gpu_tensor             *rhs_f16,
+        const void                 *model,
+        uint64_t                    model_bytes,
+        const ds4_gpu_tensor       *x,
+        uint32_t                    n_tok,
+        ds4_gpu_test_q4_qb_mm_arm   arm,
+        bool                        materialize_rhs) {
+    if (!run_mm_arm_projection(
+            out, rhs_f16, model, model_bytes, x, n_tok,
+            arm, materialize_rhs)) {
+        return 0;
+    }
+    return ds4_gpu_head_rms_norm_rope_tail_tensor(
+        out, n_tok, N_HEAD, HEAD_DIM, N_ROT,
+        17u, 0u, false,
+        10000.0f, 1.0f, 0.0f, 1.0f, 32.0f, 1.0f, 1.0e-6f);
+}
+
+static void check_cache_storage_unchanged(
+        const ds4_gpu_q4_attn_q_b_f16_cache_report *before,
+        const ds4_gpu_q4_attn_q_b_f16_cache_report *after,
+        const char *what) {
+    if (after->entries != before->entries ||
+        after->bytes != before->bytes ||
+        after->lookups != before->lookups ||
+        after->hits != before->hits ||
+        after->misses != before->misses ||
+        after->builds != before->builds ||
+        after->build_failures != before->build_failures ||
+        after->build_circuit_open != before->build_circuit_open) {
+        fail(what);
+    }
+}
+
+static double monotonic_ms(void) {
+    struct timespec ts;
+    CHECK(clock_gettime(CLOCK_MONOTONIC, &ts) == 0, "monotonic clock");
+    return (double)ts.tv_sec * 1000.0 + (double)ts.tv_nsec / 1.0e6;
+}
+
+static int compare_double(const void *lhs, const void *rhs) {
+    const double a = *(const double *)lhs;
+    const double b = *(const double *)rhs;
+    return (a > b) - (a < b);
+}
+
+static double timing_quantile(const double samples[TIMING_SAMPLES],
+                              double q) {
+    double sorted[TIMING_SAMPLES];
+    memcpy(sorted, samples, sizeof(sorted));
+    qsort(sorted, TIMING_SAMPLES, sizeof(double), compare_double);
+    const double position = q * (double)(TIMING_SAMPLES - 1u);
+    const uint32_t lower = (uint32_t)position;
+    const uint32_t upper = lower + 1u < TIMING_SAMPLES ? lower + 1u : lower;
+    const double fraction = position - (double)lower;
+    return sorted[lower] + (sorted[upper] - sorted[lower]) * fraction;
+}
+
+static double timing_paired_geomean_speedup(
+        const double baseline[TIMING_SAMPLES],
+        const double candidate[TIMING_SAMPLES]) {
+    double log_sum = 0.0;
+    for (uint32_t i = 0; i < TIMING_SAMPLES; i++) {
+        log_sum += log(baseline[i] / candidate[i]);
+    }
+    /* TIMING_SAMPLES is two complete Williams cycles.  The geometric mean
+     * preserves their multiplicative position balancing. */
+    return exp(log_sum / (double)TIMING_SAMPLES);
+}
+
+int main(void) {
+    static const uint32_t token_cases[] = {32u, 33u, 64u};
+    const uint64_t page = (uint64_t)getpagesize();
+    const uint64_t row_bytes =
+        (uint64_t)BLOCKS_PER_ROW * sizeof(block_q4_K);
+    const uint64_t weight_bytes = (uint64_t)OUT_DIM * row_bytes;
+    const uint64_t ssd_weight_offset =
+        2u * weight_bytes + SSD_SOURCE_LEADING;
+    const uint64_t model_bytes = align_up(
+        ssd_weight_offset + weight_bytes, page);
+    const uint64_t support_model_bytes = align_up(weight_bytes, page);
+    const uint64_t f16_cache_bytes =
+        (uint64_t)OUT_DIM * IN_DIM * sizeof(uint16_t);
+    const uint64_t input_count = (uint64_t)MAX_TOKENS * IN_DIM;
+    const uint64_t input_storage_count =
+        GUARD_FLOATS + input_count + GUARD_FLOATS;
+    const uint64_t max_output_count = (uint64_t)MAX_TOKENS * OUT_DIM;
+    const uint64_t output_storage_count =
+        GUARD_FLOATS + max_output_count + GUARD_FLOATS;
+    const uint64_t q_half_storage_count =
+        GUARD_HALFS + max_output_count + GUARD_HALFS;
+
+    CHECK(Q4_K_TYPE == 12u, "Q4_K GGUF type must be 12");
+    CHECK(row_bytes == 576u, "unexpected Q4_K row size");
+    CHECK(weight_bytes == 18u * 1024u * 1024u,
+          "unexpected production q_b Q4_K size");
+    CHECK(ssd_weight_offset % page == SSD_SOURCE_LEADING,
+          "SSD source offset must exercise a non-page-aligned exact view");
+    CHECK(f16_cache_bytes == 64u * 1024u * 1024u,
+          "unexpected production q_b F16 sidecar size");
+    CHECK(ds4_gpu_test_q4_attn_q_b_f16_working_set_policy(
+              0u, 0u, 0u) == 0,
+          "unknown working set must reject");
+    CHECK(ds4_gpu_test_q4_attn_q_b_f16_working_set_policy(
+              800u, 600u, 100u) == 1,
+          "working-set equality boundary");
+    CHECK(ds4_gpu_test_q4_attn_q_b_f16_working_set_policy(
+              800u, 600u, 101u) == 0,
+          "working-set one-byte overflow");
+    CHECK(ds4_gpu_test_q4_attn_q_b_f16_working_set_policy(
+              800u, 701u, 0u) == 0,
+          "allocated working set beyond safety limit");
+    CHECK(ds4_gpu_test_q4_attn_q_b_f16_working_set_policy(
+              UINT64_MAX, 0u, UINT64_MAX) == 0,
+          "working-set overflow-sized request");
+
+    CHECK(unsetenv(k_disable) == 0, "clear cache disable env");
+    CHECK(unsetenv(k_enable_ssd_streaming) == 0,
+          "clear SSD-streaming cache opt-in env");
+    CHECK(unsetenv(k_disable_f16_rhs) == 0,
+          "clear compact F16 RHS disable env");
+    CHECK(unsetenv(k_disable_transient_f16) == 0,
+          "clear transient F16 disable env");
+    CHECK(unsetenv(k_require) == 0, "clear cache require env");
+    CHECK(unsetenv(k_min_tokens) == 0, "clear cache minimum env");
+    CHECK(unsetenv(k_transient_min_tokens) == 0,
+          "clear transient F16 minimum env");
+    CHECK(unsetenv(k_cache_mb) == 0, "clear cache budget env");
+    CHECK(setenv(k_min_tokens, "32", 1) == 0,
+          "set 32-token cache minimum");
+    CHECK(setenv(k_require, "1", 1) == 0,
+          "require Q4 attn_q_b F16 cache");
+
+    CHECK(ds4_gpu_init() != 0, "Metal init");
+    if (!ds4_gpu_device_is_pre_m5_apple_silicon()) {
+        fprintf(stderr,
+                "Metal Q4 attn_q_b F16 cache oracle SKIP: "
+                "requires Apple M1--M4\n");
+        ds4_gpu_cleanup();
+        return 0;
+    }
+
+    void *model = NULL;
+    CHECK(posix_memalign(&model, (size_t)page, (size_t)model_bytes) == 0,
+          "page-aligned model allocation");
+    memset(model, 0, (size_t)model_bytes);
+    fill_q4_matrix(model);
+    fill_q4_matrix((block_q4_K *)((uint8_t *)model + weight_bytes));
+    ((block_q4_K *)((uint8_t *)model + weight_bytes))[0].d ^= 0x001fu;
+    fill_q4_matrix((block_q4_K *)((uint8_t *)model + ssd_weight_offset));
+    ((block_q4_K *)((uint8_t *)model + ssd_weight_offset))[0].d ^= 0x005bu;
+
+    void *support_model = NULL;
+    CHECK(posix_memalign(&support_model, (size_t)page,
+                         (size_t)support_model_bytes) == 0,
+          "page-aligned support model allocation");
+    memset(support_model, 0, (size_t)support_model_bytes);
+    fill_q4_matrix(support_model);
+    ((block_q4_K *)support_model)[0].dmin ^= 0x003du;
+
+    float *input_host = malloc(
+        (size_t)input_storage_count * sizeof(float));
+    float *input_readback = malloc(
+        (size_t)input_storage_count * sizeof(float));
+    float *reference_host = malloc(
+        (size_t)output_storage_count * sizeof(float));
+    float *candidate_host = malloc(
+        (size_t)output_storage_count * sizeof(float));
+    uint16_t *q_half_host = malloc(
+        (size_t)q_half_storage_count * sizeof(uint16_t));
+    CHECK(input_host && input_readback && reference_host &&
+          candidate_host && q_half_host, "host tensor allocation");
+
+    poison_f32(input_host, input_storage_count, 0x7fc50000u);
+    fill_inputs(input_host + GUARD_FLOATS);
+    poison_f16(q_half_host, q_half_storage_count);
+
+    ds4_gpu_tensor *x_base = ds4_gpu_tensor_alloc(
+        input_storage_count * sizeof(float));
+    ds4_gpu_tensor *reference_base = ds4_gpu_tensor_alloc(
+        output_storage_count * sizeof(float));
+    ds4_gpu_tensor *candidate_base = ds4_gpu_tensor_alloc(
+        output_storage_count * sizeof(float));
+    ds4_gpu_tensor *q_half_base = ds4_gpu_tensor_alloc(
+        q_half_storage_count * sizeof(uint16_t));
+    CHECK(x_base && reference_base && candidate_base && q_half_base,
+          "Metal tensor allocation");
+
+    ds4_gpu_tensor *x = ds4_gpu_tensor_view(
+        x_base, GUARD_FLOATS * sizeof(float),
+        input_count * sizeof(float));
+    CHECK(x != NULL, "input tensor view");
+    CHECK(ds4_gpu_tensor_write(
+              x_base, 0, input_host,
+              input_storage_count * sizeof(float)) != 0,
+          "input upload");
+    CHECK(ds4_gpu_tensor_write(
+              q_half_base, 0, q_half_host,
+              q_half_storage_count * sizeof(uint16_t)) != 0,
+          "q_half poison upload");
+
+    /* Residency must be selected before installing the synthetic model. */
+    ds4_gpu_set_quality(false);
+    ds4_gpu_set_ssd_streaming(false);
+    CHECK(ds4_gpu_set_model_map(model, model_bytes) != 0,
+          "resident model map");
+    CHECK(ds4_gpu_prepare_support_model(
+              support_model, support_model_bytes, 0,
+              support_model_bytes, 0) != 0,
+          "resident support model map");
+    ds4_gpu_test_q4_attn_q_b_f16_cache_reset();
+
+    /* REQUIRE applies only at or above MIN_TOKENS.  A short tail must remain
+     * a non-candidate rather than failing after a successful session prewarm. */
+    {
+        CHECK(setenv(k_min_tokens, "512", 1) == 0,
+              "set short-tail cache minimum");
+        ds4_gpu_tensor *short_out = ds4_gpu_tensor_view(
+            candidate_base, GUARD_FLOATS * sizeof(float),
+            (uint64_t)32u * OUT_DIM * sizeof(float));
+        ds4_gpu_tensor *short_half = ds4_gpu_tensor_view(
+            q_half_base, GUARD_HALFS * sizeof(uint16_t),
+            (uint64_t)32u * OUT_DIM * sizeof(uint16_t));
+        CHECK(short_out && short_half, "short-tail tensor views");
+        CHECK(run_candidate(short_out, short_half, model, model_bytes,
+                            x, 32u) == 0,
+              "below-min REQUIRE batch must remain a non-candidate");
+        ds4_gpu_tensor_free(short_half);
+        ds4_gpu_tensor_free(short_out);
+
+        ds4_gpu_q4_attn_q_b_f16_cache_report short_report;
+        ds4_gpu_test_q4_attn_q_b_f16_cache_report(&short_report);
+        CHECK(short_report.candidate_calls == 0u &&
+                  short_report.lookups == 0u &&
+                  short_report.fallbacks == 0u &&
+                  short_report.rejects == 0u &&
+                  short_report.build_circuit_open == 0u,
+              "below-min REQUIRE candidate accounting");
+        CHECK(setenv(k_min_tokens, "32", 1) == 0,
+              "restore 32-token cache minimum");
+    }
+
+    /* A stable admission failure opens the build circuit.  Raising the
+     * logical budget alone must not trigger repeated allocations; an
+     * explicit cache reset is required before builds may resume. */
+    {
+        const uint64_t gate_output_count = (uint64_t)32u * OUT_DIM;
+        const uint64_t gate_output_bytes =
+            gate_output_count * sizeof(float);
+        const uint64_t gate_half_bytes =
+            gate_output_count * sizeof(uint16_t);
+        CHECK(setenv(k_cache_mb, "63", 1) == 0,
+              "set undersized sidecar budget");
+        poison_f32(candidate_host, output_storage_count,
+                   k_candidate_poison);
+        poison_f16(q_half_host, q_half_storage_count);
+        CHECK(ds4_gpu_tensor_write(
+                  candidate_base, 0, candidate_host,
+                  output_storage_count * sizeof(float)) != 0,
+              "budget output poison upload");
+        CHECK(ds4_gpu_tensor_write(
+                  q_half_base, 0, q_half_host,
+                  q_half_storage_count * sizeof(uint16_t)) != 0,
+              "budget q_half poison upload");
+
+        ds4_gpu_tensor *budget_out = ds4_gpu_tensor_view(
+            candidate_base, GUARD_FLOATS * sizeof(float),
+            gate_output_bytes);
+        ds4_gpu_tensor *budget_half = ds4_gpu_tensor_view(
+            q_half_base, GUARD_HALFS * sizeof(uint16_t),
+            gate_half_bytes);
+        CHECK(budget_out && budget_half, "budget tensor views");
+        CHECK(run_candidate(budget_out, budget_half, model, model_bytes,
+                            x, 32u) == -1,
+              "undersized budget must fail required candidate");
+        ds4_gpu_tensor_free(budget_half);
+        ds4_gpu_tensor_free(budget_out);
+
+        ds4_gpu_q4_attn_q_b_f16_cache_report budget_report;
+        ds4_gpu_test_q4_attn_q_b_f16_cache_report(&budget_report);
+        CHECK(budget_report.entries == 0u && budget_report.bytes == 0u &&
+                  budget_report.builds == 0u &&
+                  budget_report.build_failures == 0u &&
+                  budget_report.rejects == 1u &&
+                  budget_report.build_circuit_open == 1u,
+              "undersized budget circuit accounting");
+
+        CHECK(setenv(k_cache_mb, "3072", 1) == 0,
+              "raise sidecar budget without reset");
+        budget_out = ds4_gpu_tensor_view(
+            candidate_base, GUARD_FLOATS * sizeof(float),
+            gate_output_bytes);
+        budget_half = ds4_gpu_tensor_view(
+            q_half_base, GUARD_HALFS * sizeof(uint16_t),
+            gate_half_bytes);
+        CHECK(budget_out && budget_half,
+              "open-circuit tensor views");
+        CHECK(run_candidate(budget_out, budget_half, model, model_bytes,
+                            x, 32u) == -1,
+              "open circuit must suppress budget-only retry");
+        ds4_gpu_tensor_free(budget_half);
+        ds4_gpu_tensor_free(budget_out);
+        ds4_gpu_test_q4_attn_q_b_f16_cache_report(&budget_report);
+        CHECK(budget_report.entries == 0u && budget_report.builds == 0u &&
+                  budget_report.build_failures == 0u &&
+                  budget_report.rejects == 2u &&
+                  budget_report.build_circuit_open == 1u,
+              "open circuit retry accounting");
+
+        CHECK(ds4_gpu_tensor_read(
+                  candidate_base, 0, candidate_host,
+                  output_storage_count * sizeof(float)) != 0,
+              "budget output readback");
+        CHECK(ds4_gpu_tensor_read(
+                  q_half_base, 0, q_half_host,
+                  q_half_storage_count * sizeof(uint16_t)) != 0,
+              "budget q_half readback");
+        CHECK(count_poison_f32_mismatches(
+                  candidate_host, 0, output_storage_count,
+                  k_candidate_poison) == 0,
+              "budget rejection touched output");
+        CHECK(count_poison_f16_mismatches(
+                  q_half_host, q_half_storage_count) == 0,
+              "budget rejection touched q_half");
+
+        CHECK(ds4_gpu_release_q4_attn_q_b_f16_sidecars() != 0,
+              "production lifecycle release after open circuit");
+        ds4_gpu_test_q4_attn_q_b_f16_cache_report(&budget_report);
+        CHECK(budget_report.entries == 0u && budget_report.bytes == 0u &&
+                  budget_report.build_circuit_open == 0u,
+              "lifecycle release did not close empty build circuit");
+        ds4_gpu_test_q4_attn_q_b_f16_cache_reset();
+        CHECK(unsetenv(k_cache_mb) == 0,
+              "clear sidecar budget override");
+    }
+
+    /* Session creation uses the batch-prewarm API before prefill timing.
+     * Exercise its transactional publication independently from lazy use. */
+    {
+        const ds4_gpu_q4_attn_q_b_f16_sidecar_desc descs[2] = {
+            {
+                .weight_offset = 0,
+                .weight_bytes = weight_bytes,
+                .in_dim = IN_DIM,
+                .out_dim = OUT_DIM,
+                .weight_type = Q4_K_TYPE,
+                .layer = 0,
+            },
+            {
+                .weight_offset = weight_bytes,
+                .weight_bytes = weight_bytes,
+                .in_dim = IN_DIM,
+                .out_dim = OUT_DIM,
+                .weight_type = Q4_K_TYPE,
+                .layer = 1,
+            },
+        };
+        const ds4_gpu_q4_attn_q_b_f16_sidecar_desc support_desc = {
+            .weight_offset = 0,
+            .weight_bytes = weight_bytes,
+            .in_dim = IN_DIM,
+            .out_dim = OUT_DIM,
+            .weight_type = Q4_K_TYPE,
+            .layer = 0,
+        };
+        uint64_t prepared_bytes = 0;
+        CHECK(setenv(k_cache_mb, "127", 1) == 0,
+              "set undersized transactional prewarm budget");
+        CHECK(ds4_gpu_prepare_q4_attn_q_b_f16_sidecars(
+                  model, model_bytes, descs, 2u, 64u, 0u,
+                  &prepared_bytes) == -1,
+              "transactional prewarm budget rejection");
+        ds4_gpu_q4_attn_q_b_f16_cache_report prewarm_report;
+        ds4_gpu_test_q4_attn_q_b_f16_cache_report(&prewarm_report);
+        CHECK(prepared_bytes == 0u && prewarm_report.entries == 0u &&
+                  prewarm_report.bytes == 0u &&
+                  prewarm_report.builds == 0u &&
+                  prewarm_report.build_circuit_open == 1u,
+              "transactional prewarm published a partial cache");
+        ds4_gpu_test_q4_attn_q_b_f16_cache_reset();
+        CHECK(unsetenv(k_cache_mb) == 0,
+              "clear transactional prewarm budget");
+
+        CHECK(ds4_gpu_prepare_q4_attn_q_b_f16_sidecars(
+                  model, model_bytes, descs, 2u, 64u, 0u,
+                  &prepared_bytes) == 1,
+              "transactional two-offset prewarm");
+        ds4_gpu_test_q4_attn_q_b_f16_cache_report(&prewarm_report);
+        CHECK(prepared_bytes == 2u * f16_cache_bytes &&
+                  prewarm_report.entries == 2u &&
+                  prewarm_report.bytes == 2u * f16_cache_bytes &&
+                  prewarm_report.builds == 2u &&
+                  prewarm_report.build_circuit_open == 0u,
+              "two-offset prewarm publication accounting");
+
+        prepared_bytes = 0;
+        CHECK(ds4_gpu_prepare_q4_attn_q_b_f16_sidecars(
+                  support_model, support_model_bytes, &support_desc,
+                  1u, 64u, 0u, &prepared_bytes) == 1,
+              "same-offset second-model prewarm");
+        ds4_gpu_test_q4_attn_q_b_f16_cache_report(&prewarm_report);
+        CHECK(prepared_bytes == f16_cache_bytes &&
+                  prewarm_report.entries == 3u &&
+                  prewarm_report.bytes == 3u * f16_cache_bytes &&
+                  prewarm_report.builds == 3u &&
+                  prewarm_report.build_circuit_open == 0u,
+              "model-map cache-key isolation");
+        const uint64_t generation_before_eviction =
+            ds4_gpu_q4_attn_q_b_f16_cache_generation();
+        CHECK(ds4_gpu_make_room_for_q4_attn_q_b_f16_session() != 0,
+              "session admission cache eviction");
+        CHECK(ds4_gpu_q4_attn_q_b_f16_cache_generation() !=
+                  generation_before_eviction,
+              "session admission did not advance cache generation");
+        ds4_gpu_test_q4_attn_q_b_f16_cache_report(&prewarm_report);
+        CHECK(prewarm_report.entries == 0u &&
+                  prewarm_report.bytes == 0u &&
+                  prewarm_report.build_circuit_open == 0u,
+              "session admission retained resident sidecars");
+        ds4_gpu_test_q4_attn_q_b_f16_cache_reset();
+    }
+
+    for (uint32_t case_i = 0;
+         case_i < sizeof(token_cases) / sizeof(token_cases[0]);
+         case_i++) {
+        const uint32_t n_tok = token_cases[case_i];
+        const uint64_t output_count = (uint64_t)n_tok * OUT_DIM;
+        const uint64_t output_bytes = output_count * sizeof(float);
+        const uint64_t q_half_count = (uint64_t)n_tok * OUT_DIM;
+        const uint64_t q_half_bytes = q_half_count * sizeof(uint16_t);
+
+        poison_f32(reference_host, output_storage_count,
+                   k_reference_poison);
+        poison_f32(candidate_host, output_storage_count,
+                   k_candidate_poison);
+        poison_f16(q_half_host, q_half_storage_count);
+        CHECK(ds4_gpu_tensor_write(
+                  reference_base, 0, reference_host,
+                  output_storage_count * sizeof(float)) != 0,
+              "reference poison upload");
+        CHECK(ds4_gpu_tensor_write(
+                  candidate_base, 0, candidate_host,
+                  output_storage_count * sizeof(float)) != 0,
+              "candidate poison upload");
+        CHECK(ds4_gpu_tensor_write(
+                  q_half_base, 0, q_half_host,
+                  q_half_storage_count * sizeof(uint16_t)) != 0,
+              "q_half poison refresh");
+
+        ds4_gpu_tensor *reference = ds4_gpu_tensor_view(
+            reference_base, GUARD_FLOATS * sizeof(float), output_bytes);
+        ds4_gpu_tensor *candidate = ds4_gpu_tensor_view(
+            candidate_base, GUARD_FLOATS * sizeof(float), output_bytes);
+        ds4_gpu_tensor *q_half = ds4_gpu_tensor_view(
+            q_half_base, GUARD_HALFS * sizeof(uint16_t), q_half_bytes);
+        CHECK(reference && candidate && q_half, "case tensor views");
+
+        CHECK(run_reference(reference, model, model_bytes, x, n_tok) == 1,
+              "baseline Q4 projection plus head norm/RoPE");
+        CHECK(run_candidate(
+                  candidate, q_half, model, model_bytes, x, n_tok) == 1,
+              "required cached-F16 candidate");
+
+        ds4_gpu_tensor_free(q_half);
+        ds4_gpu_tensor_free(candidate);
+        ds4_gpu_tensor_free(reference);
+
+        CHECK(ds4_gpu_tensor_read(
+                  reference_base, 0, reference_host,
+                  output_storage_count * sizeof(float)) != 0,
+              "reference readback");
+        CHECK(ds4_gpu_tensor_read(
+                  candidate_base, 0, candidate_host,
+                  output_storage_count * sizeof(float)) != 0,
+              "candidate readback");
+        CHECK(ds4_gpu_tensor_read(
+                  q_half_base, 0, q_half_host,
+                  q_half_storage_count * sizeof(uint16_t)) != 0,
+              "q_half readback");
+
+        uint64_t first = UINT64_MAX;
+        const uint64_t bit_mismatches = count_bit_mismatches(
+            reference_host + GUARD_FLOATS,
+            candidate_host + GUARD_FLOATS,
+            output_count, &first);
+        const uint64_t reference_prefix = count_poison_f32_mismatches(
+            reference_host, 0, GUARD_FLOATS, k_reference_poison);
+        const uint64_t reference_suffix = count_poison_f32_mismatches(
+            reference_host, GUARD_FLOATS + output_count,
+            output_storage_count, k_reference_poison);
+        const uint64_t candidate_prefix = count_poison_f32_mismatches(
+            candidate_host, 0, GUARD_FLOATS, k_candidate_poison);
+        const uint64_t candidate_suffix = count_poison_f32_mismatches(
+            candidate_host, GUARD_FLOATS + output_count,
+            output_storage_count, k_candidate_poison);
+        const uint64_t q_half_rhs_count = (uint64_t)n_tok * IN_DIM;
+        const uint64_t q_half_written =
+            count_poison_f16_mismatches_range(
+                q_half_host,
+                GUARD_HALFS,
+                GUARD_HALFS + q_half_rhs_count);
+        const uint64_t q_half_prefix =
+            count_poison_f16_mismatches_range(
+                q_half_host, 0, GUARD_HALFS);
+        const uint64_t q_half_tail =
+            count_poison_f16_mismatches_range(
+                q_half_host,
+                GUARD_HALFS + q_half_rhs_count,
+                q_half_storage_count);
+
+        fprintf(stderr,
+                "Metal Q4 attn_q_b F16 cache N=%u bitwise=%llu "
+                "ref_guard=%llu/%llu candidate_guard=%llu/%llu "
+                "q_half_written=%llu guard=%llu/%llu\n",
+                n_tok,
+                (unsigned long long)bit_mismatches,
+                (unsigned long long)reference_prefix,
+                (unsigned long long)reference_suffix,
+                (unsigned long long)candidate_prefix,
+                (unsigned long long)candidate_suffix,
+                (unsigned long long)q_half_written,
+                (unsigned long long)q_half_prefix,
+                (unsigned long long)q_half_tail);
+        if (first != UINT64_MAX) {
+            fprintf(stderr,
+                    "  first bitwise mismatch token=%llu row=%llu\n",
+                    (unsigned long long)(first / OUT_DIM),
+                    (unsigned long long)(first % OUT_DIM));
+        }
+        CHECK(bit_mismatches == 0, "candidate bitwise mismatch");
+        CHECK(reference_prefix == 0 && reference_suffix == 0,
+              "reference output canary");
+        CHECK(candidate_prefix == 0 && candidate_suffix == 0,
+              "candidate output canary");
+        CHECK(q_half_written == q_half_rhs_count,
+              "Q4 candidate did not materialize the complete F16 RHS");
+        CHECK(q_half_prefix == 0 && q_half_tail == 0,
+              "Q4 candidate F16 RHS scratch canary");
+
+        ds4_gpu_q4_attn_q_b_f16_cache_report report;
+        ds4_gpu_test_q4_attn_q_b_f16_cache_report(&report);
+        CHECK(report.entries == 1u, "one cache entry");
+        CHECK(report.bytes == f16_cache_bytes, "64 MiB cache entry");
+        CHECK(report.lookups == (uint64_t)case_i + 1u,
+              "cache lookup count");
+        CHECK(report.hits == (uint64_t)case_i,
+              "cache hit count");
+        CHECK(report.misses == 1u, "single cold cache miss");
+        CHECK(report.builds == 1u, "single cache build");
+        CHECK(report.build_failures == 0u, "no cache build failure");
+        CHECK(report.candidate_calls == (uint64_t)case_i + 1u,
+              "candidate call count");
+        CHECK(report.fallbacks == 0u && report.rejects == 0u,
+              "no correctness-case fallback");
+    }
+
+    /* Compact RHS staging is subordinate to the resident sidecar.  Invalid
+     * scratch must use F16 weights with the original F32 RHS, even when the
+     * sidecar itself is required, rather than rejecting or replaying Q4. */
+    {
+        const uint32_t n_tok = 33u;
+        const uint64_t output_count = (uint64_t)n_tok * OUT_DIM;
+        const uint64_t output_bytes = output_count * sizeof(float);
+        const uint64_t compact_rhs_bytes =
+            (uint64_t)n_tok * IN_DIM * sizeof(uint16_t);
+
+        poison_f32(reference_host, output_storage_count,
+                   k_reference_poison);
+        poison_f16(q_half_host, q_half_storage_count);
+        CHECK(ds4_gpu_tensor_write(
+                  reference_base, 0, reference_host,
+                  output_storage_count * sizeof(float)) != 0,
+              "nested fallback reference poison upload");
+        CHECK(ds4_gpu_tensor_write(
+                  q_half_base, 0, q_half_host,
+                  q_half_storage_count * sizeof(uint16_t)) != 0,
+              "nested fallback scratch poison upload");
+
+        ds4_gpu_tensor *reference = ds4_gpu_tensor_view(
+            reference_base, GUARD_FLOATS * sizeof(float), output_bytes);
+        ds4_gpu_tensor *candidate = ds4_gpu_tensor_view(
+            candidate_base, GUARD_FLOATS * sizeof(float), output_bytes);
+        ds4_gpu_tensor *short_half = ds4_gpu_tensor_view(
+            q_half_base, GUARD_HALFS * sizeof(uint16_t),
+            compact_rhs_bytes - sizeof(uint16_t));
+        CHECK(reference && candidate && short_half,
+              "nested fallback tensor views");
+        CHECK(run_reference(reference, model, model_bytes, x, n_tok) == 1,
+              "nested fallback reference");
+        CHECK(ds4_gpu_tensor_read(
+                  reference_base, 0, reference_host,
+                  output_storage_count * sizeof(float)) != 0,
+              "nested fallback reference readback");
+
+        ds4_gpu_tensor *scratch_cases[] = {x, short_half};
+        const char *scratch_names[] = {"alias", "undersized"};
+        for (uint32_t scratch_i = 0; scratch_i < 2u; scratch_i++) {
+            poison_f32(candidate_host, output_storage_count,
+                       k_candidate_poison);
+            CHECK(ds4_gpu_tensor_write(
+                      candidate_base, 0, candidate_host,
+                      output_storage_count * sizeof(float)) != 0,
+                  "nested fallback candidate poison upload");
+            CHECK(run_candidate(
+                      candidate, scratch_cases[scratch_i],
+                      model, model_bytes, x, n_tok) == 1,
+                  "nested F16/F32 fallback candidate");
+            CHECK(ds4_gpu_tensor_read(
+                      candidate_base, 0, candidate_host,
+                      output_storage_count * sizeof(float)) != 0,
+                  "nested fallback candidate readback");
+            uint64_t first = UINT64_MAX;
+            CHECK(count_bit_mismatches(
+                      reference_host + GUARD_FLOATS,
+                      candidate_host + GUARD_FLOATS,
+                      output_count, &first) == 0,
+                  "nested F16/F32 fallback bitwise mismatch");
+            CHECK(count_poison_f32_mismatches(
+                      candidate_host, 0, GUARD_FLOATS,
+                      k_candidate_poison) == 0 &&
+                  count_poison_f32_mismatches(
+                      candidate_host, GUARD_FLOATS + output_count,
+                      output_storage_count, k_candidate_poison) == 0,
+                  "nested fallback output canary");
+            fprintf(stderr,
+                    "Metal Q4 attn_q_b compact RHS %s fallback: PASS\n",
+                    scratch_names[scratch_i]);
+        }
+        CHECK(ds4_gpu_tensor_read(
+                  q_half_base, 0, q_half_host,
+                  q_half_storage_count * sizeof(uint16_t)) != 0,
+              "nested fallback scratch readback");
+        CHECK(count_poison_f16_mismatches(
+                  q_half_host, q_half_storage_count) == 0,
+              "undersized compact RHS fallback touched scratch");
+
+        ds4_gpu_tensor_free(short_half);
+        ds4_gpu_tensor_free(candidate);
+        ds4_gpu_tensor_free(reference);
+    }
+
+    /* Compare the raw projection before RMSNorm/RoPE so normalization cannot
+     * hide a uniform scale or dequantization error. */
+    {
+        const uint32_t n_tok = 33u;
+        const uint64_t output_count = (uint64_t)n_tok * OUT_DIM;
+        const uint64_t output_bytes = output_count * sizeof(float);
+        poison_f32(reference_host, output_storage_count,
+                   k_reference_poison);
+        poison_f32(candidate_host, output_storage_count,
+                   k_candidate_poison);
+        CHECK(ds4_gpu_tensor_write(
+                  reference_base, 0, reference_host,
+                  output_storage_count * sizeof(float)) != 0,
+              "raw reference poison upload");
+        CHECK(ds4_gpu_tensor_write(
+                  candidate_base, 0, candidate_host,
+                  output_storage_count * sizeof(float)) != 0,
+              "raw candidate poison upload");
+        ds4_gpu_tensor *reference = ds4_gpu_tensor_view(
+            reference_base, GUARD_FLOATS * sizeof(float), output_bytes);
+        ds4_gpu_tensor *candidate = ds4_gpu_tensor_view(
+            candidate_base, GUARD_FLOATS * sizeof(float), output_bytes);
+        CHECK(reference && candidate, "raw projection tensor views");
+        CHECK(ds4_gpu_matmul_quant_tensor(
+                  reference, model, model_bytes, 0, Q4_K_TYPE,
+                  IN_DIM, OUT_DIM, x, n_tok) == 1,
+              "raw native Q4 projection");
+        CHECK(ds4_gpu_test_q4_attn_q_b_f16_projection_tensor(
+                  candidate, model, model_bytes, 0,
+                  IN_DIM, OUT_DIM, x, n_tok) == 1,
+              "raw cached-F16 projection");
+        ds4_gpu_tensor_free(candidate);
+        ds4_gpu_tensor_free(reference);
+        CHECK(ds4_gpu_tensor_read(
+                  reference_base, 0, reference_host,
+                  output_storage_count * sizeof(float)) != 0,
+              "raw reference readback");
+        CHECK(ds4_gpu_tensor_read(
+                  candidate_base, 0, candidate_host,
+                  output_storage_count * sizeof(float)) != 0,
+              "raw candidate readback");
+        uint64_t first = UINT64_MAX;
+        CHECK(count_bit_mismatches(
+                  reference_host + GUARD_FLOATS,
+                  candidate_host + GUARD_FLOATS,
+                  output_count, &first) == 0,
+              "raw projection bitwise mismatch");
+        CHECK(count_poison_f32_mismatches(
+                  reference_host, 0, GUARD_FLOATS,
+                  k_reference_poison) == 0 &&
+              count_poison_f32_mismatches(
+                  reference_host, GUARD_FLOATS + output_count,
+                  output_storage_count, k_reference_poison) == 0,
+              "raw reference output canary");
+        CHECK(count_poison_f32_mismatches(
+                  candidate_host, 0, GUARD_FLOATS,
+                  k_candidate_poison) == 0 &&
+              count_poison_f32_mismatches(
+                  candidate_host, GUARD_FLOATS + output_count,
+                  output_storage_count, k_candidate_poison) == 0,
+              "raw candidate output canary");
+        fprintf(stderr,
+                "Metal Q4 attn_q_b F16 cache raw projection N=33: PASS\n");
+
+        /* Keep the boundary raw oracle in the default (non-timing) target as
+         * well: the tail must not be able to mask a projection discrepancy. */
+        static const ds4_gpu_test_q4_qb_mm_arm raw_arms[] = {
+            DS4_GPU_TEST_Q4_QB_MM_Q4_F16,
+            DS4_GPU_TEST_Q4_QB_MM_F16_F32,
+            DS4_GPU_TEST_Q4_QB_MM_F16_F16,
+        };
+        ds4_gpu_tensor *raw_rhs_f16 = ds4_gpu_tensor_view(
+            q_half_base, GUARD_HALFS * sizeof(uint16_t),
+            (uint64_t)n_tok * IN_DIM * sizeof(uint16_t));
+        CHECK(raw_rhs_f16 != NULL, "raw four-way F16 RHS view");
+        const uint64_t raw_rhs_count = (uint64_t)n_tok * IN_DIM;
+        for (uint32_t arm_i = 0;
+             arm_i < sizeof(raw_arms) / sizeof(raw_arms[0]);
+             arm_i++) {
+            const bool rhs_is_f16 =
+                mm_arm_uses_f16_rhs(raw_arms[arm_i]);
+            const uint32_t rhs_modes = rhs_is_f16 ? 2u : 1u;
+            for (uint32_t rhs_mode = 0; rhs_mode < rhs_modes;
+                 rhs_mode++) {
+                const bool materialize_rhs = rhs_mode == 0u;
+                if (rhs_is_f16) {
+                    poison_f16(q_half_host, q_half_storage_count);
+                    CHECK(ds4_gpu_tensor_write(
+                              q_half_base, 0, q_half_host,
+                              q_half_storage_count * sizeof(uint16_t)) != 0,
+                          "raw four-way RHS poison upload");
+                    if (!materialize_rhs) {
+                        CHECK(ds4_gpu_tensor_copy_f32_to_f16(
+                                  raw_rhs_f16, 0u, x, 0u,
+                                  raw_rhs_count) != 0,
+                              "raw four-way explicit RHS prepack");
+                    }
+                }
+
+                poison_f32(candidate_host, output_storage_count,
+                           k_candidate_poison);
+                CHECK(ds4_gpu_tensor_write(
+                          candidate_base, 0, candidate_host,
+                          output_storage_count * sizeof(float)) != 0,
+                      "raw four-way candidate poison upload");
+                candidate = ds4_gpu_tensor_view(
+                    candidate_base, GUARD_FLOATS * sizeof(float),
+                    output_bytes);
+                CHECK(candidate != NULL, "raw four-way candidate view");
+                CHECK(run_mm_arm_projection(
+                          candidate, raw_rhs_f16,
+                          model, model_bytes, x, n_tok,
+                          raw_arms[arm_i], materialize_rhs) == 1,
+                      "raw four-way projection");
+                ds4_gpu_tensor_free(candidate);
+                CHECK(ds4_gpu_tensor_read(
+                          candidate_base, 0, candidate_host,
+                          output_storage_count * sizeof(float)) != 0,
+                      "raw four-way candidate readback");
+                first = UINT64_MAX;
+                CHECK(count_bit_mismatches(
+                          reference_host + GUARD_FLOATS,
+                          candidate_host + GUARD_FLOATS,
+                          output_count, &first) == 0,
+                      "raw four-way bitwise mismatch");
+                CHECK(count_poison_f32_mismatches(
+                          candidate_host, 0, GUARD_FLOATS,
+                          k_candidate_poison) == 0 &&
+                      count_poison_f32_mismatches(
+                          candidate_host, GUARD_FLOATS + output_count,
+                          output_storage_count, k_candidate_poison) == 0,
+                      "raw four-way candidate canary");
+                if (rhs_is_f16) {
+                    CHECK(ds4_gpu_tensor_read(
+                              q_half_base, 0, q_half_host,
+                              q_half_storage_count * sizeof(uint16_t)) != 0,
+                          "raw four-way RHS readback");
+                    CHECK(count_poison_f16_mismatches_range(
+                              q_half_host, GUARD_HALFS,
+                              GUARD_HALFS + raw_rhs_count) ==
+                              raw_rhs_count,
+                          "raw four-way RHS payload was not materialized");
+                    CHECK(count_poison_f16_mismatches_range(
+                              q_half_host, 0, GUARD_HALFS) == 0 &&
+                          count_poison_f16_mismatches_range(
+                              q_half_host,
+                              GUARD_HALFS + raw_rhs_count,
+                              q_half_storage_count) == 0,
+                          "raw four-way RHS canary");
+                }
+                fprintf(stderr,
+                        "Metal Q4 attn_q_b raw N=33 %s %s: PASS\n",
+                        mm_arm_name(raw_arms[arm_i]),
+                        rhs_is_f16
+                            ? (materialize_rhs
+                                ? "with-pack" : "prepacked")
+                            : "control");
+            }
+        }
+        ds4_gpu_tensor_free(raw_rhs_f16);
+    }
+
+    /* Exercise the production command-batch lifecycle from a cold cache.
+     * The sidecar build must complete before publication even though the
+     * projection and norm/RoPE remain encoded in the caller's batch.  A
+     * second batch then proves that the published entry is a real hot hit. */
+    {
+        const uint32_t n_tok = 64u;
+        const uint64_t output_count = (uint64_t)n_tok * OUT_DIM;
+        const uint64_t output_bytes = output_count * sizeof(float);
+        const uint64_t q_half_bytes =
+            output_count * sizeof(uint16_t);
+
+        ds4_gpu_test_q4_attn_q_b_f16_cache_reset();
+        poison_f32(reference_host, output_storage_count,
+                   k_reference_poison);
+        poison_f32(candidate_host, output_storage_count,
+                   k_candidate_poison);
+        CHECK(ds4_gpu_tensor_write(
+                  reference_base, 0, reference_host,
+                  output_storage_count * sizeof(float)) != 0,
+              "batch reference poison upload");
+        CHECK(ds4_gpu_tensor_write(
+                  candidate_base, 0, candidate_host,
+                  output_storage_count * sizeof(float)) != 0,
+              "batch candidate poison upload");
+
+        ds4_gpu_tensor *reference = ds4_gpu_tensor_view(
+            reference_base, GUARD_FLOATS * sizeof(float), output_bytes);
+        ds4_gpu_tensor *candidate = ds4_gpu_tensor_view(
+            candidate_base, GUARD_FLOATS * sizeof(float), output_bytes);
+        ds4_gpu_tensor *q_half = ds4_gpu_tensor_view(
+            q_half_base, GUARD_HALFS * sizeof(uint16_t), q_half_bytes);
+        CHECK(reference && candidate && q_half,
+              "cold batch tensor views");
+        CHECK(run_reference(reference, model, model_bytes, x, n_tok) == 1,
+              "cold batch reference");
+        CHECK(ds4_gpu_begin_commands() != 0,
+              "begin cold candidate batch");
+        CHECK(run_candidate(candidate, q_half, model, model_bytes,
+                            x, n_tok) == 1,
+              "encode cold batched candidate");
+        CHECK(ds4_gpu_end_commands() != 0,
+              "finish cold candidate batch");
+        ds4_gpu_tensor_free(q_half);
+        ds4_gpu_tensor_free(candidate);
+        ds4_gpu_tensor_free(reference);
+
+        CHECK(ds4_gpu_tensor_read(
+                  reference_base, 0, reference_host,
+                  output_storage_count * sizeof(float)) != 0,
+              "cold batch reference readback");
+        CHECK(ds4_gpu_tensor_read(
+                  candidate_base, 0, candidate_host,
+                  output_storage_count * sizeof(float)) != 0,
+              "cold batch candidate readback");
+        uint64_t first = UINT64_MAX;
+        CHECK(count_bit_mismatches(
+                  reference_host + GUARD_FLOATS,
+                  candidate_host + GUARD_FLOATS,
+                  output_count, &first) == 0,
+              "cold batched candidate bitwise mismatch");
+
+        ds4_gpu_q4_attn_q_b_f16_cache_report report;
+        ds4_gpu_test_q4_attn_q_b_f16_cache_report(&report);
+        CHECK(report.entries == 1u && report.builds == 1u &&
+                  report.misses == 1u && report.hits == 0u,
+              "cold batch cache publication");
+
+        poison_f32(candidate_host, output_storage_count,
+                   k_candidate_poison);
+        CHECK(ds4_gpu_tensor_write(
+                  candidate_base, 0, candidate_host,
+                  output_storage_count * sizeof(float)) != 0,
+              "hot batch candidate poison upload");
+        candidate = ds4_gpu_tensor_view(
+            candidate_base, GUARD_FLOATS * sizeof(float), output_bytes);
+        q_half = ds4_gpu_tensor_view(
+            q_half_base, GUARD_HALFS * sizeof(uint16_t), q_half_bytes);
+        CHECK(candidate && q_half, "hot batch tensor views");
+        CHECK(ds4_gpu_begin_commands() != 0,
+              "begin hot candidate batch");
+        CHECK(run_candidate(candidate, q_half, model, model_bytes,
+                            x, n_tok) == 1,
+              "encode hot batched candidate");
+        CHECK(ds4_gpu_end_commands() != 0,
+              "finish hot candidate batch");
+        ds4_gpu_tensor_free(q_half);
+        ds4_gpu_tensor_free(candidate);
+        CHECK(ds4_gpu_tensor_read(
+                  candidate_base, 0, candidate_host,
+                  output_storage_count * sizeof(float)) != 0,
+              "hot batch candidate readback");
+        first = UINT64_MAX;
+        CHECK(count_bit_mismatches(
+                  reference_host + GUARD_FLOATS,
+                  candidate_host + GUARD_FLOATS,
+                  output_count, &first) == 0,
+              "hot batched candidate bitwise mismatch");
+        ds4_gpu_test_q4_attn_q_b_f16_cache_report(&report);
+        CHECK(report.entries == 1u && report.builds == 1u &&
+                  report.misses == 1u && report.hits == 1u &&
+                  report.lookups == 2u,
+              "hot batch cache hit");
+        fprintf(stderr,
+                "Metal Q4 attn_q_b F16 cache cold/hot command batch: PASS\n");
+    }
+
+    /* DISABLE wins over REQUIRE, must reject before cache lookup, and must
+     * leave the entire output allocation untouched. */
+    ds4_gpu_q4_attn_q_b_f16_cache_report gate_before;
+    ds4_gpu_q4_attn_q_b_f16_cache_report gate_after;
+    const ds4_gpu_q4_attn_q_b_f16_sidecar_desc ssd_desc = {
+        .weight_offset = ssd_weight_offset,
+        .weight_bytes = weight_bytes,
+        .in_dim = IN_DIM,
+        .out_dim = OUT_DIM,
+        .weight_type = Q4_K_TYPE,
+        .layer = 0,
+    };
+    uint64_t ssd_prepared_bytes = 0;
+    ds4_gpu_test_q4_attn_q_b_f16_cache_report(&gate_before);
+    poison_f32(candidate_host, output_storage_count, k_candidate_poison);
+    CHECK(ds4_gpu_tensor_write(
+              candidate_base, 0, candidate_host,
+              output_storage_count * sizeof(float)) != 0,
+          "DISABLE output poison");
+    CHECK(setenv(k_disable, "1", 1) == 0, "set cache disable env");
+    ds4_gpu_tensor *gate_out = ds4_gpu_tensor_view(
+        candidate_base, GUARD_FLOATS * sizeof(float),
+        (uint64_t)32u * OUT_DIM * sizeof(float));
+    ds4_gpu_tensor *gate_half = ds4_gpu_tensor_view(
+        q_half_base, GUARD_HALFS * sizeof(uint16_t),
+        (uint64_t)32u * OUT_DIM * sizeof(uint16_t));
+    CHECK(gate_out && gate_half, "DISABLE tensor views");
+    CHECK(run_candidate(
+              gate_out, gate_half, model, model_bytes, x, 32u) == -1,
+          "DISABLE must win over REQUIRE");
+    ds4_gpu_tensor_free(gate_half);
+    ds4_gpu_tensor_free(gate_out);
+    CHECK(ds4_gpu_tensor_read(
+              candidate_base, 0, candidate_host,
+              output_storage_count * sizeof(float)) != 0,
+          "DISABLE output readback");
+    CHECK(count_poison_f32_mismatches(
+              candidate_host, 0, output_storage_count,
+              k_candidate_poison) == 0,
+          "DISABLE touched output");
+    ds4_gpu_test_q4_attn_q_b_f16_cache_report(&gate_after);
+    check_cache_storage_unchanged(
+        &gate_before, &gate_after, "DISABLE touched cache storage/state");
+    CHECK(gate_after.candidate_calls == gate_before.candidate_calls + 1u &&
+          gate_after.fallbacks == gate_before.fallbacks + 1u &&
+          gate_after.rejects == gate_before.rejects + 1u,
+          "DISABLE accounting");
+    CHECK(unsetenv(k_disable) == 0, "clear cache disable env");
+
+    /* The persistent-sidecar SSD extension is opt-in: its default must
+     * continue to drop resident-only sidecars and, under REQUIRE, reject the
+     * persistent cache path without rebuilding it.  The independent transient
+     * path is exercised below with REQUIRE cleared. */
+    gate_before = gate_after;
+    CHECK(gate_before.entries != 0u,
+          "default SSD transition lacks a resident sidecar to release");
+    const uint64_t default_ssd_generation =
+        ds4_gpu_q4_attn_q_b_f16_cache_generation();
+    const uint64_t ssd_output_count = (uint64_t)32u * OUT_DIM;
+    poison_f32(reference_host, output_storage_count,
+               k_reference_poison);
+    CHECK(ds4_gpu_tensor_write(
+              reference_base, 0, reference_host,
+              output_storage_count * sizeof(float)) != 0,
+          "SSD exact-view reference poison");
+    ds4_gpu_tensor *ssd_reference = ds4_gpu_tensor_view(
+        reference_base, GUARD_FLOATS * sizeof(float),
+        ssd_output_count * sizeof(float));
+    CHECK(ssd_reference != NULL, "SSD exact-view reference tensor view");
+    CHECK(run_reference_at(
+              ssd_reference, model, model_bytes, ssd_weight_offset,
+              x, 32u) == 1,
+          "SSD exact-view native Q4 reference");
+    ds4_gpu_tensor_free(ssd_reference);
+    CHECK(ds4_gpu_tensor_read(
+              reference_base, 0, reference_host,
+              output_storage_count * sizeof(float)) != 0,
+          "SSD exact-view reference readback");
+    poison_f32(candidate_host, output_storage_count, k_candidate_poison);
+    CHECK(ds4_gpu_tensor_write(
+              candidate_base, 0, candidate_host,
+              output_storage_count * sizeof(float)) != 0,
+          "SSD output poison");
+    ds4_gpu_set_ssd_streaming(true);
+    ds4_gpu_test_q4_attn_q_b_f16_cache_report(&gate_after);
+    CHECK(gate_after.entries == 0u && gate_after.bytes == 0u,
+          "enabling SSD mode retained resident sidecars");
+    CHECK(ds4_gpu_q4_attn_q_b_f16_cache_generation() !=
+              default_ssd_generation,
+          "default SSD transition did not advance cache generation");
+    CHECK(gate_after.build_circuit_open == 0u,
+          "enabling SSD mode retained the build circuit state");
+    CHECK(gate_after.candidate_calls == gate_before.candidate_calls &&
+          gate_after.fallbacks == gate_before.fallbacks &&
+          gate_after.rejects == gate_before.rejects,
+          "SSD transition reset cache accounting");
+
+    gate_before = gate_after;
+    CHECK(ds4_gpu_prepare_q4_attn_q_b_f16_sidecars(
+              model, model_bytes, &ssd_desc, 1u, 32u, 0u,
+              &ssd_prepared_bytes) == -1,
+          "REQUIRE alone must not enable SSD-streaming prewarm");
+    CHECK(ssd_prepared_bytes == 0u,
+          "default SSD prewarm reported prepared bytes");
+    ds4_gpu_test_q4_attn_q_b_f16_cache_report(&gate_after);
+    check_cache_storage_unchanged(
+        &gate_before, &gate_after,
+        "default SSD prewarm touched cache storage/state");
+    gate_before = gate_after;
+    gate_out = ds4_gpu_tensor_view(
+        candidate_base, GUARD_FLOATS * sizeof(float),
+        (uint64_t)32u * OUT_DIM * sizeof(float));
+    gate_half = ds4_gpu_tensor_view(
+        q_half_base, GUARD_HALFS * sizeof(uint16_t),
+        (uint64_t)32u * OUT_DIM * sizeof(uint16_t));
+    CHECK(gate_out && gate_half, "SSD tensor views");
+    CHECK(run_candidate_at(
+              gate_out, gate_half, model, model_bytes,
+              ssd_weight_offset, x, 32u) == -1,
+          "SSD mode must reject required resident cache");
+    ds4_gpu_tensor_free(gate_half);
+    ds4_gpu_tensor_free(gate_out);
+    CHECK(ds4_gpu_tensor_read(
+              candidate_base, 0, candidate_host,
+              output_storage_count * sizeof(float)) != 0,
+          "SSD output readback");
+    CHECK(count_poison_f32_mismatches(
+              candidate_host, 0, output_storage_count,
+              k_candidate_poison) == 0,
+          "SSD rejection touched output");
+    ds4_gpu_test_q4_attn_q_b_f16_cache_report(&gate_after);
+    CHECK(gate_after.entries == 0u && gate_after.bytes == 0u &&
+          gate_after.build_circuit_open == 0u,
+          "SSD rejection rebuilt resident cache");
+    CHECK(gate_after.candidate_calls == gate_before.candidate_calls + 1u &&
+          gate_after.fallbacks == gate_before.fallbacks + 1u &&
+          gate_after.rejects == gate_before.rejects + 1u,
+          "SSD rejection accounting");
+
+    /* Explicit opt-in permits the production prewarm to build the sidecar and
+     * the prefill dispatch to hit it while experts remain SSD-streamed.
+     * REQUIRE is deliberately not the opt-in: it only makes failure strict. */
+    CHECK(unsetenv(k_require) == 0,
+          "clear REQUIRE before standalone SSD opt-in");
+    CHECK(setenv(k_enable_ssd_streaming, "1", 1) == 0,
+          "enable Q4 attn_q_b F16 cache with SSD streaming");
+    gate_before = gate_after;
+    ssd_prepared_bytes = 0;
+    CHECK(ds4_gpu_prepare_q4_attn_q_b_f16_sidecars(
+              model, model_bytes, &ssd_desc, 1u, 32u, 0u,
+              &ssd_prepared_bytes) == 1,
+          "SSD opt-in prewarm build");
+    ds4_gpu_test_q4_attn_q_b_f16_cache_report(&gate_after);
+    CHECK(ssd_prepared_bytes == f16_cache_bytes &&
+              gate_after.entries == 1u &&
+              gate_after.bytes == f16_cache_bytes &&
+              gate_after.builds == gate_before.builds + 1u &&
+              gate_after.misses == gate_before.misses &&
+              gate_after.hits == gate_before.hits &&
+              gate_after.lookups == gate_before.lookups &&
+              gate_after.candidate_calls == gate_before.candidate_calls &&
+              gate_after.fallbacks == gate_before.fallbacks &&
+              gate_after.rejects == gate_before.rejects &&
+              gate_after.build_circuit_open == 0u,
+          "SSD opt-in prewarm build accounting");
+
+    gate_before = gate_after;
+    poison_f32(candidate_host, output_storage_count, k_candidate_poison);
+    CHECK(ds4_gpu_tensor_write(
+              candidate_base, 0, candidate_host,
+              output_storage_count * sizeof(float)) != 0,
+          "SSD opt-in first-hit output poison");
+    gate_out = ds4_gpu_tensor_view(
+        candidate_base, GUARD_FLOATS * sizeof(float),
+        (uint64_t)32u * OUT_DIM * sizeof(float));
+    gate_half = ds4_gpu_tensor_view(
+        q_half_base, GUARD_HALFS * sizeof(uint16_t),
+        (uint64_t)32u * OUT_DIM * sizeof(uint16_t));
+    CHECK(gate_out && gate_half, "SSD opt-in first-hit tensor views");
+    CHECK(run_candidate_at(
+              gate_out, gate_half, model, model_bytes,
+              ssd_weight_offset, x, 32u) == 1,
+          "SSD opt-in first prefill cache hit");
+    ds4_gpu_tensor_free(gate_half);
+    ds4_gpu_tensor_free(gate_out);
+    ds4_gpu_test_q4_attn_q_b_f16_cache_report(&gate_after);
+    CHECK(gate_after.entries == gate_before.entries &&
+              gate_after.bytes == gate_before.bytes &&
+              gate_after.builds == gate_before.builds &&
+              gate_after.misses == gate_before.misses &&
+              gate_after.hits == gate_before.hits + 1u &&
+              gate_after.lookups == gate_before.lookups + 1u &&
+              gate_after.candidate_calls ==
+                  gate_before.candidate_calls + 1u &&
+              gate_after.fallbacks == gate_before.fallbacks &&
+              gate_after.rejects == gate_before.rejects &&
+              gate_after.build_circuit_open == 0u,
+          "SSD opt-in first-hit accounting");
+    CHECK(ds4_gpu_tensor_read(
+              candidate_base, 0, candidate_host,
+              output_storage_count * sizeof(float)) != 0,
+          "SSD opt-in first-hit output readback");
+    CHECK(count_poison_f32_mismatches(
+              candidate_host, GUARD_FLOATS,
+              GUARD_FLOATS + (uint64_t)32u * OUT_DIM,
+              k_candidate_poison) != 0u,
+          "SSD opt-in first hit did not write output");
+    uint64_t ssd_first_mismatch = UINT64_MAX;
+    CHECK(count_bit_mismatches(
+              reference_host + GUARD_FLOATS,
+              candidate_host + GUARD_FLOATS,
+              ssd_output_count,
+              &ssd_first_mismatch) == 0u,
+          "SSD non-page-aligned exact-view candidate bitwise mismatch");
+
+    gate_before = gate_after;
+    gate_out = ds4_gpu_tensor_view(
+        candidate_base, GUARD_FLOATS * sizeof(float),
+        (uint64_t)32u * OUT_DIM * sizeof(float));
+    gate_half = ds4_gpu_tensor_view(
+        q_half_base, GUARD_HALFS * sizeof(uint16_t),
+        (uint64_t)32u * OUT_DIM * sizeof(uint16_t));
+    CHECK(gate_out && gate_half, "SSD opt-in hot tensor views");
+    CHECK(run_candidate_at(
+              gate_out, gate_half, model, model_bytes,
+              ssd_weight_offset, x, 32u) == 1,
+          "SSD opt-in cache hit");
+    ds4_gpu_tensor_free(gate_half);
+    ds4_gpu_tensor_free(gate_out);
+    ds4_gpu_test_q4_attn_q_b_f16_cache_report(&gate_after);
+    CHECK(gate_after.entries == gate_before.entries &&
+              gate_after.bytes == gate_before.bytes &&
+              gate_after.builds == gate_before.builds &&
+              gate_after.misses == gate_before.misses &&
+              gate_after.hits == gate_before.hits + 1u &&
+              gate_after.lookups == gate_before.lookups + 1u &&
+              gate_after.candidate_calls ==
+                  gate_before.candidate_calls + 1u &&
+              gate_after.fallbacks == gate_before.fallbacks &&
+              gate_after.rejects == gate_before.rejects,
+          "SSD opt-in hot-hit accounting");
+
+    /* DISABLE remains the highest-priority policy even when SSD opt-in and
+     * REQUIRE are both active, and it must not evict a ready sidecar. */
+    CHECK(setenv(k_require, "1", 1) == 0,
+          "restore REQUIRE for SSD DISABLE priority test");
+    CHECK(setenv(k_disable, "1", 1) == 0,
+          "disable SSD opt-in Q4 attn_q_b F16 cache");
+    gate_before = gate_after;
+    ssd_prepared_bytes = 0;
+    CHECK(ds4_gpu_prepare_q4_attn_q_b_f16_sidecars(
+              model, model_bytes, &ssd_desc, 1u, 32u, 0u,
+              &ssd_prepared_bytes) == -1,
+          "DISABLE must win over SSD opt-in prewarm and REQUIRE");
+    CHECK(ssd_prepared_bytes == 0u,
+          "disabled SSD opt-in prewarm reported prepared bytes");
+    ds4_gpu_test_q4_attn_q_b_f16_cache_report(&gate_after);
+    check_cache_storage_unchanged(
+        &gate_before, &gate_after,
+        "SSD opt-in prewarm DISABLE touched cache storage/state");
+    gate_before = gate_after;
+    gate_out = ds4_gpu_tensor_view(
+        candidate_base, GUARD_FLOATS * sizeof(float),
+        (uint64_t)32u * OUT_DIM * sizeof(float));
+    gate_half = ds4_gpu_tensor_view(
+        q_half_base, GUARD_HALFS * sizeof(uint16_t),
+        (uint64_t)32u * OUT_DIM * sizeof(uint16_t));
+    CHECK(gate_out && gate_half, "SSD opt-in DISABLE tensor views");
+    CHECK(run_candidate_at(
+              gate_out, gate_half, model, model_bytes,
+              ssd_weight_offset, x, 32u) == -1,
+          "DISABLE must win over SSD opt-in and REQUIRE");
+    ds4_gpu_tensor_free(gate_half);
+    ds4_gpu_tensor_free(gate_out);
+    ds4_gpu_test_q4_attn_q_b_f16_cache_report(&gate_after);
+    check_cache_storage_unchanged(
+        &gate_before, &gate_after,
+        "SSD opt-in DISABLE touched cache storage/state");
+    CHECK(gate_after.candidate_calls == gate_before.candidate_calls + 1u &&
+              gate_after.fallbacks == gate_before.fallbacks + 1u &&
+              gate_after.rejects == gate_before.rejects + 1u,
+          "SSD opt-in DISABLE accounting");
+    CHECK(unsetenv(k_disable) == 0,
+          "clear SSD opt-in cache disable env");
+
+    /* Reasserting SSD mode with opt-in preserves the sidecar.  Explicit
+     * lifecycle release remains authoritative and advances the generation. */
+    const uint64_t opt_in_ssd_generation =
+        ds4_gpu_q4_attn_q_b_f16_cache_generation();
+    gate_before = gate_after;
+    ds4_gpu_set_ssd_streaming(true);
+    ds4_gpu_test_q4_attn_q_b_f16_cache_report(&gate_after);
+    check_cache_storage_unchanged(
+        &gate_before, &gate_after,
+        "SSD opt-in transition evicted ready sidecar");
+    CHECK(ds4_gpu_q4_attn_q_b_f16_cache_generation() ==
+              opt_in_ssd_generation,
+          "SSD opt-in transition advanced cache generation");
+
+    CHECK(ds4_gpu_release_q4_attn_q_b_f16_sidecars() != 0,
+          "SSD opt-in lifecycle release");
+    ds4_gpu_test_q4_attn_q_b_f16_cache_report(&gate_after);
+    CHECK(gate_after.entries == 0u && gate_after.bytes == 0u &&
+              gate_after.build_circuit_open == 0u,
+          "SSD opt-in lifecycle release retained cache state");
+    CHECK(ds4_gpu_q4_attn_q_b_f16_cache_generation() !=
+              opt_in_ssd_generation,
+          "SSD opt-in lifecycle release did not advance generation");
+    fprintf(stderr,
+            "Metal Q4 attn_q_b F16 cache SSD opt-in policy/lifecycle: "
+            "PASS\n");
+
+    CHECK(unsetenv(k_enable_ssd_streaming) == 0,
+          "clear SSD-streaming cache opt-in env");
+    ds4_gpu_set_ssd_streaming(false);
+
+    /* Exercise the public production selector, not only its benchmark hook.
+     * Lower the threshold for this bounded oracle so aligned and boundary
+     * geometries fit in the guarded allocations.  Resident and SSD-streamed
+     * modes must use one transient scratch without publishing persistent
+     * sidecars. */
+    ds4_gpu_test_q4_attn_q_b_f16_cache_reset();
+    CHECK(unsetenv(k_require) == 0,
+          "clear REQUIRE for transient production oracle");
+    const ds4_gpu_q4_attn_q_b_f16_sidecar_desc transient_desc = {
+        .weight_offset = 0u,
+        .weight_bytes = weight_bytes,
+        .in_dim = IN_DIM,
+        .out_dim = OUT_DIM,
+        .weight_type = Q4_K_TYPE,
+        .layer = 0u,
+    };
+    uint64_t transient_prepared_bytes = 0u;
+    CHECK(ds4_gpu_prepare_q4_attn_q_b_f16_sidecars(
+              model, model_bytes, &transient_desc, 1u, 4096u, 0u,
+              &transient_prepared_bytes) == 1,
+          "transient production preflight");
+    CHECK(transient_prepared_bytes == f16_cache_bytes,
+          "transient production preflight did not allocate one scratch");
+    ds4_gpu_q4_attn_q_b_f16_cache_report transient_preflight_report;
+    ds4_gpu_test_q4_attn_q_b_f16_cache_report(
+        &transient_preflight_report);
+    CHECK(transient_preflight_report.entries == 0u &&
+              transient_preflight_report.bytes == 0u &&
+              transient_preflight_report.lookups == 0u &&
+              transient_preflight_report.builds == 0u,
+          "transient production preflight published a sidecar");
+    CHECK(setenv(k_transient_min_tokens, "32", 1) == 0,
+          "lower transient production threshold for oracle");
+    for (uint32_t case_i = 0;
+         case_i < sizeof(token_cases) / sizeof(token_cases[0]);
+         case_i++) {
+        const uint32_t n_tok = token_cases[case_i];
+        const uint64_t output_count = (uint64_t)n_tok * OUT_DIM;
+        const uint64_t output_bytes = output_count * sizeof(float);
+        const uint64_t q_half_count = (uint64_t)n_tok * IN_DIM;
+        const uint64_t q_half_bytes = q_half_count * sizeof(uint16_t);
+
+        poison_f32(reference_host, output_storage_count,
+                   k_reference_poison);
+        poison_f32(candidate_host, output_storage_count,
+                   k_candidate_poison);
+        poison_f16(q_half_host, q_half_storage_count);
+        CHECK(ds4_gpu_tensor_write(
+                  reference_base, 0, reference_host,
+                  output_storage_count * sizeof(float)) != 0,
+              "transient production reference poison upload");
+        CHECK(ds4_gpu_tensor_write(
+                  candidate_base, 0, candidate_host,
+                  output_storage_count * sizeof(float)) != 0,
+              "transient production candidate poison upload");
+        CHECK(ds4_gpu_tensor_write(
+                  q_half_base, 0, q_half_host,
+                  q_half_storage_count * sizeof(uint16_t)) != 0,
+              "transient production q_half poison upload");
+
+        ds4_gpu_tensor *transient_reference = ds4_gpu_tensor_view(
+            reference_base, GUARD_FLOATS * sizeof(float), output_bytes);
+        ds4_gpu_tensor *transient_candidate = ds4_gpu_tensor_view(
+            candidate_base, GUARD_FLOATS * sizeof(float), output_bytes);
+        ds4_gpu_tensor *transient_half = ds4_gpu_tensor_view(
+            q_half_base, GUARD_HALFS * sizeof(uint16_t), q_half_bytes);
+        CHECK(transient_reference && transient_candidate && transient_half,
+              "transient production tensor views");
+        CHECK(run_reference(
+                  transient_reference, model, model_bytes, x, n_tok) == 1,
+              "transient production native Q4 reference");
+        CHECK(run_candidate(
+                  transient_candidate, transient_half,
+                  model, model_bytes, x, n_tok) == 1,
+              "transient production public entry point");
+        ds4_gpu_tensor_free(transient_half);
+        ds4_gpu_tensor_free(transient_candidate);
+        ds4_gpu_tensor_free(transient_reference);
+
+        CHECK(ds4_gpu_tensor_read(
+                  reference_base, 0, reference_host,
+                  output_storage_count * sizeof(float)) != 0,
+              "transient production reference readback");
+        CHECK(ds4_gpu_tensor_read(
+                  candidate_base, 0, candidate_host,
+                  output_storage_count * sizeof(float)) != 0,
+              "transient production candidate readback");
+        CHECK(ds4_gpu_tensor_read(
+                  q_half_base, 0, q_half_host,
+                  q_half_storage_count * sizeof(uint16_t)) != 0,
+              "transient production q_half readback");
+        uint64_t transient_first = UINT64_MAX;
+        CHECK(count_bit_mismatches(
+                  reference_host + GUARD_FLOATS,
+                  candidate_host + GUARD_FLOATS,
+                  output_count, &transient_first) == 0u,
+              "transient production bitwise mismatch");
+        CHECK(count_poison_f32_mismatches(
+                  candidate_host, 0u, GUARD_FLOATS,
+                  k_candidate_poison) == 0u &&
+              count_poison_f32_mismatches(
+                  candidate_host, GUARD_FLOATS + output_count,
+                  output_storage_count, k_candidate_poison) == 0u,
+              "transient production touched output guards");
+        CHECK(count_poison_f16_mismatches_range(
+                  q_half_host, 0u, GUARD_HALFS) == 0u &&
+              count_poison_f16_mismatches_range(
+                  q_half_host, GUARD_HALFS,
+                  GUARD_HALFS + q_half_count) == q_half_count &&
+              count_poison_f16_mismatches_range(
+                  q_half_host, GUARD_HALFS + q_half_count,
+                  q_half_storage_count) == 0u,
+              "transient production q_half payload/canary mismatch");
+    }
+
+    /* Two different layer weights in one real command batch exercise the
+     * production encoder boundaries and prove that stream-local scratch is
+     * not overwritten before the preceding F16/F16 consumer and tail. */
+    {
+        const uint32_t n_tok = 33u;
+        const uint64_t output_count = (uint64_t)n_tok * OUT_DIM;
+        const uint64_t output_bytes = output_count * sizeof(float);
+        const uint64_t q_half_count = (uint64_t)n_tok * IN_DIM;
+        const uint64_t q_half_bytes =
+            q_half_count * sizeof(uint16_t);
+        poison_f32(reference_host, output_storage_count,
+                   k_reference_poison);
+        poison_f32(candidate_host, output_storage_count,
+                   k_candidate_poison);
+        poison_f16(q_half_host, q_half_storage_count);
+        CHECK(ds4_gpu_tensor_write(
+                  reference_base, 0, reference_host,
+                  output_storage_count * sizeof(float)) != 0,
+              "transient batch second-output poison upload");
+        CHECK(ds4_gpu_tensor_write(
+                  candidate_base, 0, candidate_host,
+                  output_storage_count * sizeof(float)) != 0,
+              "transient batch first-output poison upload");
+        CHECK(ds4_gpu_tensor_write(
+                  q_half_base, 0, q_half_host,
+                  q_half_storage_count * sizeof(uint16_t)) != 0,
+              "transient batch q_half poison upload");
+
+        ds4_gpu_tensor *batch_first = ds4_gpu_tensor_view(
+            candidate_base, GUARD_FLOATS * sizeof(float), output_bytes);
+        ds4_gpu_tensor *batch_second = ds4_gpu_tensor_view(
+            reference_base, GUARD_FLOATS * sizeof(float), output_bytes);
+        ds4_gpu_tensor *batch_half = ds4_gpu_tensor_view(
+            q_half_base, GUARD_HALFS * sizeof(uint16_t), q_half_bytes);
+        CHECK(batch_first && batch_second && batch_half,
+              "transient production batch tensor views");
+        CHECK(ds4_gpu_begin_commands() != 0,
+              "begin transient production command batch");
+        CHECK(run_candidate_at(
+                  batch_first, batch_half, model, model_bytes, 0u,
+                  x, n_tok) == 1,
+              "encode first transient production batch layer");
+        CHECK(run_candidate_at(
+                  batch_second, batch_half, model, model_bytes,
+                  weight_bytes, x, n_tok) == 1,
+              "encode second transient production batch layer");
+        CHECK(ds4_gpu_end_commands() != 0,
+              "finish transient production command batch");
+        ds4_gpu_tensor_free(batch_half);
+        ds4_gpu_tensor_free(batch_second);
+        ds4_gpu_tensor_free(batch_first);
+
+        CHECK(ds4_gpu_tensor_read(
+                  candidate_base, 0, candidate_host,
+                  output_storage_count * sizeof(float)) != 0,
+              "transient batch first candidate readback");
+        ds4_gpu_tensor *batch_reference = ds4_gpu_tensor_view(
+            candidate_base, GUARD_FLOATS * sizeof(float), output_bytes);
+        CHECK(batch_reference != NULL,
+              "transient batch first reference view");
+        CHECK(run_reference_at(
+                  batch_reference, model, model_bytes, 0u,
+                  x, n_tok) == 1,
+              "transient batch first native reference");
+        ds4_gpu_tensor_free(batch_reference);
+        CHECK(ds4_gpu_tensor_read(
+                  candidate_base, 0, reference_host,
+                  output_storage_count * sizeof(float)) != 0,
+              "transient batch first reference readback");
+        uint64_t batch_first_mismatch = UINT64_MAX;
+        CHECK(count_bit_mismatches(
+                  reference_host + GUARD_FLOATS,
+                  candidate_host + GUARD_FLOATS,
+                  output_count, &batch_first_mismatch) == 0u,
+              "transient batch first layer mismatch");
+
+        CHECK(ds4_gpu_tensor_read(
+                  reference_base, 0, candidate_host,
+                  output_storage_count * sizeof(float)) != 0,
+              "transient batch second candidate readback");
+        batch_reference = ds4_gpu_tensor_view(
+            candidate_base, GUARD_FLOATS * sizeof(float), output_bytes);
+        CHECK(batch_reference != NULL,
+              "transient batch second reference view");
+        CHECK(run_reference_at(
+                  batch_reference, model, model_bytes, weight_bytes,
+                  x, n_tok) == 1,
+              "transient batch second native reference");
+        ds4_gpu_tensor_free(batch_reference);
+        CHECK(ds4_gpu_tensor_read(
+                  candidate_base, 0, reference_host,
+                  output_storage_count * sizeof(float)) != 0,
+              "transient batch second reference readback");
+        uint64_t batch_second_mismatch = UINT64_MAX;
+        CHECK(count_bit_mismatches(
+                  reference_host + GUARD_FLOATS,
+                  candidate_host + GUARD_FLOATS,
+                  output_count, &batch_second_mismatch) == 0u,
+              "transient batch second layer mismatch");
+
+        CHECK(ds4_gpu_tensor_read(
+                  q_half_base, 0, q_half_host,
+                  q_half_storage_count * sizeof(uint16_t)) != 0,
+              "transient batch q_half readback");
+        CHECK(count_poison_f16_mismatches_range(
+                  q_half_host, 0u, GUARD_HALFS) == 0u &&
+              count_poison_f16_mismatches_range(
+                  q_half_host, GUARD_HALFS,
+                  GUARD_HALFS + q_half_count) == q_half_count &&
+              count_poison_f16_mismatches_range(
+                  q_half_host, GUARD_HALFS + q_half_count,
+                  q_half_storage_count) == 0u,
+              "transient batch q_half payload/canary mismatch");
+        fprintf(stderr,
+                "Metal Q4 attn_q_b transient F16 command-batch "
+                "two-layer scratch reuse N=33: PASS\n");
+    }
+
+    ds4_gpu_q4_attn_q_b_f16_cache_report transient_report;
+    ds4_gpu_test_q4_attn_q_b_f16_cache_report(&transient_report);
+    CHECK(transient_report.entries == 0u &&
+              transient_report.bytes == 0u &&
+              transient_report.lookups == 0u &&
+              transient_report.builds == 0u,
+          "transient production published a persistent sidecar");
+
+    /* Install a second model identity whose registered view covers only a
+     * disjoint prefix.  Its byte-identical Q4 matrix starts at a deliberately
+     * non-page-aligned, uncovered offset, so this succeeds only when the SSD
+     * path creates an exact owned source view.  Running inside a real command
+     * batch also exercises the completion-handler lifetime under
+     * DS4_METAL_UNRETAINED_COMMAND_BUFFERS. */
+    const uint64_t ssd_transient_weight_offset = page + SSD_SOURCE_LEADING;
+    const uint64_t ssd_transient_model_bytes = align_up(
+        ssd_transient_weight_offset + weight_bytes, page);
+    void *ssd_transient_model = NULL;
+    CHECK(posix_memalign(&ssd_transient_model, (size_t)page,
+                         (size_t)ssd_transient_model_bytes) == 0,
+          "SSD transient exact-view model allocation");
+    memset(ssd_transient_model, 0, (size_t)ssd_transient_model_bytes);
+    fill_q4_matrix((block_q4_K *)((uint8_t *)ssd_transient_model +
+                                  ssd_transient_weight_offset));
+
+    const uint32_t ssd_transient_tokens = 64u;
+    const uint64_t ssd_transient_output_count =
+        (uint64_t)ssd_transient_tokens * OUT_DIM;
+    const uint64_t ssd_transient_output_bytes =
+        ssd_transient_output_count * sizeof(float);
+    const uint64_t ssd_transient_half_count =
+        (uint64_t)ssd_transient_tokens * IN_DIM;
+    const uint64_t ssd_transient_half_bytes =
+        ssd_transient_half_count * sizeof(uint16_t);
+    poison_f32(reference_host, output_storage_count, k_reference_poison);
+    poison_f32(candidate_host, output_storage_count, k_candidate_poison);
+    poison_f16(q_half_host, q_half_storage_count);
+    CHECK(ds4_gpu_tensor_write(
+              reference_base, 0, reference_host,
+              output_storage_count * sizeof(float)) != 0,
+          "transient SSD reference poison upload");
+    CHECK(ds4_gpu_tensor_write(
+              candidate_base, 0, candidate_host,
+              output_storage_count * sizeof(float)) != 0,
+          "transient SSD candidate poison upload");
+    CHECK(ds4_gpu_tensor_write(
+              q_half_base, 0, q_half_host,
+              q_half_storage_count * sizeof(uint16_t)) != 0,
+          "transient SSD q_half poison upload");
+    ds4_gpu_tensor *transient_ssd_reference = ds4_gpu_tensor_view(
+        reference_base, GUARD_FLOATS * sizeof(float),
+        ssd_transient_output_bytes);
+    ds4_gpu_tensor *transient_ssd_out = ds4_gpu_tensor_view(
+        candidate_base, GUARD_FLOATS * sizeof(float),
+        ssd_transient_output_bytes);
+    ds4_gpu_tensor *transient_ssd_half = ds4_gpu_tensor_view(
+        q_half_base, GUARD_HALFS * sizeof(uint16_t),
+        ssd_transient_half_bytes);
+    CHECK(transient_ssd_reference && transient_ssd_out && transient_ssd_half,
+          "transient SSD exact-view tensor views");
+    CHECK(run_reference(
+              transient_ssd_reference, model, model_bytes, x,
+              ssd_transient_tokens) == 1,
+          "transient SSD native Q4 reference");
+    ds4_gpu_tensor_free(transient_ssd_reference);
+    CHECK(ds4_gpu_tensor_read(
+              reference_base, 0, reference_host,
+              output_storage_count * sizeof(float)) != 0,
+          "transient SSD reference readback");
+
+    /* Stream 0 already owns the resident oracle's 64 MiB scratch.  Move the
+     * SSD transient case to a fresh stream so its successful preflight also
+     * covers cold scratch + exact-source admission and allocation. */
+    ds4_gpu_set_stream(1);
+    ds4_gpu_set_ssd_streaming(true);
+    CHECK(ds4_gpu_set_model_map_range(
+              ssd_transient_model, ssd_transient_model_bytes,
+              0u, page, page) != 0,
+          "install disjoint SSD transient model prefix");
+    const ds4_gpu_q4_attn_q_b_f16_sidecar_desc ssd_transient_desc = {
+        .weight_offset = ssd_transient_weight_offset,
+        .weight_bytes = weight_bytes,
+        .in_dim = IN_DIM,
+        .out_dim = OUT_DIM,
+        .weight_type = Q4_K_TYPE,
+        .layer = 0u,
+    };
+    ds4_gpu_q4_attn_q_b_f16_cache_report ssd_lifetime_before;
+    ds4_gpu_q4_attn_q_b_f16_cache_report ssd_lifetime_mid;
+    ds4_gpu_q4_attn_q_b_f16_cache_report ssd_lifetime_after;
+    ds4_gpu_stream_test_stats ssd_transients_before;
+    ds4_gpu_stream_test_stats ssd_transients_mid;
+    ds4_gpu_stream_test_stats ssd_transients_after;
+    ds4_gpu_test_q4_attn_q_b_f16_cache_report(&ssd_lifetime_before);
+
+    /* A rejected session reserve must latch the transient selector off.  The
+     * public call then returns the native-Q4 fallback sentinel without
+     * touching either output, even though its smaller runtime-only gate would
+     * otherwise fit.  A later successful preflight re-arms the same slot. */
+    uint64_t ssd_transient_prepared_bytes = UINT64_MAX;
+    CHECK(ds4_gpu_prepare_q4_attn_q_b_f16_sidecars(
+              ssd_transient_model, ssd_transient_model_bytes,
+              &ssd_transient_desc, 1u, 4096u, UINT64_MAX,
+              &ssd_transient_prepared_bytes) == 0,
+          "transient SSD oversized-reserve preflight rejection");
+    CHECK(ssd_transient_prepared_bytes == 0u,
+          "transient SSD rejected preflight allocated scratch");
+    CHECK(ds4_gpu_begin_commands() != 0,
+          "begin transient SSD rejected command batch");
+    CHECK(run_candidate_at(
+              transient_ssd_out, transient_ssd_half,
+              ssd_transient_model, ssd_transient_model_bytes,
+              ssd_transient_weight_offset, x,
+              ssd_transient_tokens) == 0,
+          "transient SSD rejected preflight escaped admission latch");
+    CHECK(ds4_gpu_end_commands() != 0,
+          "finish transient SSD rejected command batch");
+    CHECK(ds4_gpu_tensor_read(
+              candidate_base, 0, candidate_host,
+              output_storage_count * sizeof(float)) != 0,
+          "transient SSD rejected output readback");
+    CHECK(ds4_gpu_tensor_read(
+              q_half_base, 0, q_half_host,
+              q_half_storage_count * sizeof(uint16_t)) != 0,
+          "transient SSD rejected q_half readback");
+    CHECK(count_poison_f32_mismatches(
+              candidate_host, 0u, output_storage_count,
+              k_candidate_poison) == 0u,
+          "transient SSD rejected preflight touched output");
+    CHECK(count_poison_f16_mismatches(
+              q_half_host, q_half_storage_count) == 0u,
+          "transient SSD rejected preflight touched q_half");
+
+    ssd_transient_prepared_bytes = UINT64_MAX;
+    CHECK(ds4_gpu_prepare_q4_attn_q_b_f16_sidecars(
+              ssd_transient_model, ssd_transient_model_bytes,
+              &ssd_transient_desc, 1u, 4096u, 0u,
+              &ssd_transient_prepared_bytes) == 1,
+          "transient SSD successful preflight re-arm");
+    CHECK(ssd_transient_prepared_bytes == f16_cache_bytes,
+          "transient SSD cold preflight did not allocate one scratch");
+    ds4_gpu_test_stream_stats(&ssd_transients_before);
+    CHECK(ds4_gpu_begin_commands() != 0,
+          "begin transient SSD exact-view command batch");
+    CHECK(run_candidate_at(
+              transient_ssd_out, transient_ssd_half,
+              ssd_transient_model, ssd_transient_model_bytes,
+              ssd_transient_weight_offset, x,
+              ssd_transient_tokens) == 1,
+          "encode transient SSD exact-view candidate");
+    ds4_gpu_test_q4_attn_q_b_f16_cache_report(&ssd_lifetime_mid);
+    ds4_gpu_test_stream_stats(&ssd_transients_mid);
+    CHECK(ssd_lifetime_mid.transient_exact_views_created ==
+              ssd_lifetime_before.transient_exact_views_created + 1u &&
+          ssd_lifetime_mid.transient_exact_views_live ==
+              ssd_lifetime_before.transient_exact_views_live + 1u &&
+          ssd_lifetime_mid.model_exact_cache_entries ==
+              ssd_lifetime_before.model_exact_cache_entries &&
+          ssd_lifetime_mid.model_exact_cache_bytes ==
+              ssd_lifetime_before.model_exact_cache_bytes &&
+          ssd_transients_mid.transient_references ==
+              ssd_transients_before.transient_references,
+          "transient SSD exact source was not command-buffer-owned");
+    CHECK(ds4_gpu_end_commands() != 0,
+          "finish transient SSD exact-view command batch");
+    ds4_gpu_test_q4_attn_q_b_f16_cache_report(&ssd_lifetime_after);
+    ds4_gpu_test_stream_stats(&ssd_transients_after);
+    CHECK(ssd_lifetime_after.transient_exact_views_created ==
+              ssd_lifetime_before.transient_exact_views_created + 1u &&
+          ssd_lifetime_after.transient_exact_views_live ==
+              ssd_lifetime_before.transient_exact_views_live &&
+          ssd_lifetime_after.model_exact_cache_entries ==
+              ssd_lifetime_before.model_exact_cache_entries &&
+          ssd_lifetime_after.model_exact_cache_bytes ==
+              ssd_lifetime_before.model_exact_cache_bytes &&
+          ssd_transients_after.transient_references ==
+              ssd_transients_before.transient_references,
+          "transient SSD exact source leaked past command completion");
+    ds4_gpu_tensor_free(transient_ssd_half);
+    ds4_gpu_tensor_free(transient_ssd_out);
+    CHECK(ds4_gpu_tensor_read(
+              candidate_base, 0, candidate_host,
+              output_storage_count * sizeof(float)) != 0,
+          "transient SSD candidate readback");
+    uint64_t transient_ssd_first = UINT64_MAX;
+    CHECK(count_bit_mismatches(
+              reference_host + GUARD_FLOATS,
+              candidate_host + GUARD_FLOATS,
+              ssd_transient_output_count,
+              &transient_ssd_first) == 0u,
+          "transient SSD exact-view bitwise mismatch");
+    CHECK(count_poison_f32_mismatches(
+              candidate_host, 0u, GUARD_FLOATS,
+              k_candidate_poison) == 0u &&
+          count_poison_f32_mismatches(
+              candidate_host,
+              GUARD_FLOATS + ssd_transient_output_count,
+              output_storage_count,
+              k_candidate_poison) == 0u,
+          "transient SSD exact-view touched output guards");
+    CHECK(ds4_gpu_tensor_read(
+              q_half_base, 0, q_half_host,
+              q_half_storage_count * sizeof(uint16_t)) != 0,
+          "transient SSD q_half readback");
+    CHECK(count_poison_f16_mismatches_range(
+              q_half_host, 0u, GUARD_HALFS) == 0u &&
+          count_poison_f16_mismatches_range(
+              q_half_host, GUARD_HALFS,
+              GUARD_HALFS + ssd_transient_half_count) ==
+                  ssd_transient_half_count &&
+          count_poison_f16_mismatches_range(
+              q_half_host,
+              GUARD_HALFS + ssd_transient_half_count,
+              q_half_storage_count) == 0u,
+          "transient SSD q_half payload/canary mismatch");
+    ds4_gpu_test_q4_attn_q_b_f16_cache_report(&transient_report);
+    CHECK(transient_report.entries == 0u &&
+              transient_report.bytes == 0u &&
+              transient_report.lookups == 0u &&
+              transient_report.builds == 0u,
+          "transient SSD exact-view published a persistent sidecar");
+    ds4_gpu_set_ssd_streaming(false);
+    ds4_gpu_set_stream(0);
+    CHECK(setenv(k_require, "1", 1) == 0,
+          "restore REQUIRE after transient production oracle");
+    CHECK(unsetenv(k_transient_min_tokens) == 0,
+          "restore transient production threshold");
+    fprintf(stderr,
+            "Metal Q4 attn_q_b transient F16 production selector "
+            "N=32/33/64 and SSD exact-view: PASS\n");
+
+    /* The input, including both guards, is immutable across every path. */
+    CHECK(ds4_gpu_tensor_read(
+              x_base, 0, input_readback,
+              input_storage_count * sizeof(float)) != 0,
+          "input readback");
+    CHECK(memcmp(input_host, input_readback,
+                 (size_t)input_storage_count * sizeof(float)) == 0,
+          "input payload/canary modified");
+
+    if (getenv(k_timing) != NULL) {
+        const char *timing_tokens_env = getenv(
+            "DS4_TEST_METAL_Q4_QB_F16_CACHE_TIMING_TOKENS");
+        char *timing_tokens_end = NULL;
+        errno = 0;
+        const unsigned long timing_tokens_value = timing_tokens_env
+            ? strtoul(timing_tokens_env, &timing_tokens_end, 10)
+            : 4096ul;
+        CHECK(!timing_tokens_env ||
+                  (timing_tokens_env[0] >= '0' &&
+                   timing_tokens_env[0] <= '9' &&
+                   errno == 0 &&
+                   timing_tokens_end != timing_tokens_env &&
+                   *timing_tokens_end == '\0'),
+              "invalid timing token count");
+        CHECK(timing_tokens_value >= 32ul &&
+                  timing_tokens_value <= 4096ul,
+              "timing tokens must be in [32, 4096]");
+        const uint32_t timing_tokens = (uint32_t)timing_tokens_value;
+        const uint64_t timing_input_count =
+            (uint64_t)timing_tokens * IN_DIM;
+        const uint64_t timing_output_count =
+            (uint64_t)timing_tokens * OUT_DIM;
+        static const ds4_gpu_test_q4_qb_mm_arm base_arms[] = {
+            DS4_GPU_TEST_Q4_QB_MM_Q4_F32,
+            DS4_GPU_TEST_Q4_QB_MM_Q4_F16,
+            DS4_GPU_TEST_Q4_QB_MM_F16_F32,
+            DS4_GPU_TEST_Q4_QB_MM_F16_F16,
+        };
+        static const ds4_gpu_test_q4_qb_mm_arm transient_panel_arms[] = {
+            DS4_GPU_TEST_Q4_QB_MM_Q4_F16,
+            DS4_GPU_TEST_Q4_QB_MM_Q4_TRANSIENT_F16_F16,
+            DS4_GPU_TEST_Q4_QB_MM_F16_F16,
+        };
+        static const ds4_gpu_test_q4_qb_mm_arm oracle_arms[] = {
+            DS4_GPU_TEST_Q4_QB_MM_Q4_F32,
+            DS4_GPU_TEST_Q4_QB_MM_Q4_F16,
+            DS4_GPU_TEST_Q4_QB_MM_F16_F32,
+            DS4_GPU_TEST_Q4_QB_MM_F16_F16,
+            DS4_GPU_TEST_Q4_QB_MM_Q4_TRANSIENT_F16_F16,
+        };
+        static const uint8_t williams_order[4][4] = {
+            {0u, 1u, 3u, 2u},
+            {1u, 2u, 0u, 3u},
+            {2u, 3u, 1u, 0u},
+            {3u, 0u, 2u, 1u},
+        };
+        static const uint8_t transient_order[2][3] = {
+            {0u, 1u, 2u},
+            {2u, 1u, 0u},
+        };
+        double with_pack_ms[DS4_GPU_TEST_Q4_QB_MM_ARM_COUNT]
+                           [TIMING_SAMPLES];
+        double prepacked_ms[DS4_GPU_TEST_Q4_QB_MM_ARM_COUNT]
+                           [TIMING_SAMPLES];
+        double transient_panel_ms[3][TIMING_SAMPLES];
+        double production_reference_ms[TIMING_SAMPLES];
+        double production_transient_ms[TIMING_SAMPLES];
+        float *timing_input = malloc(
+            (size_t)timing_input_count * sizeof(float));
+        CHECK(timing_input != NULL, "timing input host allocation");
+        for (uint32_t token = 0; token < timing_tokens; token++) {
+            for (uint32_t col = 0; col < IN_DIM; col++) {
+                const uint32_t key = token * 131u + col * 17u +
+                                     ((col >> 3u) ^ (token * 29u));
+                timing_input[(uint64_t)token * IN_DIM + col] =
+                    (float)((int)(key % 129u) - 64) / 509.0f;
+            }
+        }
+
+        /* Timing allocations are independent from the guarded correctness
+         * tensors.  A single output is sufficient because each projection
+         * overwrites it completely before the in-place norm/RoPE stage. */
+        ds4_gpu_tensor *timing_x = ds4_gpu_tensor_alloc(
+            timing_input_count * sizeof(float));
+        ds4_gpu_tensor *timing_out = ds4_gpu_tensor_alloc(
+            timing_output_count * sizeof(float));
+        ds4_gpu_tensor *timing_rhs_f16 = ds4_gpu_tensor_alloc(
+            timing_input_count * sizeof(uint16_t));
+        CHECK(timing_x && timing_out && timing_rhs_f16,
+              "timing Metal tensor allocation");
+        CHECK(ds4_gpu_tensor_write(
+                  timing_x, 0, timing_input,
+                  timing_input_count * sizeof(float)) != 0,
+              "timing input upload");
+        free(timing_input);
+
+        ds4_gpu_test_q4_attn_q_b_f16_cache_reset();
+        CHECK(ds4_gpu_synchronize() != 0, "pre-cold synchronize");
+        const double cold_t0 = monotonic_ms();
+        CHECK(run_candidate(timing_out, timing_rhs_f16, model, model_bytes,
+                            timing_x, timing_tokens) == 1,
+              "timing cold candidate");
+        const double cold_ms = monotonic_ms() - cold_t0;
+        CHECK(ds4_gpu_synchronize() != 0, "post-cold synchronize");
+
+        CHECK(ds4_gpu_tensor_copy_f32_to_f16(
+                  timing_rhs_f16, 0u, timing_x, 0u,
+                  timing_input_count) != 0,
+              "timing prepacked F16 RHS");
+        for (uint32_t arm_i = 0;
+             arm_i < sizeof(base_arms) / sizeof(base_arms[0]);
+             arm_i++) {
+            CHECK(run_mm_arm_projection(
+                      timing_out, timing_rhs_f16, model, model_bytes,
+                      timing_x, timing_tokens, base_arms[arm_i], true) == 1,
+                  "four-way with-pack warmup");
+            CHECK(run_mm_arm_projection(
+                      timing_out, timing_rhs_f16, model, model_bytes,
+                      timing_x, timing_tokens, base_arms[arm_i], false) == 1,
+                  "four-way prepacked warmup");
+        }
+
+        for (uint32_t sample = 0; sample < TIMING_SAMPLES; sample++) {
+            const uint8_t *order = williams_order[sample & 3u];
+            for (uint32_t pos = 0; pos < 4u; pos++) {
+                const uint32_t arm_i = order[pos];
+                const ds4_gpu_test_q4_qb_mm_arm arm = base_arms[arm_i];
+                if ((sample & 1u) == 0u) {
+                    double t0 = monotonic_ms();
+                    CHECK(run_mm_arm_projection(
+                              timing_out, timing_rhs_f16,
+                              model, model_bytes, timing_x, timing_tokens,
+                              arm, true) == 1,
+                          "four-way with-pack timing");
+                    with_pack_ms[arm_i][sample] = monotonic_ms() - t0;
+                    t0 = monotonic_ms();
+                    CHECK(run_mm_arm_projection(
+                              timing_out, timing_rhs_f16,
+                              model, model_bytes, timing_x, timing_tokens,
+                              arm, false) == 1,
+                          "four-way prepacked timing");
+                    prepacked_ms[arm_i][sample] = monotonic_ms() - t0;
+                } else {
+                    double t0 = monotonic_ms();
+                    CHECK(run_mm_arm_projection(
+                              timing_out, timing_rhs_f16,
+                              model, model_bytes, timing_x, timing_tokens,
+                              arm, false) == 1,
+                          "four-way prepacked timing");
+                    prepacked_ms[arm_i][sample] = monotonic_ms() - t0;
+                    t0 = monotonic_ms();
+                    CHECK(run_mm_arm_projection(
+                              timing_out, timing_rhs_f16,
+                              model, model_bytes, timing_x, timing_tokens,
+                              arm, true) == 1,
+                          "four-way with-pack timing");
+                    with_pack_ms[arm_i][sample] = monotonic_ms() - t0;
+                }
+            }
+        }
+        fprintf(stderr,
+                "Metal Q4 attn_q_b four-way resident timing N=%u "
+                "cold-sidecar=%.3f ms\n",
+                timing_tokens, cold_ms);
+        for (uint32_t arm_i = 0;
+             arm_i < sizeof(base_arms) / sizeof(base_arms[0]);
+             arm_i++) {
+            const bool rhs_is_f16 =
+                mm_arm_uses_f16_rhs(base_arms[arm_i]);
+            const double with_pack =
+                timing_quantile(with_pack_ms[arm_i], 0.5);
+            const double with_pack_p25 =
+                timing_quantile(with_pack_ms[arm_i], 0.25);
+            const double with_pack_p75 =
+                timing_quantile(with_pack_ms[arm_i], 0.75);
+            const double prepacked =
+                timing_quantile(prepacked_ms[arm_i], 0.5);
+            const double prepacked_p25 =
+                timing_quantile(prepacked_ms[arm_i], 0.25);
+            const double prepacked_p75 =
+                timing_quantile(prepacked_ms[arm_i], 0.75);
+            const double with_pack_speedup =
+                timing_paired_geomean_speedup(
+                with_pack_ms[DS4_GPU_TEST_Q4_QB_MM_Q4_F32],
+                with_pack_ms[arm_i]);
+            const double prepacked_speedup =
+                timing_paired_geomean_speedup(
+                prepacked_ms[DS4_GPU_TEST_Q4_QB_MM_Q4_F32],
+                prepacked_ms[arm_i]);
+            fprintf(stderr,
+                    "  %-8s %s=%.3f ms [%.3f, %.3f] %.3fx paired-gmean "
+                    "%s=%.3f ms [%.3f, %.3f] %.3fx paired-gmean\n",
+                    mm_arm_name(base_arms[arm_i]),
+                    rhs_is_f16 ? "with-pack" : "control-a",
+                    with_pack, with_pack_p25, with_pack_p75,
+                    with_pack_speedup,
+                    rhs_is_f16 ? "prepacked" : "control-b",
+                    prepacked, prepacked_p25, prepacked_p75,
+                    prepacked_speedup);
+        }
+
+        /* This arm deliberately rebuilds a transient F16 weight matrix for
+         * every projection, then dispatches the same F16/F16 multiply as the
+         * resident sidecar control.  Keep it at the production-prefill
+         * N=4096 geometry and time only with the already-packed RHS.  The two
+         * controls swap first/last position every sample while the transient
+         * arm remains between them, yielding eight directly paired samples. */
+        if (timing_tokens != 4096u) {
+            fprintf(stderr,
+                    "Metal Q4 attn_q_b transient prepacked timing N=%u: "
+                    "SKIP (dedicated geometry is N=4096)\n",
+                    timing_tokens);
+        } else if (!ds4_gpu_test_q4_attn_q_b_mm_arm_supported(
+                       DS4_GPU_TEST_Q4_QB_MM_Q4_TRANSIENT_F16_F16)) {
+            fprintf(stderr,
+                    "Metal Q4 attn_q_b transient prepacked timing N=4096: "
+                    "SKIP (pipeline unsupported)\n");
+        } else {
+            for (uint32_t arm_i = 0;
+                 arm_i < sizeof(transient_panel_arms) /
+                             sizeof(transient_panel_arms[0]);
+                 arm_i++) {
+                CHECK(run_mm_arm_projection(
+                          timing_out, timing_rhs_f16, model, model_bytes,
+                          timing_x, timing_tokens,
+                          transient_panel_arms[arm_i], false) == 1,
+                      "transient panel prepacked warmup");
+            }
+
+            for (uint32_t sample = 0; sample < TIMING_SAMPLES; sample++) {
+                const uint8_t *order = transient_order[sample & 1u];
+                for (uint32_t pos = 0; pos < 3u; pos++) {
+                    const uint32_t arm_i = order[pos];
+                    const double t0 = monotonic_ms();
+                    CHECK(run_mm_arm_projection(
+                              timing_out, timing_rhs_f16,
+                              model, model_bytes, timing_x, timing_tokens,
+                              transient_panel_arms[arm_i], false) == 1,
+                          "transient panel prepacked timing");
+                    transient_panel_ms[arm_i][sample] =
+                        monotonic_ms() - t0;
+                }
+            }
+
+            const double legacy_median =
+                timing_quantile(transient_panel_ms[0], 0.5);
+            const double legacy_p25 =
+                timing_quantile(transient_panel_ms[0], 0.25);
+            const double legacy_p75 =
+                timing_quantile(transient_panel_ms[0], 0.75);
+            const double transient_median =
+                timing_quantile(transient_panel_ms[1], 0.5);
+            const double transient_p25 =
+                timing_quantile(transient_panel_ms[1], 0.25);
+            const double transient_p75 =
+                timing_quantile(transient_panel_ms[1], 0.75);
+            const double sidecar_median =
+                timing_quantile(transient_panel_ms[2], 0.5);
+            const double sidecar_p25 =
+                timing_quantile(transient_panel_ms[2], 0.25);
+            const double sidecar_p75 =
+                timing_quantile(transient_panel_ms[2], 0.75);
+            const double sidecar_vs_legacy =
+                timing_paired_geomean_speedup(
+                    transient_panel_ms[0], transient_panel_ms[2]);
+            const double transient_vs_legacy =
+                timing_paired_geomean_speedup(
+                    transient_panel_ms[0], transient_panel_ms[1]);
+            const double transient_vs_sidecar =
+                timing_paired_geomean_speedup(
+                    transient_panel_ms[2], transient_panel_ms[1]);
+            fprintf(stderr,
+                    "Metal Q4 attn_q_b transient prepacked timing N=4096\n"
+                    "  %-22s %.3f ms [%.3f, %.3f] control\n"
+                    "  %-22s %.3f ms [%.3f, %.3f] "
+                    "%.3fx vs legacy paired-gmean\n"
+                    "  %-22s %.3f ms [%.3f, %.3f] "
+                    "%.3fx vs legacy, %.3fx vs sidecar paired-gmean\n",
+                    mm_arm_name(transient_panel_arms[0]),
+                    legacy_median, legacy_p25, legacy_p75,
+                    mm_arm_name(transient_panel_arms[2]),
+                    sidecar_median, sidecar_p25, sidecar_p75,
+                    sidecar_vs_legacy,
+                    mm_arm_name(transient_panel_arms[1]),
+                    transient_median, transient_p25, transient_p75,
+                    transient_vs_legacy, transient_vs_sidecar);
+        }
+
+        /* Verify the exact timing geometry after sampling so readback and the
+         * second output allocation cannot perturb the measured resident path.
+         * Chunked reads bound host memory even for N=4096. */
+        ds4_gpu_tensor *timing_reference = ds4_gpu_tensor_alloc(
+            timing_output_count * sizeof(float));
+        CHECK(timing_reference != NULL,
+              "timing verification output allocation");
+        CHECK(run_mm_arm_projection(
+                  timing_reference, timing_rhs_f16, model, model_bytes,
+                  timing_x, timing_tokens,
+                  DS4_GPU_TEST_Q4_QB_MM_Q4_F32, true) == 1,
+              "timing raw verification reference");
+
+        const uint64_t verify_bytes =
+            timing_output_count * sizeof(float);
+        const uint64_t verify_chunk_bytes = 4u * 1024u * 1024u;
+        float *verify_reference = malloc((size_t)verify_chunk_bytes);
+        float *verify_candidate = malloc((size_t)verify_chunk_bytes);
+        CHECK(verify_reference && verify_candidate,
+              "timing verification host chunks");
+        for (uint32_t pass = 0; pass < 2u; pass++) {
+            if (pass == 1u) {
+                CHECK(ds4_gpu_head_rms_norm_rope_tail_tensor(
+                          timing_reference, timing_tokens,
+                          N_HEAD, HEAD_DIM, N_ROT,
+                          17u, 0u, false,
+                          10000.0f, 1.0f, 0.0f, 1.0f,
+                          32.0f, 1.0f, 1.0e-6f) != 0,
+                      "timing tail verification reference");
+            }
+            for (uint32_t arm_i = 1;
+                 arm_i < sizeof(oracle_arms) / sizeof(oracle_arms[0]);
+                 arm_i++) {
+                const ds4_gpu_test_q4_qb_mm_arm arm =
+                    oracle_arms[arm_i];
+                if (mm_arm_is_transient(arm) &&
+                    timing_tokens != 4096u) {
+                    continue;
+                }
+                const bool prepacked_experiment =
+                    mm_arm_is_prepacked_experiment(arm);
+                if (prepacked_experiment &&
+                    !ds4_gpu_test_q4_attn_q_b_mm_arm_supported(arm)) {
+                    continue;
+                }
+                const bool rhs_is_f16 = mm_arm_uses_f16_rhs(arm);
+                /* The transient arm uses only the prepacked RHS; the compact
+                 * copy oracle already covers the same producer above. */
+                const uint32_t rhs_modes =
+                    prepacked_experiment ? 1u : (rhs_is_f16 ? 2u : 1u);
+                for (uint32_t rhs_mode = 0; rhs_mode < rhs_modes;
+                     rhs_mode++) {
+                    const bool materialize_rhs =
+                        prepacked_experiment ? false : rhs_mode == 0u;
+                    const int ok = pass == 0u
+                        ? run_mm_arm_projection(
+                            timing_out, timing_rhs_f16,
+                            model, model_bytes, timing_x, timing_tokens,
+                            arm, materialize_rhs)
+                        : run_mm_arm_with_tail(
+                            timing_out, timing_rhs_f16,
+                            model, model_bytes, timing_x, timing_tokens,
+                            arm, materialize_rhs);
+                    CHECK(ok == 1, "timing projection verification arm");
+
+                    uint64_t verify_mismatches = 0;
+                    uint64_t verify_first = UINT64_MAX;
+                    for (uint64_t offset = 0; offset < verify_bytes;
+                         offset += verify_chunk_bytes) {
+                        const uint64_t chunk_bytes =
+                            verify_bytes - offset < verify_chunk_bytes
+                                ? verify_bytes - offset
+                                : verify_chunk_bytes;
+                        CHECK(ds4_gpu_tensor_read(
+                                  timing_reference, offset,
+                                  verify_reference, chunk_bytes) != 0,
+                              "timing verification reference readback");
+                        CHECK(ds4_gpu_tensor_read(
+                                  timing_out, offset,
+                                  verify_candidate, chunk_bytes) != 0,
+                              "timing verification candidate readback");
+                        uint64_t chunk_first = UINT64_MAX;
+                        verify_mismatches += count_bit_mismatches(
+                            verify_reference, verify_candidate,
+                            chunk_bytes / sizeof(float), &chunk_first);
+                        if (verify_first == UINT64_MAX &&
+                            chunk_first != UINT64_MAX) {
+                            verify_first =
+                                offset / sizeof(float) + chunk_first;
+                        }
+                    }
+                    fprintf(stderr,
+                            "Metal Q4 attn_q_b projection oracle N=%u "
+                            "%s %s %s bitwise=%llu\n",
+                            timing_tokens,
+                            pass == 0u ? "raw" : "tail",
+                            mm_arm_name(arm),
+                            rhs_is_f16
+                                ? (materialize_rhs
+                                    ? "with-pack" : "prepacked")
+                                : "control",
+                            (unsigned long long)verify_mismatches);
+                    if (verify_first != UINT64_MAX) {
+                        fprintf(stderr,
+                                "  first mismatch token=%llu row=%llu\n",
+                                (unsigned long long)(verify_first / OUT_DIM),
+                                (unsigned long long)(verify_first % OUT_DIM));
+                    }
+                    CHECK(verify_mismatches == 0,
+                          "timing projection bitwise mismatch");
+                }
+            }
+        }
+
+        /* Time the complete public production selector after the kernel-only
+         * panels: F32->F16 RHS copy, Q4->F16 transient expansion, F16/F16 MM,
+         * and head norm/RoPE are all included.  The control is the native
+         * Q4/F32 projection with the identical tail.  Alternate first place
+         * to keep command-order and thermal bias paired. */
+        if (timing_tokens == 4096u) {
+            CHECK(ds4_gpu_release_q4_attn_q_b_f16_sidecars() != 0,
+                  "release sidecar before production timing");
+            ds4_gpu_test_q4_attn_q_b_f16_cache_reset();
+            CHECK(unsetenv(k_require) == 0,
+                  "clear REQUIRE for production timing");
+            CHECK(unsetenv(k_transient_min_tokens) == 0,
+                  "use default production transient threshold");
+
+            CHECK(run_reference(
+                      timing_reference, model, model_bytes,
+                      timing_x, timing_tokens) == 1,
+                  "production timing native warmup");
+            CHECK(run_candidate(
+                      timing_out, timing_rhs_f16, model, model_bytes,
+                      timing_x, timing_tokens) == 1,
+                  "production timing transient warmup");
+
+            for (uint32_t sample = 0; sample < TIMING_SAMPLES; sample++) {
+                for (uint32_t pos = 0; pos < 2u; pos++) {
+                    const bool run_transient =
+                        ((sample & 1u) == 0u) ? pos == 1u : pos == 0u;
+                    const double t0 = monotonic_ms();
+                    const int ok = run_transient
+                        ? run_candidate(
+                            timing_out, timing_rhs_f16,
+                            model, model_bytes, timing_x, timing_tokens)
+                        : run_reference(
+                            timing_reference, model, model_bytes,
+                            timing_x, timing_tokens);
+                    CHECK(ok == 1, "production timing dispatch");
+                    const double elapsed = monotonic_ms() - t0;
+                    if (run_transient) {
+                        production_transient_ms[sample] = elapsed;
+                    } else {
+                        production_reference_ms[sample] = elapsed;
+                    }
+                }
+            }
+
+            const double production_reference_median =
+                timing_quantile(production_reference_ms, 0.5);
+            const double production_reference_p25 =
+                timing_quantile(production_reference_ms, 0.25);
+            const double production_reference_p75 =
+                timing_quantile(production_reference_ms, 0.75);
+            const double production_transient_median =
+                timing_quantile(production_transient_ms, 0.5);
+            const double production_transient_p25 =
+                timing_quantile(production_transient_ms, 0.25);
+            const double production_transient_p75 =
+                timing_quantile(production_transient_ms, 0.75);
+            const double production_speedup =
+                timing_paired_geomean_speedup(
+                    production_reference_ms, production_transient_ms);
+            fprintf(stderr,
+                    "Metal Q4 attn_q_b public production timing N=4096\n"
+                    "  native Q4/F32 + tail  %.3f ms [%.3f, %.3f] control\n"
+                    "  transient full + tail %.3f ms [%.3f, %.3f] "
+                    "%.3fx paired-gmean\n",
+                    production_reference_median,
+                    production_reference_p25,
+                    production_reference_p75,
+                    production_transient_median,
+                    production_transient_p25,
+                    production_transient_p75,
+                    production_speedup);
+
+            /* Re-run both arms immediately before readback, then prove that
+             * the complete public entry point is exact at production N. */
+            CHECK(run_reference(
+                      timing_reference, model, model_bytes,
+                      timing_x, timing_tokens) == 1,
+                  "production timing final native reference");
+            CHECK(run_candidate(
+                      timing_out, timing_rhs_f16, model, model_bytes,
+                      timing_x, timing_tokens) == 1,
+                  "production timing final transient candidate");
+            uint64_t production_mismatches = 0u;
+            for (uint64_t offset = 0; offset < verify_bytes;
+                 offset += verify_chunk_bytes) {
+                const uint64_t chunk_bytes =
+                    verify_bytes - offset < verify_chunk_bytes
+                        ? verify_bytes - offset
+                        : verify_chunk_bytes;
+                CHECK(ds4_gpu_tensor_read(
+                          timing_reference, offset,
+                          verify_reference, chunk_bytes) != 0,
+                      "production timing reference readback");
+                CHECK(ds4_gpu_tensor_read(
+                          timing_out, offset,
+                          verify_candidate, chunk_bytes) != 0,
+                      "production timing transient readback");
+                uint64_t production_chunk_first = UINT64_MAX;
+                production_mismatches += count_bit_mismatches(
+                    verify_reference, verify_candidate,
+                    chunk_bytes / sizeof(float),
+                    &production_chunk_first);
+            }
+            CHECK(production_mismatches == 0u,
+                  "production timing public entry point mismatch");
+            ds4_gpu_q4_attn_q_b_f16_cache_report production_report;
+            ds4_gpu_test_q4_attn_q_b_f16_cache_report(
+                &production_report);
+            CHECK(production_report.entries == 0u &&
+                      production_report.bytes == 0u &&
+                      production_report.lookups == 0u &&
+                      production_report.builds == 0u,
+                  "production timing retained a sidecar");
+            CHECK(setenv(k_require, "1", 1) == 0,
+                  "restore REQUIRE after production timing");
+        }
+        free(verify_candidate);
+        free(verify_reference);
+        ds4_gpu_tensor_free(timing_reference);
+
+        ds4_gpu_tensor_free(timing_rhs_f16);
+        ds4_gpu_tensor_free(timing_out);
+        ds4_gpu_tensor_free(timing_x);
+    }
+
+    ds4_gpu_tensor_free(x);
+    ds4_gpu_tensor_free(q_half_base);
+    ds4_gpu_tensor_free(candidate_base);
+    ds4_gpu_tensor_free(reference_base);
+    ds4_gpu_tensor_free(x_base);
+    ds4_gpu_cleanup();
+
+    free(q_half_host);
+    free(candidate_host);
+    free(reference_host);
+    free(input_readback);
+    free(input_host);
+    free(support_model);
+    free(ssd_transient_model);
+    free(model);
+
+    CHECK(unsetenv(k_require) == 0, "clear cache require env at exit");
+    CHECK(unsetenv(k_min_tokens) == 0,
+          "clear cache minimum env at exit");
+    CHECK(unsetenv(k_disable_transient_f16) == 0,
+          "clear transient F16 disable env at exit");
+    CHECK(unsetenv(k_transient_min_tokens) == 0,
+          "clear transient F16 minimum env at exit");
+    fprintf(stderr,
+            "Metal Q4 attn_q_b F16 cache production geometry "
+            "1024x32768 N=32/33/64: PASS\n");
+    return 0;
+}
+
+#else
+
+int main(void) {
+    fprintf(stderr,
+            "Metal Q4 attn_q_b F16 cache oracle SKIP: requires macOS\n");
+    return 0;
+}
+
+#endif
diff --git a/tests/test_metal_q4_streams.c b/tests/test_metal_q4_streams.c
new file mode 100644
index 0000000000..447c6db4c4
--- /dev/null
+++ b/tests/test_metal_q4_streams.c
@@ -0,0 +1,885 @@
+#define _DARWIN_C_SOURCE
+
+/* Resident Q4_K oracle for the per-stream Metal command queues.
+ *
+ * This deliberately uses a tiny synthetic mmap-shaped model instead of a
+ * production GGUF: the local Q4 target is much larger than unified memory.
+ * Each row is evaluated three ways: synchronous FIFO, one native row batch,
+ * and one command buffer per stream.  All three retain the same Q4_K kernel
+ * reduction order and therefore must be bit-identical.
+ *
+ * The default is a short correctness/leak smoke.  Set
+ * DS4_TEST_Q4_STREAM_SOAK=N for a bounded longer overlap soak, and
+ * DS4_TEST_Q4_STREAM_TIMING=1 to report wall-clock A/B numbers.
+ */
+
+#include "ds4.h"
+#include "ds4_gpu.h"
+
+#include <math.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#ifdef __APPLE__
+
+#include <mach/mach.h>
+#include <sys/resource.h>
+#include <time.h>
+
+#define Q4_K_TYPE 12u
+#define QK_K 256u
+/* DeepSeek-V4 Flash AProjQ4 q_a/attn_kv projection geometry. */
+#define IN_DIM 4096u
+#define OUT0_DIM 1024u
+#define OUT1_DIM 512u
+#define MAX_STREAMS 8u
+#define GUARD_FLOATS 64u
+#define MAX_TIMING_BLOCKS 51u
+
+typedef struct {
+    uint16_t d;
+    uint16_t dmin;
+    uint8_t scales[12];
+    uint8_t qs[QK_K / 2u];
+} block_q4_K;
+
+typedef enum {
+    ARM_FIFO,
+    ARM_NATIVE,
+    ARM_OVERLAP,
+} test_arm;
+
+typedef struct {
+    uint64_t footprint;
+    uint64_t resident;
+    uint64_t virtual_size;
+    uint64_t max_rss;
+} task_memory;
+
+typedef struct {
+    void *model;
+    uint64_t model_size;
+    uint64_t weight0_offset;
+    uint64_t weight1_offset;
+    uint64_t row_bytes;
+    ds4_gpu_tensor *x;
+    ds4_gpu_tensor *x_row[MAX_STREAMS];
+    ds4_gpu_tensor *out[3][2];
+    ds4_gpu_tensor *out_row[3][2][MAX_STREAMS];
+    float *host[3][2];
+    uint64_t out_count[2];
+    float *x_host;
+} fixture;
+
+static void fail(const char *what) {
+    fprintf(stderr, "Q4 stream oracle FAIL: %s\n", what);
+    exit(1);
+}
+
+typedef struct {
+    const char *name;
+    char *value;
+    bool present;
+} saved_env;
+
+static saved_env save_env(const char *name) {
+    const char *value = getenv(name);
+    saved_env saved = {
+        .name = name,
+        .value = value ? strdup(value) : NULL,
+        .present = value != NULL,
+    };
+    if (value && !saved.value) fail("environment snapshot");
+    return saved;
+}
+
+static void restore_env(saved_env *saved) {
+    const int rc = saved->present
+        ? setenv(saved->name, saved->value, 1)
+        : unsetenv(saved->name);
+    free(saved->value);
+    saved->value = NULL;
+    if (rc != 0) fail("environment restore");
+}
+
+static void expect_overlap_policy(const char *label, int expected,
+                                  int count, bool resident,
+                                  bool ssd_streaming, bool quality) {
+    const int actual = ds4_test_q4_stream_overlap_policy(
+        count, resident, ssd_streaming, quality);
+    if (actual != expected) {
+        fprintf(stderr,
+                "Q4 stream policy %s: got=%d expected=%d "
+                "count=%d resident=%d ssd=%d quality=%d\n",
+                label, actual, expected, count,
+                resident ? 1 : 0, ssd_streaming ? 1 : 0,
+                quality ? 1 : 0);
+        fail("scheduler admission policy");
+    }
+}
+
+static void test_overlap_policy(void) {
+    saved_env enabled = save_env("DS4_METAL_ENABLE_Q4_STREAM_OVERLAP");
+    saved_env disabled = save_env("DS4_METAL_DISABLE_Q4_STREAM_OVERLAP");
+
+    if (unsetenv(enabled.name) != 0 || unsetenv(disabled.name) != 0) {
+        fail("environment clear");
+    }
+    expect_overlap_policy("default-off", 0, 2, true, false, false);
+
+    if (setenv(enabled.name, "1", 1) != 0) fail("enable policy");
+    expect_overlap_policy("minimum-count", 1, 2, true, false, false);
+    expect_overlap_policy("maximum-count", 1, 8, true, false, false);
+    expect_overlap_policy("count-one", 0, 1, true, false, false);
+    expect_overlap_policy("count-nine", 0, 9, true, false, false);
+    expect_overlap_policy("nonresident", 0, 2, false, false, false);
+    expect_overlap_policy("ssd", 0, 2, true, true, false);
+    expect_overlap_policy("quality", 0, 2, true, false, true);
+
+    if (setenv(disabled.name, "1", 1) != 0) fail("disable policy");
+    expect_overlap_policy("disable-precedence", 0, 2, true, false, false);
+    if (setenv(disabled.name, "0", 1) != 0) fail("clear disable policy");
+    expect_overlap_policy("disable-zero", 1, 2, true, false, false);
+    if (setenv(enabled.name, "0", 1) != 0) fail("clear enable policy");
+    expect_overlap_policy("enable-zero", 0, 2, true, false, false);
+
+    restore_env(&disabled);
+    restore_env(&enabled);
+    fprintf(stderr,
+            "Q4 stream policy PASS default-off=1 disable-precedence=1 "
+            "count-bounds=1 resident-only=1 ssd-fallback=1 quality-fallback=1\n");
+}
+
+static void test_indexer_q_type_policy(void) {
+    if (!ds4_test_indexer_q_type_supported(1u) ||
+        !ds4_test_indexer_q_type_supported(8u) ||
+        !ds4_test_indexer_q_type_supported(Q4_K_TYPE) ||
+        ds4_test_indexer_q_type_supported(0u) ||
+        ds4_test_indexer_q_type_supported(2u)) {
+        fail("indexer query projection type policy");
+    }
+}
+
+static uint64_t env_u64(const char *name, uint64_t fallback,
+                        uint64_t minimum, uint64_t maximum) {
+    const char *value = getenv(name);
+    if (!value || !value[0]) return fallback;
+    char *end = NULL;
+    unsigned long long parsed = strtoull(value, &end, 10);
+    if (end == value || *end != '\0' || parsed < minimum || parsed > maximum) {
+        fprintf(stderr, "Q4 stream oracle invalid %s=%s (range %llu..%llu)\n",
+                name, value,
+                (unsigned long long)minimum,
+                (unsigned long long)maximum);
+        exit(1);
+    }
+    return (uint64_t)parsed;
+}
+
+static uint64_t align_up(uint64_t value, uint64_t alignment) {
+    return (value + alignment - 1u) / alignment * alignment;
+}
+
+static double now_seconds(void) {
+    struct timespec ts;
+    if (clock_gettime(CLOCK_MONOTONIC, &ts) != 0) fail("clock_gettime");
+    return (double)ts.tv_sec + (double)ts.tv_nsec * 1e-9;
+}
+
+static task_memory read_task_memory(void) {
+    task_memory result = {0};
+    task_vm_info_data_t info;
+    mach_msg_type_number_t count = TASK_VM_INFO_COUNT;
+    if (task_info(mach_task_self(), TASK_VM_INFO,
+                  (task_info_t)&info, &count) == KERN_SUCCESS) {
+        result.footprint = (uint64_t)info.phys_footprint;
+        result.resident = (uint64_t)info.resident_size;
+        result.virtual_size = (uint64_t)info.virtual_size;
+    }
+    struct rusage usage;
+    if (getrusage(RUSAGE_SELF, &usage) == 0) {
+        /* ru_maxrss is bytes on Darwin. */
+        result.max_rss = (uint64_t)usage.ru_maxrss;
+    }
+    return result;
+}
+
+static void print_task_memory(const char *label, task_memory memory) {
+    const double mib = 1024.0 * 1024.0;
+    fprintf(stderr,
+            "Q4 stream memory %-12s footprint=%.2f MiB resident=%.2f MiB "
+            "virtual=%.2f MiB peak_rss=%.2f MiB\n",
+            label,
+            (double)memory.footprint / mib,
+            (double)memory.resident / mib,
+            (double)memory.virtual_size / mib,
+            (double)memory.max_rss / mib);
+}
+
+static float f16_to_f32(uint16_t h) {
+    const uint32_t sign = (uint32_t)(h & 0x8000u) << 16u;
+    uint32_t exp = (h >> 10u) & 0x1fu;
+    uint32_t mant = h & 0x03ffu;
+    uint32_t bits;
+    if (exp == 0u) {
+        if (mant == 0u) {
+            bits = sign;
+        } else {
+            exp = 1u;
+            while ((mant & 0x0400u) == 0u) {
+                mant <<= 1u;
+                exp--;
+            }
+            mant &= 0x03ffu;
+            bits = sign | ((exp + 127u - 15u) << 23u) | (mant << 13u);
+        }
+    } else if (exp == 31u) {
+        bits = sign | 0x7f800000u | (mant << 13u);
+    } else {
+        bits = sign | ((exp + 127u - 15u) << 23u) | (mant << 13u);
+    }
+    float value;
+    memcpy(&value, &bits, sizeof(value));
+    return value;
+}
+
+static void q4_scale_min(const uint8_t packed[12], uint32_t group,
+                         uint8_t *scale, uint8_t *minimum) {
+    if (group < 4u) {
+        *scale = packed[group] & 63u;
+        *minimum = packed[group + 4u] & 63u;
+    } else {
+        *scale = (packed[group + 4u] & 15u) |
+                 ((packed[group - 4u] >> 6u) << 4u);
+        *minimum = (packed[group + 4u] >> 4u) |
+                   ((packed[group] >> 6u) << 4u);
+    }
+}
+
+static void q4_pack_scales(uint8_t packed[12], const uint8_t scale[8],
+                           const uint8_t minimum[8]) {
+    memset(packed, 0, 12u);
+    for (uint32_t group = 0; group < 4u; group++) {
+        packed[group] = scale[group] & 63u;
+        packed[group + 4u] = minimum[group] & 63u;
+    }
+    for (uint32_t group = 4u; group < 8u; group++) {
+        packed[group + 4u] = (scale[group] & 15u) |
+                             ((minimum[group] & 15u) << 4u);
+        packed[group - 4u] |= (scale[group] >> 4u) << 6u;
+        packed[group] |= (minimum[group] >> 4u) << 6u;
+    }
+}
+
+static void fill_q4_matrix(block_q4_K *matrix, uint32_t rows, uint32_t salt) {
+    const uint32_t blocks_per_row = IN_DIM / QK_K;
+    for (uint32_t row = 0; row < rows; row++) {
+        for (uint32_t block = 0; block < blocks_per_row; block++) {
+            block_q4_K *b = matrix + (uint64_t)row * blocks_per_row + block;
+            const uint32_t key = salt + row * 1009u + block * 313u;
+            uint8_t scale[8];
+            uint8_t minimum[8];
+            for (uint32_t group = 0; group < 8u; group++) {
+                scale[group] = (uint8_t)(1u + (key + group * 7u) % 31u);
+                minimum[group] = (uint8_t)((key / 3u + group * 5u) % 17u);
+            }
+            q4_pack_scales(b->scales, scale, minimum);
+            for (uint32_t i = 0; i < QK_K / 2u; i++) {
+                b->qs[i] = (uint8_t)(key + i * 37u + (i >> 2u) * 11u);
+            }
+            /* Exact binary scales: 2^-5 and 2^-7. */
+            b->d = 0x2800u;
+            b->dmin = 0x2000u;
+        }
+    }
+}
+
+static float q4_dot(const block_q4_K *row, const float *x) {
+    float sum = 0.0f;
+    for (uint32_t k = 0; k < IN_DIM; k++) {
+        const block_q4_K *b = row + k / QK_K;
+        const uint32_t in_block = k % QK_K;
+        const uint32_t group = in_block / 32u;
+        const uint32_t lane = in_block % 32u;
+        uint8_t scale, minimum;
+        q4_scale_min(b->scales, group, &scale, &minimum);
+        const uint32_t byte_offset = (group >> 1u) * 32u + lane;
+        const uint32_t shift = (group & 1u) * 4u;
+        const uint32_t q = (b->qs[byte_offset] >> shift) & 15u;
+        const float w = f16_to_f32(b->d) * (float)scale * (float)q -
+                        f16_to_f32(b->dmin) * (float)minimum;
+        sum += w * x[k];
+    }
+    return sum;
+}
+
+static uint64_t checksum(const float *values, uint64_t count) {
+    const uint8_t *bytes = (const uint8_t *)values;
+    const uint64_t byte_count = count * sizeof(float);
+    uint64_t hash = UINT64_C(1469598103934665603);
+    for (uint64_t i = 0; i < byte_count; i++) {
+        hash ^= bytes[i];
+        hash *= UINT64_C(1099511628211);
+    }
+    return hash;
+}
+
+static const char *arm_name(test_arm arm) {
+    switch (arm) {
+    case ARM_FIFO: return "fifo";
+    case ARM_NATIVE: return "native";
+    case ARM_OVERLAP: return "overlap";
+    }
+    return "unknown";
+}
+
+static int encode_pair(fixture *f, ds4_gpu_tensor *out0,
+                       ds4_gpu_tensor *out1, const ds4_gpu_tensor *x,
+                       uint32_t rows) {
+    return ds4_gpu_matmul_q4_K_pair_tensor(
+        out0, out1, f->model, f->model_size,
+        f->weight0_offset, f->weight1_offset,
+        IN_DIM, OUT0_DIM, OUT1_DIM, x, rows) > 0;
+}
+
+static int run_arm_once_with_options(fixture *f, test_arm arm,
+                                     uint32_t streams,
+                                     bool hold_test_transient) {
+    if (arm == ARM_NATIVE) {
+        ds4_gpu_set_stream(0);
+        return encode_pair(f, f->out[arm][0], f->out[arm][1], f->x, streams);
+    }
+    if (arm == ARM_FIFO) {
+        ds4_gpu_set_stream(0);
+        for (uint32_t i = 0; i < streams; i++) {
+            if (!encode_pair(f, f->out_row[arm][0][i],
+                             f->out_row[arm][1][i], f->x_row[i], 1u)) {
+                return 0;
+            }
+        }
+        return 1;
+    }
+
+    uint32_t submitted = 0;
+    for (uint32_t i = 0; i < streams; i++) {
+        ds4_gpu_set_stream((int)i);
+        if (!ds4_gpu_begin_commands() ||
+            (hold_test_transient &&
+             !ds4_gpu_test_hold_stream_transient(4096u)) ||
+            !encode_pair(f, f->out_row[arm][0][i],
+                         f->out_row[arm][1][i], f->x_row[i], 1u) ||
+            !ds4_gpu_end_commands_async()) {
+            goto fail_overlap;
+        }
+        submitted++;
+    }
+    for (uint32_t i = 0; i < submitted; i++) {
+        if (!ds4_gpu_wait_stream((int)i)) goto fail_overlap;
+    }
+    ds4_gpu_set_stream(0);
+    return 1;
+
+fail_overlap:
+    for (uint32_t i = 0; i < submitted; i++) {
+        (void)ds4_gpu_wait_stream((int)i);
+    }
+    ds4_gpu_set_stream(0);
+    return 0;
+}
+
+static int run_arm_once(fixture *f, test_arm arm, uint32_t streams) {
+    return run_arm_once_with_options(f, arm, streams, true);
+}
+
+/* Exercise the intermediate-command-buffer lifetime too: the first pair and
+ * its test resource must survive flush, while wait_stream must retire both
+ * the pending and final command buffers before releasing transients. */
+static int run_overlap_flush_once(fixture *f, uint32_t streams) {
+    uint32_t submitted = 0;
+    for (uint32_t i = 0; i < streams; i++) {
+        ds4_gpu_set_stream((int)i);
+        if (!ds4_gpu_begin_commands() ||
+            !ds4_gpu_test_hold_stream_transient(4096u) ||
+            !encode_pair(f, f->out_row[ARM_OVERLAP][0][i],
+                         f->out_row[ARM_OVERLAP][1][i], f->x_row[i], 1u) ||
+            !ds4_gpu_flush_commands() ||
+            !encode_pair(f, f->out_row[ARM_OVERLAP][0][i],
+                         f->out_row[ARM_OVERLAP][1][i], f->x_row[i], 1u) ||
+            !ds4_gpu_end_commands_async()) {
+            goto fail_overlap;
+        }
+        submitted++;
+    }
+    for (uint32_t i = 0; i < submitted; i++) {
+        if (!ds4_gpu_wait_stream((int)i)) goto fail_overlap;
+    }
+    ds4_gpu_set_stream(0);
+    return 1;
+
+fail_overlap:
+    for (uint32_t i = 0; i < submitted; i++) {
+        (void)ds4_gpu_wait_stream((int)i);
+    }
+    ds4_gpu_set_stream(0);
+    return 0;
+}
+
+static void poison_arm(fixture *f, test_arm arm, float poison) {
+    for (uint32_t output = 0; output < 2u; output++) {
+        for (uint64_t i = 0; i < f->out_count[output]; i++) {
+            f->host[arm][output][i] = poison;
+        }
+        if (!ds4_gpu_tensor_write(f->out[arm][output], 0,
+                                  f->host[arm][output],
+                                  f->out_count[output] * sizeof(float))) {
+            fail("output poison write");
+        }
+    }
+}
+
+static void read_arm(fixture *f, test_arm arm) {
+    for (uint32_t output = 0; output < 2u; output++) {
+        if (!ds4_gpu_tensor_read(f->out[arm][output], 0,
+                                 f->host[arm][output],
+                                 f->out_count[output] * sizeof(float))) {
+            fail("output read");
+        }
+    }
+}
+
+static void check_outputs(fixture *f, uint32_t streams, float poison) {
+    const uint32_t dims[2] = {OUT0_DIM, OUT1_DIM};
+    const uint64_t offsets[2] = {f->weight0_offset, f->weight1_offset};
+    for (test_arm arm = ARM_FIFO; arm <= ARM_OVERLAP; arm++) read_arm(f, arm);
+
+    for (uint32_t output = 0; output < 2u; output++) {
+        const uint64_t active = (uint64_t)streams * dims[output];
+        const block_q4_K *matrix = (const block_q4_K *)
+            ((const uint8_t *)f->model + offsets[output]);
+        float max_abs = 0.0f;
+        float max_rel = 0.0f;
+        for (uint32_t stream = 0; stream < streams; stream++) {
+            for (uint32_t row = 0; row < dims[output]; row++) {
+                const uint64_t index = (uint64_t)stream * dims[output] + row;
+                const float expected = q4_dot(
+                    matrix + (uint64_t)row * (IN_DIM / QK_K),
+                    f->x_host + (uint64_t)stream * IN_DIM);
+                const float actual = f->host[ARM_FIFO][output][index];
+                const float absolute = fabsf(actual - expected);
+                const float relative = absolute / fmaxf(1.0f, fabsf(expected));
+                if (absolute > max_abs) max_abs = absolute;
+                if (relative > max_rel) max_rel = relative;
+                if (!isfinite(actual) || (absolute > 0.004f && relative > 2e-5f)) {
+                    fprintf(stderr,
+                            "Q4 stream CPU mismatch output=%u stream=%u row=%u "
+                            "expected=%g actual=%g abs=%g rel=%g\n",
+                            output, stream, row, expected, actual,
+                            absolute, relative);
+                    fail("CPU tolerance");
+                }
+            }
+        }
+        for (uint64_t i = active; i < f->out_count[output]; i++) {
+            for (test_arm arm = ARM_FIFO; arm <= ARM_OVERLAP; arm++) {
+                if (memcmp(&f->host[arm][output][i], &poison,
+                           sizeof(poison)) != 0) {
+                    fprintf(stderr,
+                            "Q4 stream canary mismatch arm=%s output=%u index=%llu\n",
+                            arm_name(arm), output, (unsigned long long)i);
+                    fail("output canary");
+                }
+            }
+        }
+        for (test_arm arm = ARM_NATIVE; arm <= ARM_OVERLAP; arm++) {
+            if (memcmp(f->host[ARM_FIFO][output], f->host[arm][output],
+                       active * sizeof(float)) != 0) {
+                uint64_t first = 0;
+                while (first < active &&
+                       memcmp(&f->host[ARM_FIFO][output][first],
+                              &f->host[arm][output][first], sizeof(float)) == 0) {
+                    first++;
+                }
+                fprintf(stderr,
+                        "Q4 stream exact mismatch arm=%s output=%u first=%llu "
+                        "fifo=%g actual=%g\n",
+                        arm_name(arm), output, (unsigned long long)first,
+                        first < active ? f->host[ARM_FIFO][output][first] : 0.0f,
+                        first < active ? f->host[arm][output][first] : 0.0f);
+                fail("bitwise parity");
+            }
+        }
+        fprintf(stderr,
+                "Q4 stream N=%u output=%u CPU max_abs=%g max_rel=%g "
+                "checksum=%016llx bitwise=1 canary=1\n",
+                streams, output, max_abs, max_rel,
+                (unsigned long long)checksum(f->host[ARM_FIFO][output], active));
+    }
+}
+
+static void check_stats_equal(const ds4_gpu_stream_test_stats *before,
+                              const ds4_gpu_stream_test_stats *after,
+                              const char *scope) {
+    if (before->tensor_live_bytes != after->tensor_live_bytes ||
+        before->tensor_live_count != after->tensor_live_count ||
+        after->transient_references != 0u ||
+        after->pending_command_buffers != 0u ||
+        after->last_command_buffers != 0u) {
+        fprintf(stderr,
+                "Q4 stream stats leak scope=%s live_bytes=%llu->%llu "
+                "live_count=%u->%u transient=%llu->%llu pending=%u last=%u\n",
+                scope,
+                (unsigned long long)before->tensor_live_bytes,
+                (unsigned long long)after->tensor_live_bytes,
+                before->tensor_live_count, after->tensor_live_count,
+                (unsigned long long)before->transient_references,
+                (unsigned long long)after->transient_references,
+                after->pending_command_buffers,
+                after->last_command_buffers);
+        fail("Metal allocation/transient counters");
+    }
+}
+
+static int compare_double(const void *a, const void *b) {
+    const double av = *(const double *)a;
+    const double bv = *(const double *)b;
+    return (av > bv) - (av < bv);
+}
+
+static double percentile(const double *samples, uint32_t count, double p) {
+    double sorted[MAX_TIMING_BLOCKS];
+    memcpy(sorted, samples, (size_t)count * sizeof(sorted[0]));
+    qsort(sorted, count, sizeof(sorted[0]), compare_double);
+    const double rank = p * (double)(count - 1u);
+    const uint32_t lo = (uint32_t)rank;
+    const uint32_t hi = lo + 1u < count ? lo + 1u : lo;
+    const double fraction = rank - (double)lo;
+    return sorted[lo] + (sorted[hi] - sorted[lo]) * fraction;
+}
+
+static double run_timing_group(fixture *f, test_arm arm, uint32_t streams,
+                               uint64_t iterations) {
+    const double start = now_seconds();
+    for (uint64_t i = 0; i < iterations; i++) {
+        /* The transient hook is a lifecycle oracle, not production work.  Do
+         * not charge its allocation/retain overhead to the overlap arm. */
+        if (!run_arm_once_with_options(f, arm, streams, false)) {
+            fail("timing arm");
+        }
+    }
+    return (now_seconds() - start) / (double)iterations;
+}
+
+static void run_timing_pair(fixture *f, test_arm baseline, test_arm candidate,
+                            uint32_t streams, uint32_t blocks,
+                            uint64_t group_iterations) {
+    double baseline_samples[MAX_TIMING_BLOCKS];
+    double candidate_samples[MAX_TIMING_BLOCKS];
+    for (uint32_t block = 0; block < blocks; block++) {
+        double base_total = 0.0;
+        double candidate_total = 0.0;
+        if ((block & 1u) == 0u) {
+            base_total += run_timing_group(
+                f, baseline, streams, group_iterations);
+            candidate_total += run_timing_group(
+                f, candidate, streams, group_iterations);
+            candidate_total += run_timing_group(
+                f, candidate, streams, group_iterations);
+            base_total += run_timing_group(
+                f, baseline, streams, group_iterations);
+        } else {
+            candidate_total += run_timing_group(
+                f, candidate, streams, group_iterations);
+            base_total += run_timing_group(
+                f, baseline, streams, group_iterations);
+            base_total += run_timing_group(
+                f, baseline, streams, group_iterations);
+            candidate_total += run_timing_group(
+                f, candidate, streams, group_iterations);
+        }
+        baseline_samples[block] = base_total * 0.5;
+        candidate_samples[block] = candidate_total * 0.5;
+    }
+
+    const double baseline_median = percentile(baseline_samples, blocks, 0.50);
+    const double candidate_median = percentile(candidate_samples, blocks, 0.50);
+    const double speedup = candidate_median > 0.0 ?
+        baseline_median / candidate_median : 0.0;
+    fprintf(stderr,
+            "Q4 stream timing microkernel=%s/%s N=%u blocks=%u group_iters=%llu "
+            "baseline_ms[p25/med/p75]=%.3f/%.3f/%.3f "
+            "candidate_ms[p25/med/p75]=%.3f/%.3f/%.3f speedup=%.3fx "
+            "aggregate_candidate=%.2f rows/s wall=encode+GPU full_logits=0\n",
+            arm_name(baseline), arm_name(candidate), streams, blocks,
+            (unsigned long long)group_iterations,
+            percentile(baseline_samples, blocks, 0.25) * 1000.0,
+            baseline_median * 1000.0,
+            percentile(baseline_samples, blocks, 0.75) * 1000.0,
+            percentile(candidate_samples, blocks, 0.25) * 1000.0,
+            candidate_median * 1000.0,
+            percentile(candidate_samples, blocks, 0.75) * 1000.0,
+            speedup,
+            candidate_median > 0.0 ? (double)streams / candidate_median : 0.0);
+}
+
+static void fixture_init(fixture *f) {
+    memset(f, 0, sizeof(*f));
+    if (sizeof(block_q4_K) != 144u) fail("unexpected Q4_K block size");
+
+    f->row_bytes = (IN_DIM / QK_K) * sizeof(block_q4_K);
+    f->weight0_offset = 0;
+    f->weight1_offset = OUT0_DIM * f->row_bytes;
+    const uint64_t weights_end = f->weight1_offset + OUT1_DIM * f->row_bytes;
+    const uint64_t page = (uint64_t)getpagesize();
+    f->model_size = align_up(weights_end, page);
+    if (posix_memalign(&f->model, (size_t)page, (size_t)f->model_size) != 0) {
+        fail("model allocation");
+    }
+    memset(f->model, 0, (size_t)f->model_size);
+    fill_q4_matrix((block_q4_K *)((uint8_t *)f->model + f->weight0_offset),
+                   OUT0_DIM, 17u);
+    fill_q4_matrix((block_q4_K *)((uint8_t *)f->model + f->weight1_offset),
+                   OUT1_DIM, 7919u);
+
+    const uint64_t x_count = (uint64_t)MAX_STREAMS * IN_DIM;
+    f->x_host = malloc((size_t)x_count * sizeof(float));
+    if (!f->x_host) fail("host activation allocation");
+    for (uint32_t stream = 0; stream < MAX_STREAMS; stream++) {
+        for (uint32_t k = 0; k < IN_DIM; k++) {
+            const int32_t value = (int32_t)(
+                (stream * 19u + k * 7u + (stream ^ k) * 3u) % 127u) - 63;
+            f->x_host[(uint64_t)stream * IN_DIM + k] = (float)value / 128.0f;
+        }
+    }
+
+    if (!ds4_gpu_init() || !ds4_gpu_set_model_map(f->model, f->model_size)) {
+        fail("Metal init/model map");
+    }
+    ds4_gpu_set_quality(false);
+    f->x = ds4_gpu_tensor_alloc(x_count * sizeof(float));
+    if (!f->x || !ds4_gpu_tensor_write(f->x, 0, f->x_host,
+                                        x_count * sizeof(float))) {
+        fail("activation tensor");
+    }
+    for (uint32_t stream = 0; stream < MAX_STREAMS; stream++) {
+        f->x_row[stream] = ds4_gpu_tensor_view(
+            f->x, (uint64_t)stream * IN_DIM * sizeof(float),
+            IN_DIM * sizeof(float));
+        if (!f->x_row[stream]) fail("activation row view");
+    }
+
+    const uint32_t dims[2] = {OUT0_DIM, OUT1_DIM};
+    for (uint32_t output = 0; output < 2u; output++) {
+        f->out_count[output] = (uint64_t)MAX_STREAMS * dims[output] + GUARD_FLOATS;
+        for (test_arm arm = ARM_FIFO; arm <= ARM_OVERLAP; arm++) {
+            f->host[arm][output] = malloc(
+                (size_t)f->out_count[output] * sizeof(float));
+            f->out[arm][output] = ds4_gpu_tensor_alloc(
+                f->out_count[output] * sizeof(float));
+            if (!f->host[arm][output] || !f->out[arm][output]) {
+                fail("output allocation");
+            }
+            for (uint32_t stream = 0; stream < MAX_STREAMS; stream++) {
+                f->out_row[arm][output][stream] = ds4_gpu_tensor_view(
+                    f->out[arm][output],
+                    (uint64_t)stream * dims[output] * sizeof(float),
+                    (uint64_t)dims[output] * sizeof(float));
+                if (!f->out_row[arm][output][stream]) {
+                    fail("output row view");
+                }
+            }
+        }
+    }
+}
+
+static void fixture_release_tensors(fixture *f) {
+    for (uint32_t output = 0; output < 2u; output++) {
+        for (test_arm arm = ARM_FIFO; arm <= ARM_OVERLAP; arm++) {
+            for (uint32_t stream = 0; stream < MAX_STREAMS; stream++) {
+                ds4_gpu_tensor_free(f->out_row[arm][output][stream]);
+            }
+            ds4_gpu_tensor_free(f->out[arm][output]);
+            free(f->host[arm][output]);
+        }
+    }
+    for (uint32_t stream = 0; stream < MAX_STREAMS; stream++) {
+        ds4_gpu_tensor_free(f->x_row[stream]);
+    }
+    ds4_gpu_tensor_free(f->x);
+    f->x = NULL;
+}
+
+static void fixture_destroy(fixture *f) {
+    fixture_release_tensors(f);
+
+    /* Cleanup itself is responsible for every stream.  Leave one empty async
+     * command buffer and one retained test resource on the last stream so a
+     * cleanup implementation that only drains stream zero cannot false-pass. */
+    ds4_gpu_set_stream((int)MAX_STREAMS - 1);
+    if (!ds4_gpu_begin_commands() ||
+        !ds4_gpu_test_hold_stream_transient(4096u) ||
+        !ds4_gpu_end_commands_async()) {
+        fail("cleanup in-flight setup");
+    }
+    ds4_gpu_stream_test_stats inflight;
+    ds4_gpu_test_stream_stats(&inflight);
+    if (inflight.last_command_buffers == 0u ||
+        inflight.transient_references == 0u) {
+        fail("cleanup in-flight state was not observable");
+    }
+    ds4_gpu_cleanup();
+    free(f->x_host);
+    free(f->model);
+    memset(f, 0, sizeof(*f));
+}
+
+int main(void) {
+    static const uint32_t stream_counts[] = {2u, 4u, 8u};
+    const float poison = -12345.25f;
+    const uint64_t warmup = env_u64("DS4_TEST_Q4_STREAM_WARMUP", 1u, 1u, 64u);
+    const uint64_t iterations = env_u64(
+        "DS4_TEST_Q4_STREAM_ITERS", 2u, 1u, 10000u);
+    const uint64_t soak = env_u64(
+        "DS4_TEST_Q4_STREAM_SOAK", 8u, 1u, 100000u);
+    const bool timing = getenv("DS4_TEST_Q4_STREAM_TIMING") != NULL;
+    const uint32_t timing_blocks = (uint32_t)env_u64(
+        "DS4_TEST_Q4_STREAM_TIMING_BLOCKS", 5u, 5u, MAX_TIMING_BLOCKS);
+    const uint64_t timing_iterations = env_u64(
+        "DS4_TEST_Q4_STREAM_TIMING_ITERS", 20u, 1u, 10000u);
+
+    test_overlap_policy();
+    test_indexer_q_type_policy();
+
+    fprintf(stderr,
+            "Q4 stream oracle model_untracked=%s warmup=%llu iterations=%llu "
+            "soak=%llu timing=%d footprint=synth-resident\n",
+            getenv("DS4_METAL_MODEL_UNTRACKED") ? "on" : "off",
+            (unsigned long long)warmup,
+            (unsigned long long)iterations,
+            (unsigned long long)soak,
+            timing ? 1 : 0);
+    const task_memory process_before = read_task_memory();
+    print_task_memory("process-start", process_before);
+
+    fixture f;
+    fixture_init(&f);
+    ds4_gpu_stream_test_stats initial;
+    ds4_gpu_test_stream_stats(&initial);
+    if (initial.active_queue_mask != 1u ||
+        initial.model_residency_queue_mask != 1u) {
+        fail("initial model residency");
+    }
+    for (uint32_t c = 0; c < sizeof(stream_counts) / sizeof(stream_counts[0]); c++) {
+        const uint32_t streams = stream_counts[c];
+        for (test_arm arm = ARM_FIFO; arm <= ARM_OVERLAP; arm++) {
+            poison_arm(&f, arm, poison);
+            for (uint64_t i = 0; i < warmup; i++) {
+                if (!run_arm_once(&f, arm, streams)) fail("warmup arm");
+            }
+        }
+
+        ds4_gpu_stream_test_stats stats_before;
+        ds4_gpu_test_stream_stats(&stats_before);
+        if (stats_before.transient_references != 0u) {
+            fail("transients retained after warmup");
+        }
+
+        for (test_arm arm = ARM_FIFO; arm <= ARM_OVERLAP; arm++) {
+            for (uint64_t i = 0; i < iterations; i++) {
+                if (!run_arm_once(&f, arm, streams)) fail("test arm");
+            }
+        }
+        check_outputs(&f, streams, poison);
+        if (timing) {
+            run_timing_pair(&f, ARM_FIFO, ARM_OVERLAP, streams,
+                            timing_blocks, timing_iterations);
+            run_timing_pair(&f, ARM_NATIVE, ARM_OVERLAP, streams,
+                            timing_blocks, timing_iterations);
+        }
+
+        ds4_gpu_stream_test_stats stats_after;
+        ds4_gpu_test_stream_stats(&stats_after);
+        check_stats_equal(&stats_before, &stats_after, "A/B");
+
+        if (!run_overlap_flush_once(&f, streams)) {
+            fail("overlap flush lifecycle");
+        }
+        ds4_gpu_stream_test_stats flush_after;
+        ds4_gpu_test_stream_stats(&flush_after);
+        check_stats_equal(&stats_before, &flush_after, "flush");
+        check_outputs(&f, streams, poison);
+        const uint32_t expected_mask = (1u << streams) - 1u;
+        if ((stats_after.active_queue_mask & expected_mask) != expected_mask ||
+            (stats_after.model_residency_queue_mask & expected_mask) != expected_mask) {
+            fprintf(stderr,
+                    "Q4 stream queue residency N=%u active=0x%02x "
+                    "model=0x%02x expected=0x%02x\n",
+                    streams, stats_after.active_queue_mask,
+                    stats_after.model_residency_queue_mask, expected_mask);
+            fail("residency missing from an active queue");
+        }
+
+        ds4_gpu_stream_test_stats soak_before;
+        ds4_gpu_test_stream_stats(&soak_before);
+        for (uint64_t i = 0; i < soak; i++) {
+            if (!run_arm_once(&f, ARM_OVERLAP, streams)) fail("overlap soak");
+        }
+        ds4_gpu_stream_test_stats soak_after;
+        ds4_gpu_test_stream_stats(&soak_after);
+        check_stats_equal(&soak_before, &soak_after, "soak");
+        fprintf(stderr,
+                "Q4 stream N=%u soak=%llu active_queues=0x%02x "
+                "model_residency=0x%02x transient=%llu counters=stable\n",
+                streams, (unsigned long long)soak,
+                soak_after.active_queue_mask,
+                soak_after.model_residency_queue_mask,
+                (unsigned long long)soak_after.transient_references);
+    }
+
+    /* Rebuild residency after all queues already exist.  Model replacement in
+     * a long-lived engine must attach the new set to every existing queue. */
+    if (!ds4_gpu_set_model_map(f.model, f.model_size)) {
+        fail("model residency rebuild");
+    }
+    ds4_gpu_stream_test_stats rebuilt;
+    ds4_gpu_test_stream_stats(&rebuilt);
+    if (rebuilt.active_queue_mask != 0xffu ||
+        rebuilt.model_residency_queue_mask != 0xffu) {
+        fprintf(stderr,
+                "Q4 residency rebuild active=0x%02x model=0x%02x\n",
+                rebuilt.active_queue_mask,
+                rebuilt.model_residency_queue_mask);
+        fail("model residency rebuild queue coverage");
+    }
+
+    print_task_memory("before-cleanup", read_task_memory());
+    fixture_destroy(&f);
+    ds4_gpu_stream_test_stats cleanup_stats;
+    ds4_gpu_test_stream_stats(&cleanup_stats);
+    if (cleanup_stats.tensor_live_bytes != 0u ||
+        cleanup_stats.tensor_live_count != 0u ||
+        cleanup_stats.transient_references != 0u ||
+        cleanup_stats.pending_command_buffers != 0u ||
+        cleanup_stats.last_command_buffers != 0u ||
+        cleanup_stats.active_queue_mask != 0u) {
+        fail("cleanup counters did not return to zero");
+    }
+    print_task_memory("after-cleanup", read_task_memory());
+    fprintf(stderr,
+            "test_metal_q4_streams PASS counts=2,4,8 bitwise=1 canary=1 "
+            "model_residency=1 counters=stable model_untracked=%s\n",
+            getenv("DS4_METAL_MODEL_UNTRACKED") ? "on" : "off");
+    return 0;
+}
+
+#else
+
+int main(void) {
+    fprintf(stderr, "test_metal_q4_streams skipped (Metal requires macOS)\n");
+    return 0;
+}
+
+#endif
diff --git a/tests/test_metal_session_batch.c b/tests/test_metal_session_batch.c
index 33eceff496..5bae389044 100644
--- a/tests/test_metal_session_batch.c
+++ b/tests/test_metal_session_batch.c
@@ -9,6 +9,7 @@
  */
 
 #include "ds4.h"
+#include "ds4_gpu.h"
 #include "ds4_tp.h"
 
 #include <float.h>
@@ -39,12 +40,23 @@ static const char *prompts[MAX_SESSION_COUNT] = {
 static float observed_max_abs;
 static bool compare_argmax_only;
 
+static bool env_flag(const char *name) {
+    const char *value = getenv(name);
+    return value && value[0] && strcmp(value, "0") != 0;
+}
+
 static double now_seconds(void) {
     struct timespec ts;
     clock_gettime(CLOCK_MONOTONIC, &ts);
     return (double)ts.tv_sec + (double)ts.tv_nsec / 1000000000.0;
 }
 
+static double monotonic_ms(void) {
+    struct timespec ts;
+    if (clock_gettime(CLOCK_MONOTONIC, &ts) != 0) return 0.0;
+    return (double)ts.tv_sec * 1000.0 + (double)ts.tv_nsec / 1000000.0;
+}
+
 static void fail(const char *what, int session, int step) {
     fprintf(stderr, "FAIL: %s session=%d step=%d\n", what, session, step);
     exit(1);
@@ -172,6 +184,22 @@ static float steering_scale_from_env(const char *name, float fallback) {
     return scale;
 }
 
+static uint32_t ssd_cache_experts_from_env(void) {
+    const char *value = getenv("DS4_TEST_SSD_CACHE_EXPERTS");
+    if (!value || !value[0]) return 30u;
+    char *end = NULL;
+    unsigned long count = strtoul(value, &end, 10);
+    if (end == value || *end != '\0' || count < 30ul ||
+        count > (unsigned long)UINT32_MAX) {
+        fprintf(stderr,
+                "FAIL: invalid DS4_TEST_SSD_CACHE_EXPERTS=%s "
+                "(expected 30..%u)\n",
+                value, UINT32_MAX);
+        exit(1);
+    }
+    return (uint32_t)count;
+}
+
 static void archive_logits(ds4_session *session, float *dst, int vocab,
                            int session_id, int step) {
     if (ds4_session_copy_logits(session, dst, vocab) != vocab) {
@@ -226,6 +254,38 @@ int main(void) {
     char *prompt_file_text = read_prompt_file(getenv("DS4_TEST_PROMPT_FILE"));
     const char *argmax_only = getenv("DS4_TEST_ARGMAX_ONLY");
     compare_argmax_only = argmax_only && strcmp(argmax_only, "0") != 0;
+    const bool ssd_streaming = env_flag("DS4_TEST_SSD_STREAMING");
+    const uint32_t ssd_cache_experts =
+        ssd_streaming ? ssd_cache_experts_from_env() : 0u;
+    const bool batch_timing = env_flag("DS4_TEST_SESSION_BATCH_TIMING");
+    const bool ssd_union_policy_switch =
+        env_flag("DS4_TEST_SSD_UNION_POLICY_SWITCH");
+    const char *batch_arm = getenv("DS4_TEST_SESSION_BATCH_ARM");
+    if (!batch_arm || !batch_arm[0]) batch_arm = "unspecified";
+    if (ssd_streaming && session_count < 5) {
+        fprintf(stderr,
+                "FAIL: DS4_TEST_SSD_STREAMING needs "
+                "DS4_TEST_SESSION_COUNT=5 to cover N=2..5\n");
+        return 1;
+    }
+    if (ssd_streaming && decode_steps < 4) {
+        fprintf(stderr,
+                "FAIL: DS4_TEST_SSD_STREAMING needs "
+                "DS4_TEST_DECODE_STEPS>=4 to cover N=2..5\n");
+        return 1;
+    }
+    if (ssd_streaming && live_controls) {
+        fprintf(stderr,
+                "FAIL: DS4_TEST_LIVE_CONTROLS would contaminate "
+                "SSD-union cache coverage counters\n");
+        return 1;
+    }
+    if (ssd_union_policy_switch && !ssd_streaming) {
+        fprintf(stderr,
+                "FAIL: DS4_TEST_SSD_UNION_POLICY_SWITCH requires "
+                "DS4_TEST_SSD_STREAMING=1\n");
+        return 1;
+    }
 
     const char *tp_mode = getenv("DS4_TEST_TP_MODE");
     const bool tp_leader = tp_mode && strcmp(tp_mode, "leader") == 0;
@@ -234,12 +294,23 @@ int main(void) {
         fprintf(stderr, "FAIL: invalid DS4_TEST_TP_MODE=%s\n", tp_mode);
         return 1;
     }
+    if (ssd_streaming && (tp_leader || tp_worker)) {
+        fprintf(stderr,
+                "FAIL: DS4_TEST_SSD_STREAMING does not support TP mode\n");
+        return 1;
+    }
     const int tp_port = tp_port_from_env();
     ds4_engine_options opt = {
         .model_path = model,
         .backend = DS4_BACKEND_METAL,
         .n_threads = 1,
         .context_size = context_size,
+        .prefill_chunk = ssd_streaming ? 128u : 0u,
+        .ssd_streaming_cache_experts = ssd_cache_experts,
+        .ssd_streaming = ssd_streaming,
+        .ssd_streaming_cold = ssd_streaming,
+        .placement_session_count_hint = ssd_streaming ? session_count : 0,
+        .share_session_prefill_workspace = ssd_streaming,
     };
     const char *steering_file =
         getenv("DS4_TEST_DIRECTIONAL_STEERING_FILE");
@@ -250,6 +321,15 @@ int main(void) {
         opt.directional_steering_ffn = steering_scale_from_env(
                 "DS4_TEST_DIRECTIONAL_STEERING_FFN", 1.0f);
     }
+    fprintf(stderr,
+            "test_metal_session_batch setup mode=%s arm=%s sessions=%d "
+            "cache_experts=%u cold=%d shared_workspace=%d\n",
+            ssd_streaming ? "ssd" : "resident",
+            batch_arm,
+            session_count,
+            opt.ssd_streaming_cache_experts,
+            opt.ssd_streaming_cold ? 1 : 0,
+            opt.share_session_prefill_workspace ? 1 : 0);
     if (tp_leader) {
         opt.tp.role = DS4_TP_LEADER;
         opt.tp.listen_host = getenv("DS4_TEST_TP_LISTEN_HOST");
@@ -368,10 +448,16 @@ int main(void) {
     float *actual = malloc((size_t)vocab * sizeof(float));
     int *argmax = malloc(frontier_count * sizeof(int));
     int generated[MAX_SESSION_COUNT][MAX_DECODE_STEPS];
+    int final_pos[MAX_SESSION_COUNT] = {0};
     ds4_session *live_control[MAX_SESSION_COUNT] = {0};
     double control_seconds = 0.0;
     double live_control_seconds = 0.0;
     if (!expected || !actual || !argmax) fail("oracle allocation", -1, -1);
+    for (int i = 0; i < MAX_SESSION_COUNT; i++) {
+        for (int step = 0; step < MAX_DECODE_STEPS; step++) {
+            generated[i][step] = -1;
+        }
+    }
 
 #define FRONTIER(step_, session_) \
     ((size_t)(step_) * (size_t)session_count + (size_t)(session_))
@@ -400,37 +486,62 @@ int main(void) {
         }
     }
 
-    const double batch_t0 = now_seconds();
+    const double batch_wall_t0 = now_seconds();
+    double batch_ms = 0.0;
+    uint64_t batch_rows_total = 0;
+    uint32_t ssd_rows_coverage = 0;
+    ds4_gpu_exact_rows_persistent_report exact_before = {0};
+    ds4_gpu_exact_rows_persistent_report exact_after = {0};
+    if (ssd_streaming) {
+        ds4_gpu_test_exact_rows_persistent_report(&exact_before);
+    }
     for (int step = 0; step < decode_steps; step++) {
+        if (ssd_union_policy_switch && step == decode_steps / 2) {
+            if (unsetenv("DS4_METAL_REQUIRE_Q4_SSD_SESSION_UNION") != 0 ||
+                setenv("DS4_METAL_ENABLE_Q4_SSD_SESSION_UNION", "1", 1) != 0) {
+                fail("SSD union policy switch", -1, step);
+            }
+            fprintf(stderr,
+                    "test_metal_session_batch policy-switch step=%d "
+                    "REQUIRE=unset ENABLE=1\n",
+                    step);
+        }
+        const int batch_rows = ssd_streaming ? 2 + step % 4 : session_count;
         ds4_decode_item items[MAX_SESSION_COUNT];
-        for (int row = 0; row < session_count; row++) {
-            int i = (step & 1) ? session_count - 1 - row : row;
+        for (int row = 0; row < batch_rows; row++) {
+            int i = (step & 1) ? batch_rows - 1 - row : row;
             int token = ds4_session_argmax(batched[i]);
             generated[i][step] = token;
             items[row].session = batched[i];
             items[row].token = token;
         }
-        if (ds4_sessions_eval_batch(items, session_count,
+        const double batch_t0 = batch_timing ? monotonic_ms() : 0.0;
+        if (ds4_sessions_eval_batch(items, batch_rows,
                                     err, sizeof(err)) != 0) {
             fprintf(stderr, "FAIL: batch step=%d: %s\n", step, err);
             return 1;
         }
+        if (batch_timing) batch_ms += monotonic_ms() - batch_t0;
+        batch_rows_total += (uint64_t)batch_rows;
+        if (ssd_streaming) ssd_rows_coverage |= 1u << (uint32_t)batch_rows;
         for (int i = 0; i < session_count; i++) {
             size_t f = FRONTIER(step + 1, i);
             archive_logits(batched[i], expected + f * (size_t)vocab,
                            vocab, i, step + 1);
             argmax[f] = ds4_session_argmax(batched[i]);
             if (live_controls) {
-                const double eval_t0 = now_seconds();
-                const int eval_rc = ds4_session_eval(
-                        live_control[i], generated[i][step],
-                        err, sizeof(err));
-                live_control_seconds += now_seconds() - eval_t0;
-                if (eval_rc != 0) {
-                    fprintf(stderr,
-                            "FAIL: live control eval session=%d step=%d: %s\n",
-                            i, step, err);
-                    return 1;
+                if (generated[i][step] >= 0) {
+                    const double eval_t0 = now_seconds();
+                    const int eval_rc = ds4_session_eval(
+                            live_control[i], generated[i][step],
+                            err, sizeof(err));
+                    live_control_seconds += now_seconds() - eval_t0;
+                    if (eval_rc != 0) {
+                        fprintf(stderr,
+                                "FAIL: live control eval session=%d step=%d: %s\n",
+                                i, step, err);
+                        return 1;
+                    }
                 }
                 compare_logits(live_control[i],
                                expected + f * (size_t)vocab,
@@ -440,13 +551,61 @@ int main(void) {
         }
     }
     const double batch_seconds =
-        now_seconds() - batch_t0 - live_control_seconds;
+        now_seconds() - batch_wall_t0 - live_control_seconds;
+    if (ssd_streaming &&
+        (ssd_rows_coverage & ((1u << 2) | (1u << 3) |
+                              (1u << 4) | (1u << 5))) !=
+            ((1u << 2) | (1u << 3) | (1u << 4) | (1u << 5))) {
+        fail("SSD batch coverage N=2..5", -1, -1);
+    }
+    if (ssd_streaming) {
+        ds4_gpu_test_exact_rows_persistent_report(&exact_after);
+        const uint64_t persistent =
+            exact_after.persistent_calls - exact_before.persistent_calls;
+        const uint64_t transient =
+            exact_after.transient_calls - exact_before.transient_calls;
+        const uint64_t fallbacks =
+            exact_after.persistent_fallbacks -
+            exact_before.persistent_fallbacks;
+        const uint64_t failures =
+            exact_after.persistent_failures - exact_before.persistent_failures;
+        const uint64_t mapped_views =
+            exact_after.mapped_view_calls - exact_before.mapped_view_calls;
+        const int layer_count = ds4_engine_layer_count(engine);
+        const uint64_t expected_persistent = layer_count > 0
+            ? (uint64_t)decode_steps * (uint64_t)(uint32_t)layer_count
+            : 0u;
+        fprintf(stderr,
+                "test_metal_session_batch exact-cache persistent=%llu "
+                "expected_persistent=%llu "
+                "transient=%llu fallbacks=%llu failures=%llu "
+                "mapped_views=%llu max_unique=%u\n",
+                (unsigned long long)persistent,
+                (unsigned long long)expected_persistent,
+                (unsigned long long)transient,
+                (unsigned long long)fallbacks,
+                (unsigned long long)failures,
+                (unsigned long long)mapped_views,
+                exact_after.max_unique);
+        if (env_flag("DS4_METAL_REQUIRE_EXACT_ROWS_PERSISTENT_CACHE") &&
+            (persistent != expected_persistent || transient != 0u ||
+             fallbacks != 0u || failures != 0u || mapped_views != 0u)) {
+            fail("SSD exact persistent cache coverage", -1, -1);
+        }
+        if (env_flag("DS4_METAL_DISABLE_Q4_SSD_SESSION_UNION") &&
+            env_flag("DS4_METAL_REQUIRE_Q4_SSD_SESSION_UNION") &&
+            (persistent != 0u || transient != 0u || fallbacks != 0u ||
+             failures != 0u || mapped_views != 0u)) {
+            fail("SSD control did not serialize", -1, -1);
+        }
+    }
     for (int i = 0; i < session_count; i++) {
+        final_pos[i] = ds4_session_pos(batched[i]);
         ds4_session_free(batched[i]);
         if (live_control[i]) ds4_session_free(live_control[i]);
     }
 
-    if (!skip_mixed) {
+    if (!skip_mixed && !ssd_streaming) {
         ds4_tokens mixed_prompt = {0};
         ds4_tokens suffix = {0};
         ds4_tokens_copy(&mixed_prompt, &prompt[0]);
@@ -474,7 +633,7 @@ int main(void) {
         return 1;
     }
     ds4_decode_item mixed_items[MAX_SESSION_COUNT];
-    for (int i = 0; !live_controls && i < session_count; i++) {
+    for (int i = 0; i < session_count; i++) {
         if (ds4_session_create(&mixed_decode[i], engine, context_size) != 0) {
             fail("mixed decode create", i, -1);
         }
@@ -561,7 +720,7 @@ int main(void) {
             compare_logits(control, expected + f * (size_t)vocab,
                            actual, vocab, argmax[f], logit_tolerance,
                            i, step);
-            if (step < decode_steps) {
+            if (step < decode_steps && generated[i][step] >= 0) {
                 const double eval_t0 = now_seconds();
                 const int eval_rc = ds4_session_eval(
                         control, generated[i][step], err, sizeof(err));
@@ -574,6 +733,9 @@ int main(void) {
                 }
             }
         }
+        if (ds4_session_pos(control) != final_pos[i]) {
+            fail("final checkpoint", i, decode_steps);
+        }
         ds4_session_free(control);
         ds4_tokens_free(&prompt[i]);
     }
@@ -585,17 +747,35 @@ int main(void) {
     if (tp) (void)ds4_tp_send_stop(tp);
     ds4_engine_close(engine);
     ds4_tp_free(tp);
+    if (batch_timing) {
+        const double rows_per_sec = batch_ms > 0.0
+            ? (double)batch_rows_total * 1000.0 / batch_ms : 0.0;
+        fprintf(stderr,
+                "test_metal_session_batch timing arm=%s batch_ms=%.3f "
+                "rows=%llu steps=%d rows_per_sec=%.3f\n",
+                batch_arm,
+                batch_ms,
+                (unsigned long long)batch_rows_total,
+                decode_steps,
+                rows_per_sec);
+    }
     fprintf(stderr,
             "test_metal_session_batch PASS sessions=%d steps=%d mixed_suffix=%d "
             "comparison=%s logit_tolerance=%g max_abs=%g batch=%.2f rows/s "
-            "serial=%.2f rows/s speedup=%.2fx\n",
+            "serial=%.2f rows/s speedup=%.2fx mode=%s arm=%s "
+            "ssd_rows=%s batch_rows=%llu batch_ms=%s\n",
             session_count, decode_steps,
-            skip_mixed ? 0 : MIXED_SUFFIX_TOKENS,
+            (skip_mixed || ssd_streaming) ? 0 : MIXED_SUFFIX_TOKENS,
             compare_argmax_only ? "argmax" : "logits",
             logit_tolerance, observed_max_abs,
-            (double)(session_count * decode_steps) / batch_seconds,
-            (double)(session_count * decode_steps) / control_seconds,
-            control_seconds / batch_seconds);
+            batch_seconds > 0.0 ? (double)batch_rows_total / batch_seconds : 0.0,
+            control_seconds > 0.0 ? (double)batch_rows_total / control_seconds : 0.0,
+            batch_seconds > 0.0 ? control_seconds / batch_seconds : 0.0,
+            ssd_streaming ? "ssd" : "resident",
+            batch_arm,
+            ssd_streaming ? "2,3,4,5" : "fixed",
+            (unsigned long long)batch_rows_total,
+            batch_timing ? "reported-above" : "disabled");
     return 0;
 #undef FRONTIER
 }
diff --git a/tests/test_mxfp4_cuda.cu b/tests/test_mxfp4_cuda.cu
index 9a1c3e8d70..048faf4e9c 100644
--- a/tests/test_mxfp4_cuda.cu
+++ b/tests/test_mxfp4_cuda.cu
@@ -10,14 +10,6 @@
 #include <random>
 #include <vector>
 
-extern "C" int ds4_cuda_q8_fold_take_q81(
-        const void *src, uint64_t in_dim, const void **q81) {
-    (void)src;
-    (void)in_dim;
-    if (q81) *q81 = nullptr;
-    return 0;
-}
-
 namespace {
 
 constexpr int QK = 32;
diff --git a/tests/test_mxfp4_metal.c b/tests/test_mxfp4_metal.c
index bb1f37fed2..80cb777f41 100644
--- a/tests/test_mxfp4_metal.c
+++ b/tests/test_mxfp4_metal.c
@@ -2,12 +2,15 @@
 
 #include "ds4_gpu.h"
 
+#include <ctype.h>
+#include <float.h>
 #include <math.h>
 #include <stdbool.h>
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <strings.h>
 #include <unistd.h>
 
 #define MXFP4_TYPE 39u
@@ -43,6 +46,50 @@ static float e8m0_to_f32(uint8_t e) {
     return value;
 }
 
+static uint32_t f32_bits(float value) {
+    uint32_t bits;
+    memcpy(&bits, &value, sizeof(bits));
+    return bits;
+}
+
+static int env_bool_enabled(const char *value) {
+    if (!value) return 0;
+    while (isspace((unsigned char)*value)) value++;
+    size_t len = strlen(value);
+    while (len > 0 && isspace((unsigned char)value[len - 1])) len--;
+    if (len == 0) return 1;
+    if ((len == 1 && value[0] == '0') ||
+        (len == 5 && strncasecmp(value, "false", len) == 0) ||
+        (len == 2 && strncasecmp(value, "no", len) == 0) ||
+        (len == 3 && strncasecmp(value, "off", len) == 0)) {
+        return 0;
+    }
+    return 1;
+}
+
+static float sigmoid_stable(float x) {
+    if (x >= 0.0f) {
+        const float e = expf(-x);
+        return 1.0f / (1.0f + e);
+    }
+    const float e = expf(x);
+    return e / (1.0f + e);
+}
+
+static float silu_stable(float x) {
+    return x * sigmoid_stable(x);
+}
+
+static float silu_reference_f64(float x) {
+    const double xd = (double)x;
+    if (xd >= 0.0) {
+        const double e = exp(-xd);
+        return (float)(xd / (1.0 + e));
+    }
+    const double e = exp(xd);
+    return (float)(xd * (e / (1.0 + e)));
+}
+
 static float dot_mxfp4(const block_mxfp4 *row, const float *x) {
     float sum = 0.0f;
     for (uint32_t block = 0; block < DIM / QK_MXFP4; block++) {
@@ -107,6 +154,116 @@ static int compare_values(const char *name, const float *actual,
     return max_abs <= tolerance;
 }
 
+static int test_silu_stable_metal(void) {
+    const float gate[] = {
+        -100.0f,
+        -92.0f,
+        -89.0f,
+        nextafterf(-87.0f, -FLT_MAX),
+        -87.0f,
+        nextafterf(-87.0f, FLT_MAX),
+        -80.0f,
+        -10.0f,
+        -1.0f,
+        -0.0f,
+        0.0f,
+        1.0f,
+        10.0f,
+    };
+    float up[sizeof(gate) / sizeof(gate[0])];
+    float actual[sizeof(gate) / sizeof(gate[0])];
+    float expected[sizeof(gate) / sizeof(gate[0])];
+    const uint32_t count = (uint32_t)(sizeof(gate) / sizeof(gate[0]));
+    const int require_ieee = env_bool_enabled(getenv("DS4_METAL_MATH_SAFE"));
+
+    for (uint32_t i = 0; i < count; i++) {
+        up[i] = 1.0f;
+        expected[i] = silu_reference_f64(gate[i]);
+    }
+
+    ds4_gpu_tensor *gate_tensor = ds4_gpu_tensor_alloc(sizeof(gate));
+    ds4_gpu_tensor *up_tensor = ds4_gpu_tensor_alloc(sizeof(up));
+    ds4_gpu_tensor *out_tensor = ds4_gpu_tensor_alloc(sizeof(actual));
+    int ok = gate_tensor && up_tensor && out_tensor;
+    if (ok) ok = ds4_gpu_tensor_write(gate_tensor, 0, gate, sizeof(gate));
+    if (ok) ok = ds4_gpu_tensor_write(up_tensor, 0, up, sizeof(up));
+    if (ok) {
+        ok = ds4_gpu_swiglu_tensor(out_tensor, gate_tensor, up_tensor,
+                                   count, 0.0f, 1.0f);
+    }
+    if (ok) ok = ds4_gpu_tensor_read(out_tensor, 0, actual, sizeof(actual));
+
+    for (uint32_t i = 0; ok && i < count; i++) {
+        const uint32_t actual_bits = f32_bits(actual[i]);
+        const uint32_t expected_bits = f32_bits(expected[i]);
+        const uint32_t actual_abs_bits = actual_bits & 0x7fffffffu;
+        const float expected_abs = fabsf(expected[i]);
+        if ((actual_bits & 0x7f800000u) == 0x7f800000u) {
+            fprintf(stderr,
+                    "MXFP4 Metal stable SiLU mismatch i=%u x=%g actual=%g expected=%g\n",
+                    i, gate[i], actual[i], expected[i]);
+            ok = 0;
+            break;
+        }
+        if (expected_abs > 0.0f && expected_abs < FLT_MIN) {
+            // The final SiLU value itself is subnormal and may be flushed by
+            // Metal even in math-safe mode. Safe mode must retain its sign.
+            if (actual_abs_bits == 0u) {
+                const int sign_mismatch =
+                    require_ieee && ((actual_bits ^ expected_bits) >> 31u) != 0u;
+                if (sign_mismatch) {
+                    fprintf(stderr,
+                            "MXFP4 Metal stable SiLU subnormal-zero sign mismatch "
+                            "i=%u x=%g actual_bits=0x%08x expected_bits=0x%08x\n",
+                            i, gate[i], actual_bits, expected_bits);
+                    ok = 0;
+                }
+                continue;
+            }
+
+            const double rel = fabs(((double)actual[i] - (double)expected[i]) /
+                                    (double)expected[i]);
+            if ((actual_bits >> 31u) == 0u || rel > 2.0e-2) {
+                fprintf(stderr,
+                        "MXFP4 Metal stable SiLU subnormal mismatch "
+                        "i=%u x=%g actual=%g expected=%g rel=%g strict=%d\n",
+                        i, gate[i], actual[i], expected[i], rel, require_ieee);
+                ok = 0;
+            }
+            continue;
+        }
+        if (expected[i] == 0.0f) {
+            const int sign_mismatch =
+                require_ieee && ((actual_bits ^ expected_bits) >> 31u) != 0u;
+            if (actual_abs_bits != 0u || sign_mismatch) {
+                fprintf(stderr,
+                        "MXFP4 Metal stable SiLU zero mismatch "
+                        "i=%u x=%g actual_bits=0x%08x expected_bits=0x%08x strict=%d\n",
+                        i, gate[i], actual_bits, expected_bits, require_ieee);
+                ok = 0;
+            }
+            continue;
+        }
+
+        const double rel = fabs(((double)actual[i] - (double)expected[i]) /
+                                (double)expected[i]);
+        const double rel_limit = expected_abs < 1.0e-20f ? 5.0e-4 : 5.0e-5;
+        if (rel > rel_limit) {
+            fprintf(stderr,
+                    "MXFP4 Metal stable SiLU relative mismatch "
+                    "i=%u x=%g actual=%g expected=%g rel=%g limit=%g\n",
+                    i, gate[i], actual[i], expected[i], rel, rel_limit);
+            ok = 0;
+        }
+    }
+
+    ds4_gpu_tensor_free(out_tensor);
+    ds4_gpu_tensor_free(up_tensor);
+    ds4_gpu_tensor_free(gate_tensor);
+    if (ok) fprintf(stderr, "MXFP4 Metal numerically stable SiLU tail PASS\n");
+    return ok;
+}
+
 int main(void) {
     const uint64_t page = (uint64_t)getpagesize();
     const uint64_t row_bytes =
@@ -178,7 +335,7 @@ int main(void) {
                 x);
             const float g = fminf(gate_ref[pair], 7.0f);
             const float u = fmaxf(-7.0f, fminf(up_ref[pair], 7.0f));
-            mid_ref[pair] = (g / (1.0f + expf(-g))) * u * weights[slot];
+            mid_ref[pair] = silu_stable(g) * u * weights[slot];
         }
     }
     for (uint32_t row = 0; row < DIM; row++) {
@@ -191,7 +348,9 @@ int main(void) {
         }
     }
 
-    int ok = ds4_gpu_init() && ds4_gpu_set_model_map(model, model_size);
+    int ok = ds4_gpu_init();
+    if (ok) ok = test_silu_stable_metal();
+    if (ok) ok = ds4_gpu_set_model_map(model, model_size);
     ok = ok && ds4_gpu_test_decode_pipeline_fast_lookup();
     if (ok) {
         fprintf(stderr,
diff --git a/tests/test_q4k_dot.c b/tests/test_q4k_dot.c
index 4903ca7fd6..8e363d6f3f 100644
--- a/tests/test_q4k_dot.c
+++ b/tests/test_q4k_dot.c
@@ -94,6 +94,52 @@ static void vec_dot_q4_K_q8_K(int n, float *s, const block_q4_K *x, const block_
     *s = sumf;
 }
 
+/* Scalar oracle for the production two-token traversal. */
+static void vec_dot_q4_K_q8_K_2(
+        int n, float *s0, float *s1, const block_q4_K *x,
+        const block_q8_K *y0, const block_q8_K *y1) {
+    const int nb = n / QK_K;
+    float sumf0 = 0.0f;
+    float sumf1 = 0.0f;
+    for (int i = 0; i < nb; i++) {
+        const float xd = f16_to_f32(x[i].d);
+        const float xmin = f16_to_f32(x[i].dmin);
+        const float d0 = y0[i].d * xd;
+        const float d1 = y1[i].d * xd;
+        const float dm0 = -y0[i].d * xmin;
+        const float dm1 = -y1[i].d * xmin;
+        int summs0 = 0;
+        int summs1 = 0;
+        for (int j = 0; j < QK_K / 32; j++) {
+            uint8_t sc_val, m_val;
+            q4_k_get_scale_min(j, x[i].scales, &sc_val, &m_val);
+            const int32_t gsum0 = (int32_t)y0[i].bsums[j * 2] +
+                                  (int32_t)y0[i].bsums[j * 2 + 1];
+            const int32_t gsum1 = (int32_t)y1[i].bsums[j * 2] +
+                                  (int32_t)y1[i].bsums[j * 2 + 1];
+            summs0 += m_val * gsum0;
+            summs1 += m_val * gsum1;
+        }
+        int isum0 = 0;
+        int isum1 = 0;
+        for (int j = 0; j < QK_K / 32; j++) {
+            uint8_t sc_val, m_val;
+            q4_k_get_scale_min(j, x[i].scales, &sc_val, &m_val);
+            const int byte_off = (j >> 1) * 32;
+            const int shift = (j & 1) * 4;
+            for (int l = 0; l < 32; l++) {
+                const int q4 = (x[i].qs[byte_off + l] >> shift) & 0xF;
+                isum0 += q4 * (int)y0[i].qs[j * 32 + l] * sc_val;
+                isum1 += q4 * (int)y1[i].qs[j * 32 + l] * sc_val;
+            }
+        }
+        sumf0 += d0 * (float)isum0 + dm0 * (float)summs0;
+        sumf1 += d1 * (float)isum1 + dm1 * (float)summs1;
+    }
+    *s0 = sumf0;
+    *s1 = sumf1;
+}
+
 /* Reference: fully dequantize Q4_K to float, then dot with Q8_K's dequantized values. */
 static float ref_dot(const block_q4_K *bx, const block_q8_K *by) {
     float x[QK_K];
@@ -204,6 +250,40 @@ static int test_dot_reference(void) {
     return ok ? 0 : 1;
 }
 
+static int test_dot_pair_bitwise(void) {
+    static const int widths[] = {256, 1024, 4096, 7168, 16384};
+    block_q4_K bx[16384 / QK_K];
+    block_q8_K by0[16384 / QK_K];
+    block_q8_K by1[16384 / QK_K];
+    int ok = 1;
+
+    for (size_t wi = 0; wi < sizeof(widths) / sizeof(widths[0]); wi++) {
+        const int nb = widths[wi] / QK_K;
+        for (uint32_t seed = 1; seed <= 16; seed++) {
+            for (int b = 0; b < nb; b++) {
+                fill_q4_K(&bx[b], seed * 101u + (uint32_t)b);
+                fill_q8_K(&by0[b], seed * 211u + (uint32_t)b);
+                fill_q8_K(&by1[b], seed * 307u + (uint32_t)b);
+            }
+            float ref0 = 0.0f, ref1 = 0.0f;
+            float got0 = 0.0f, got1 = 0.0f;
+            vec_dot_q4_K_q8_K(widths[wi], &ref0, bx, by0);
+            vec_dot_q4_K_q8_K(widths[wi], &ref1, bx, by1);
+            vec_dot_q4_K_q8_K_2(widths[wi], &got0, &got1, bx, by0, by1);
+            if (memcmp(&got0, &ref0, sizeof(got0)) != 0 ||
+                memcmp(&got1, &ref1, sizeof(got1)) != 0) {
+                printf("    width=%d seed=%u: pair=(%g,%g) single=(%g,%g)\n",
+                       widths[wi], seed, got0, got1, ref0, ref1);
+                ok = 0;
+            }
+        }
+    }
+
+    printf("  two-token dot vs two singles (bitwise): %s\n",
+           ok ? "PASS" : "FAIL");
+    return ok ? 0 : 1;
+}
+
 /* Test with a hand-crafted known block. */
 static int test_dot_known(void) {
     block_q4_K bx;
@@ -250,7 +330,8 @@ int main(void) {
     failures += test_scale_extraction();
     failures += test_dot_known();
     failures += test_dot_reference();
+    failures += test_dot_pair_bitwise();
 
-    printf("\n%d/%d tests passed\n", 4 - failures, 4);
+    printf("\n%d/%d tests passed\n", 5 - failures, 5);
     return failures ? 1 : 0;
 }
diff --git a/tests/test_quantizer_indexer_q4.c b/tests/test_quantizer_indexer_q4.c
new file mode 100644
index 0000000000..f73593a6aa
--- /dev/null
+++ b/tests/test_quantizer_indexer_q4.c
@@ -0,0 +1,488 @@
+#define _DARWIN_C_SOURCE
+#define _POSIX_C_SOURCE 200809L
+
+/*
+ * End-to-end regression for direct F16 -> Q4_K requantization of
+ * blk.*.indexer.attn_q_b.weight.
+ *
+ * The fixture is deliberately GGUF-library-free.  It writes three tiny
+ * tensors, invokes the production CLI, then parses the output and compares
+ * the Q4_K payload with the public quantization facade.  Seventeen rows cross
+ * the writer's 16-row conversion batch boundary.
+ */
+
+#include "quants.h"
+
+#include <errno.h>
+#include <inttypes.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <unistd.h>
+
+enum {
+    GGUF_VERSION = 3,
+    GGUF_ALIGNMENT = 32,
+    FIXTURE_TENSORS = 3,
+    INDEXER_COLS = 256,
+    INDEXER_ROWS = 17,
+};
+
+static const char *const k_before_name = "test.sentinel_before";
+static const char *const k_indexer_name =
+    "blk.2.indexer.attn_q_b.weight";
+static const char *const k_after_name = "test.sentinel_after";
+
+static uint8_t k_before_payload[28];
+static uint8_t k_after_payload[26];
+
+typedef struct {
+    char *name;
+    uint32_t n_dims;
+    uint64_t dims[DS4Q_MAX_DIMS];
+    uint32_t type;
+    uint64_t offset;
+    size_t size;
+} tensor_info;
+
+typedef struct {
+    FILE *fp;
+    uint64_t data_offset;
+    tensor_info tensors[FIXTURE_TENSORS];
+} parsed_gguf;
+
+static int g_failures;
+
+static void fail(const char *what) {
+    fprintf(stderr, "test_quantizer_indexer_q4 FAIL: %s\n", what);
+    g_failures++;
+}
+
+static uint64_t align_up(uint64_t value, uint64_t alignment) {
+    return (value + alignment - 1u) / alignment * alignment;
+}
+
+static bool write_bytes(FILE *fp, const void *data, size_t size) {
+    return size == 0 || fwrite(data, 1, size, fp) == size;
+}
+
+static bool write_u16_le(FILE *fp, uint16_t value) {
+    const uint8_t bytes[2] = {
+        (uint8_t)value,
+        (uint8_t)(value >> 8u),
+    };
+    return write_bytes(fp, bytes, sizeof(bytes));
+}
+
+static bool write_u32_le(FILE *fp, uint32_t value) {
+    uint8_t bytes[4];
+    for (uint32_t i = 0; i < 4u; i++) {
+        bytes[i] = (uint8_t)(value >> (8u * i));
+    }
+    return write_bytes(fp, bytes, sizeof(bytes));
+}
+
+static bool write_u64_le(FILE *fp, uint64_t value) {
+    uint8_t bytes[8];
+    for (uint32_t i = 0; i < 8u; i++) {
+        bytes[i] = (uint8_t)(value >> (8u * i));
+    }
+    return write_bytes(fp, bytes, sizeof(bytes));
+}
+
+static bool write_string(FILE *fp, const char *value) {
+    const size_t len = strlen(value);
+    return write_u64_le(fp, len) && write_bytes(fp, value, len);
+}
+
+static bool write_zeros(FILE *fp, uint64_t count) {
+    static const uint8_t zeros[256] = {0};
+    while (count != 0) {
+        const size_t chunk = count < sizeof(zeros) ? (size_t)count : sizeof(zeros);
+        if (!write_bytes(fp, zeros, chunk)) return false;
+        count -= chunk;
+    }
+    return true;
+}
+
+static uint16_t *make_indexer_f16(uint32_t cols, uint32_t rows) {
+    const size_t count = (size_t)cols * rows;
+    float *values = malloc(count * sizeof(*values));
+    uint16_t *half = malloc(count * sizeof(*half));
+    if (!values || !half) {
+        free(half);
+        free(values);
+        return NULL;
+    }
+    for (uint32_t row = 0; row < rows; row++) {
+        for (uint32_t col = 0; col < cols; col++) {
+            const int32_t raw =
+                (int32_t)((row * 67u + col * 29u + (row ^ col) * 3u) % 257u) -
+                128;
+            values[(size_t)row * cols + col] = (float)raw / 32.0f;
+        }
+    }
+    ds4q_f32_to_f16_row(values, half, (int64_t)count);
+    free(values);
+    return half;
+}
+
+static bool write_tensor_info(FILE *fp, const char *name, uint32_t n_dims,
+                              uint64_t d0, uint64_t d1, uint32_t type,
+                              uint64_t offset) {
+    return write_string(fp, name) &&
+           write_u32_le(fp, n_dims) &&
+           write_u64_le(fp, d0) &&
+           (n_dims == 1u || write_u64_le(fp, d1)) &&
+           write_u32_le(fp, type) &&
+           write_u64_le(fp, offset);
+}
+
+static bool write_fixture(const char *path, uint32_t cols, uint32_t rows,
+                          const uint16_t *indexer_f16) {
+    const uint64_t before_offset = 0;
+    const uint64_t indexer_offset =
+        align_up(sizeof(k_before_payload), GGUF_ALIGNMENT);
+    const uint64_t indexer_bytes = (uint64_t)cols * rows * sizeof(uint16_t);
+    const uint64_t after_offset =
+        align_up(indexer_offset + indexer_bytes, GGUF_ALIGNMENT);
+
+    FILE *fp = fopen(path, "wb");
+    if (!fp) return false;
+    bool ok = write_bytes(fp, "GGUF", 4) &&
+              write_u32_le(fp, GGUF_VERSION) &&
+              write_u64_le(fp, FIXTURE_TENSORS) &&
+              write_u64_le(fp, 0) &&
+              write_tensor_info(fp, k_before_name, 1,
+                                sizeof(k_before_payload) / sizeof(uint32_t),
+                                0, DS4Q_TYPE_F32, before_offset) &&
+              write_tensor_info(fp, k_indexer_name, 2, cols, rows,
+                                DS4Q_TYPE_F16, indexer_offset) &&
+              write_tensor_info(fp, k_after_name, 1,
+                                sizeof(k_after_payload) / sizeof(uint16_t),
+                                0, DS4Q_TYPE_F16, after_offset);
+
+    const off_t metadata_end = ftello(fp);
+    if (metadata_end < 0) ok = false;
+    const uint64_t data_offset = metadata_end < 0 ? 0 :
+        align_up((uint64_t)metadata_end, GGUF_ALIGNMENT);
+    if (ok) ok = write_zeros(fp, data_offset - (uint64_t)metadata_end);
+    if (ok) ok = write_bytes(fp, k_before_payload, sizeof(k_before_payload));
+    if (ok) ok = write_zeros(fp, indexer_offset - sizeof(k_before_payload));
+    for (uint64_t i = 0; ok && i < (uint64_t)cols * rows; i++) {
+        ok = write_u16_le(fp, indexer_f16[i]);
+    }
+    if (ok) {
+        ok = write_zeros(fp, after_offset - indexer_offset - indexer_bytes) &&
+             write_bytes(fp, k_after_payload, sizeof(k_after_payload));
+    }
+    if (fclose(fp) != 0) ok = false;
+    return ok;
+}
+
+static bool read_exact(FILE *fp, void *data, size_t size) {
+    return size == 0 || fread(data, 1, size, fp) == size;
+}
+
+static bool read_u32_le(FILE *fp, uint32_t *value) {
+    uint8_t bytes[4];
+    if (!read_exact(fp, bytes, sizeof(bytes))) return false;
+    *value = 0;
+    for (uint32_t i = 0; i < 4u; i++) {
+        *value |= (uint32_t)bytes[i] << (8u * i);
+    }
+    return true;
+}
+
+static bool read_u64_le(FILE *fp, uint64_t *value) {
+    uint8_t bytes[8];
+    if (!read_exact(fp, bytes, sizeof(bytes))) return false;
+    *value = 0;
+    for (uint32_t i = 0; i < 8u; i++) {
+        *value |= (uint64_t)bytes[i] << (8u * i);
+    }
+    return true;
+}
+
+static char *read_string(FILE *fp) {
+    uint64_t len = 0;
+    if (!read_u64_le(fp, &len) || len > 4096u || len > SIZE_MAX - 1u) return NULL;
+    char *value = malloc((size_t)len + 1u);
+    if (!value) return NULL;
+    if (!read_exact(fp, value, (size_t)len)) {
+        free(value);
+        return NULL;
+    }
+    value[len] = '\0';
+    return value;
+}
+
+static size_t tensor_size(const tensor_info *tensor) {
+    if (tensor->n_dims == 0 || tensor->n_dims > DS4Q_MAX_DIMS ||
+        tensor->dims[0] > INT64_MAX) {
+        return 0;
+    }
+    const size_t row = ds4q_row_size((ds4q_type)tensor->type,
+                                     (int64_t)tensor->dims[0]);
+    if (row == 0) return 0;
+    size_t size = row;
+    for (uint32_t d = 1; d < tensor->n_dims; d++) {
+        if (tensor->dims[d] > SIZE_MAX ||
+            (tensor->dims[d] != 0 && size > SIZE_MAX / tensor->dims[d])) {
+            return 0;
+        }
+        size *= (size_t)tensor->dims[d];
+    }
+    return size;
+}
+
+static void close_parsed(parsed_gguf *gguf) {
+    if (gguf->fp) fclose(gguf->fp);
+    gguf->fp = NULL;
+    for (uint32_t i = 0; i < FIXTURE_TENSORS; i++) {
+        free(gguf->tensors[i].name);
+        gguf->tensors[i].name = NULL;
+    }
+}
+
+static bool parse_output(const char *path, parsed_gguf *gguf) {
+    memset(gguf, 0, sizeof(*gguf));
+    gguf->fp = fopen(path, "rb");
+    if (!gguf->fp) return false;
+
+    char magic[4];
+    uint32_t version = 0;
+    uint64_t n_tensors = 0;
+    uint64_t n_kv = 0;
+    bool ok = read_exact(gguf->fp, magic, sizeof(magic)) &&
+              memcmp(magic, "GGUF", sizeof(magic)) == 0 &&
+              read_u32_le(gguf->fp, &version) &&
+              read_u64_le(gguf->fp, &n_tensors) &&
+              read_u64_le(gguf->fp, &n_kv) &&
+              version == GGUF_VERSION &&
+              n_tensors == FIXTURE_TENSORS && n_kv == 0;
+
+    for (uint32_t i = 0; ok && i < FIXTURE_TENSORS; i++) {
+        tensor_info *tensor = &gguf->tensors[i];
+        tensor->name = read_string(gguf->fp);
+        ok = tensor->name && read_u32_le(gguf->fp, &tensor->n_dims) &&
+             tensor->n_dims >= 1u && tensor->n_dims <= DS4Q_MAX_DIMS;
+        for (uint32_t d = 0; ok && d < tensor->n_dims; d++) {
+            ok = read_u64_le(gguf->fp, &tensor->dims[d]);
+        }
+        ok = ok && read_u32_le(gguf->fp, &tensor->type) &&
+             read_u64_le(gguf->fp, &tensor->offset);
+        if (ok) {
+            tensor->size = tensor_size(tensor);
+            ok = tensor->size != 0;
+        }
+    }
+    const off_t metadata_end = ok ? ftello(gguf->fp) : -1;
+    if (metadata_end < 0) ok = false;
+    if (ok) {
+        gguf->data_offset = align_up((uint64_t)metadata_end, GGUF_ALIGNMENT);
+    } else {
+        close_parsed(gguf);
+    }
+    return ok;
+}
+
+static bool read_payload(const parsed_gguf *gguf, uint32_t index,
+                         void *data, size_t size) {
+    if (!gguf->fp || index >= FIXTURE_TENSORS ||
+        size != gguf->tensors[index].size ||
+        gguf->data_offset > UINT64_MAX - gguf->tensors[index].offset) {
+        return false;
+    }
+    const uint64_t absolute =
+        gguf->data_offset + gguf->tensors[index].offset;
+    return absolute <= (uint64_t)INT64_MAX &&
+           fseeko(gguf->fp, (off_t)absolute, SEEK_SET) == 0 &&
+           read_exact(gguf->fp, data, size);
+}
+
+static int run_quantizer(const char *tool, const char *source,
+                         const char *output) {
+    const pid_t pid = fork();
+    if (pid < 0) return -1;
+    if (pid == 0) {
+        execl(tool, tool,
+              "--source-gguf", source,
+              "--out", output,
+              "--indexer-q", "q4_k",
+              (char *)NULL);
+        _exit(127);
+    }
+    int status = 0;
+    while (waitpid(pid, &status, 0) < 0) {
+        if (errno != EINTR) return -1;
+    }
+    return WIFEXITED(status) ? WEXITSTATUS(status) : -1;
+}
+
+static void verify_positive(const char *output, const uint16_t *half) {
+    parsed_gguf gguf;
+    if (!parse_output(output, &gguf)) {
+        fail("parse positive output");
+        return;
+    }
+
+    const tensor_info *before = &gguf.tensors[0];
+    const tensor_info *indexer = &gguf.tensors[1];
+    const tensor_info *after = &gguf.tensors[2];
+    if (strcmp(before->name, k_before_name) != 0 ||
+        before->type != DS4Q_TYPE_F32 ||
+        before->size != sizeof(k_before_payload)) {
+        fail("before sentinel metadata");
+    }
+    if (strcmp(indexer->name, k_indexer_name) != 0 ||
+        indexer->type != DS4Q_TYPE_Q4_K || indexer->n_dims != 2u ||
+        indexer->dims[0] != INDEXER_COLS ||
+        indexer->dims[1] != INDEXER_ROWS ||
+        indexer->size != INDEXER_ROWS * 144u) {
+        fail("indexer Q4_K metadata");
+    }
+    if (strcmp(after->name, k_after_name) != 0 ||
+        after->type != DS4Q_TYPE_F16 ||
+        after->size != sizeof(k_after_payload)) {
+        fail("after sentinel metadata");
+    }
+
+    uint8_t before_actual[sizeof(k_before_payload)];
+    uint8_t after_actual[sizeof(k_after_payload)];
+    if (!read_payload(&gguf, 0, before_actual, sizeof(before_actual)) ||
+        memcmp(before_actual, k_before_payload, sizeof(before_actual)) != 0) {
+        fail("before sentinel payload changed");
+    }
+    if (!read_payload(&gguf, 2, after_actual, sizeof(after_actual)) ||
+        memcmp(after_actual, k_after_payload, sizeof(after_actual)) != 0) {
+        fail("after sentinel payload changed");
+    }
+
+    const size_t count = (size_t)INDEXER_COLS * INDEXER_ROWS;
+    const size_t q4_size = INDEXER_ROWS * ds4q_row_size(
+        DS4Q_TYPE_Q4_K, INDEXER_COLS);
+    float *rounded = malloc(count * sizeof(*rounded));
+    uint8_t *expected = malloc(q4_size);
+    uint8_t *actual = malloc(q4_size);
+    if (!rounded || !expected || !actual) {
+        fail("reference allocation");
+    } else {
+        for (size_t i = 0; i < count; i++) {
+            rounded[i] = ds4q_f16_to_f32(half[i]);
+        }
+        ds4q_quantize_init(DS4Q_TYPE_Q4_K);
+        const size_t written = ds4q_quantize_chunk(
+            DS4Q_TYPE_Q4_K, rounded, expected, 0,
+            INDEXER_ROWS, INDEXER_COLS, NULL);
+        if (written != q4_size) {
+            fail("reference Q4_K size");
+        } else if (!read_payload(&gguf, 1, actual, q4_size)) {
+            fail("read indexer payload");
+        } else if (memcmp(actual, expected, q4_size) != 0) {
+            size_t first = 0;
+            while (first < q4_size && actual[first] == expected[first]) first++;
+            fprintf(stderr,
+                    "test_quantizer_indexer_q4: first Q4 mismatch at %zu/%zu\n",
+                    first, q4_size);
+            fail("indexer Q4_K payload mismatch");
+        }
+    }
+    free(actual);
+    free(expected);
+    free(rounded);
+    close_parsed(&gguf);
+}
+
+int main(int argc, char **argv) {
+    const char *tool = argc > 1 ? argv[1] : "./gguf-tools/deepseek4-quantize";
+    if (argc > 2) {
+        fprintf(stderr, "usage: %s [deepseek4-quantize]\n", argv[0]);
+        return 2;
+    }
+    if (access(tool, X_OK) != 0) {
+        fprintf(stderr, "test_quantizer_indexer_q4: executable not found: %s\n",
+                tool);
+        return 2;
+    }
+    for (size_t i = 0; i < sizeof(k_before_payload); i++) {
+        k_before_payload[i] = (uint8_t)(0x31u + i * 7u);
+    }
+    for (size_t i = 0; i < sizeof(k_after_payload); i++) {
+        k_after_payload[i] = (uint8_t)(0xd3u - i * 5u);
+    }
+
+    char tmpdir[] = "/tmp/ds4-indexer-q4.XXXXXX";
+    if (!mkdtemp(tmpdir)) {
+        perror("mkdtemp");
+        return 1;
+    }
+    char source[512];
+    char output[512];
+    char bad_source[512];
+    char bad_output[512];
+    if (snprintf(source, sizeof(source), "%s/source.gguf", tmpdir) >=
+            (int)sizeof(source) ||
+        snprintf(output, sizeof(output), "%s/output.gguf", tmpdir) >=
+            (int)sizeof(output) ||
+        snprintf(bad_source, sizeof(bad_source), "%s/bad-source.gguf", tmpdir) >=
+            (int)sizeof(bad_source) ||
+        snprintf(bad_output, sizeof(bad_output), "%s/bad-output.gguf", tmpdir) >=
+            (int)sizeof(bad_output)) {
+        fail("temporary path too long");
+        goto cleanup_dir;
+    }
+
+    uint16_t *half = make_indexer_f16(INDEXER_COLS, INDEXER_ROWS);
+    if (!half || !write_fixture(source, INDEXER_COLS, INDEXER_ROWS, half)) {
+        fail("write positive fixture");
+    } else {
+        const int status = run_quantizer(tool, source, output);
+        if (status != 0) {
+            fprintf(stderr,
+                    "test_quantizer_indexer_q4: positive quantizer exit=%d\n",
+                    status);
+            fail("positive quantizer invocation");
+        } else {
+            verify_positive(output, half);
+        }
+    }
+    free(half);
+
+    uint16_t *bad_half = make_indexer_f16(INDEXER_COLS - 1u, 2u);
+    if (!bad_half ||
+        !write_fixture(bad_source, INDEXER_COLS - 1u, 2u, bad_half)) {
+        fail("write unaligned fixture");
+    } else {
+        const int status = run_quantizer(tool, bad_source, bad_output);
+        if (status == 0) fail("unaligned width unexpectedly accepted");
+        if (access(bad_output, F_OK) == 0) {
+            fail("unaligned conversion created output");
+        }
+    }
+    free(bad_half);
+
+    unlink(bad_output);
+    unlink(bad_source);
+    unlink(output);
+    unlink(source);
+cleanup_dir:
+    if (rmdir(tmpdir) != 0 && errno != ENOENT) {
+        fail("remove temporary directory");
+    }
+
+    if (g_failures != 0) {
+        fprintf(stderr, "test_quantizer_indexer_q4: %d failure(s)\n", g_failures);
+        return 1;
+    }
+    fprintf(stderr,
+            "test_quantizer_indexer_q4 PASS f16_rows=17 q4_bytes=%zu "
+            "sentinels=2 unaligned_rejected=1\n",
+            (size_t)INDEXER_ROWS * ds4q_row_size(DS4Q_TYPE_Q4_K, INDEXER_COLS));
+    return 0;
+}
diff --git a/tests/test_rocm_q4_dense_pair.cpp b/tests/test_rocm_q4_dense_pair.cpp
new file mode 100644
index 0000000000..abd6a2d38c
--- /dev/null
+++ b/tests/test_rocm_q4_dense_pair.cpp
@@ -0,0 +1,3833 @@
+// SPDX-License-Identifier: MIT
+// Deterministic ROCm Q4_K dense/pair/tiled-prefill oracle.
+//
+// The test deliberately goes through the public tensor/model-map API.  Weight
+// rows use the raw 144-byte GGUF Q4_K layout, while the CPU reference mirrors
+// the backend's F32 -> Q8_K quantizer and Q4_K x Q8_K integer dot product.
+// Prefill controls are forced through the rollback path before the TILE8
+// REQUIRE path so a future default promotion cannot turn parity into a
+// candidate-vs-candidate false green.
+
+#include "ds4_gpu.h"
+
+#if defined(__has_include)
+#  if __has_include(<hip/hip_runtime.h>)
+#    include <hip/hip_runtime.h>
+#    define DS4_TEST_HAS_HIP_RUNTIME 1
+#  endif
+#endif
+#ifndef DS4_TEST_HAS_HIP_RUNTIME
+#  define DS4_TEST_HAS_HIP_RUNTIME 0
+#endif
+
+#include <algorithm>
+#include <cmath>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <limits>
+#include <string>
+#include <sys/mman.h>
+#include <vector>
+
+extern "C" int ds4_rocm_test_q4_prefill_k1024_tile4_policy(
+    int ssd_streaming, int weight_device_resident, int ssd_enabled,
+    int disabled, int required);
+extern "C" void ds4_rocm_test_q4_prefill_wmma_reset(void);
+extern "C" uint64_t ds4_rocm_test_q4_prefill_wmma_get_calls(void);
+extern "C" uint64_t ds4_rocm_test_q4_prefill_wmma_k64_get_calls(void);
+extern "C" uint64_t ds4_rocm_test_q4_prefill_wmma_k128_get_calls(void);
+extern "C" int ds4_rocm_test_q4_prefill_wmma_k64_control_policy(
+    int control);
+extern "C" int ds4_rocm_test_q4_prefill_wmma_k128_policy(
+    int disabled, int k64_enabled, uint32_t row_tile,
+    int load4_compatible);
+extern "C" int ds4_rocm_test_q4_prefill_wmma_requested_policy(
+    int ssd_streaming, int enabled, int ssd_enabled, int disabled,
+    int required);
+extern "C" int
+ds4_rocm_test_q4_prefill_wmma_attention_a_requested_policy(
+    int ssd_streaming, int enabled, int ssd_enabled, int disabled,
+    int required);
+extern "C" int
+ds4_rocm_test_q4_prefill_wmma_attention_b_requested_policy(
+    int ssd_streaming, int enabled, int ssd_enabled, int disabled,
+    int required);
+extern "C" int ds4_rocm_test_q4_prefill_wmma_yields_to_q8_wave32(
+    int q8_selected, int wmma_required);
+extern "C" uint32_t ds4_rocm_test_q4_prefill_wmma_row_tile(
+    uint32_t out_dim);
+extern "C" int ds4_rocm_test_q4_prefill_q8_wave32_policy(
+    int prefill_scope, int runtime_compatible, int enabled, int disabled,
+    int required);
+extern "C" void ds4_rocm_test_q4_prefill_q8_wave32_reset(void);
+extern "C" uint64_t ds4_rocm_test_q4_prefill_q8_wave32_get_calls(void);
+extern "C" int ds4_rocm_test_q8_K_quantize_tensor(
+    ds4_gpu_tensor *out, const ds4_gpu_tensor *x, uint32_t in_dim,
+    uint32_t n_rows, int use_wave32);
+extern "C" int ds4_rocm_test_q4_attn_q_b_yield_to_q8_wave32_policy(
+    uint32_t weight_type, uint32_t n_tok, int q8_wave32_required,
+    int f16_cache_required);
+extern "C" int ds4_rocm_test_q4_pair_pre_enqueue_failure_policy(
+    int prefill_scope, int tile8_required, int q8_wave32_required);
+
+namespace {
+
+constexpr uint32_t kQkK = 256u;
+constexpr uint32_t kK = 4096u;
+constexpr uint32_t kM0 = 65u;
+constexpr uint32_t kM1 = 33u;
+constexpr uint32_t kQ4Type = 12u;
+constexpr uint32_t kQ8Type = 8u;
+constexpr uint32_t kTailK = 1024u;
+constexpr uint32_t kQbOutDim = 32768u;
+constexpr uint32_t kQbHeads = 64u;
+constexpr uint32_t kQbHeadDim = 512u;
+constexpr uint32_t kQbRot = 64u;
+constexpr uint32_t kAttnGroupDim = 4096u;
+constexpr uint32_t kAttnRank = 32u;
+constexpr uint32_t kAttnGroups = 8u;
+constexpr uint32_t kAttnLowDim = kAttnGroups * kAttnRank;
+constexpr uint32_t kAttnOutDim = 65u;
+constexpr uint32_t kDecodeAttnGroupDim = 4096u;
+constexpr uint32_t kDecodeAttnRank = 1024u;
+constexpr uint32_t kDecodeAttnGroups = 8u;
+constexpr uint32_t kDecodeAttnLowDim =
+    kDecodeAttnGroups * kDecodeAttnRank;
+constexpr uint32_t kDecodeAttnOutDim = 4096u;
+constexpr size_t kOutputGuardFloats = 257u;
+constexpr float kCpuAbsTolerance = 2.0e-3f;
+constexpr float kCpuRelTolerance = 3.0e-5f;
+constexpr int kSkip = 77;
+
+constexpr const char *kPrefillEnable =
+    "DS4_ROCM_ENABLE_Q4_PREFILL_TILE8";
+constexpr const char *kPrefillDisable =
+    "DS4_ROCM_DISABLE_Q4_PREFILL_TILE8";
+constexpr const char *kPrefillRequire =
+    "DS4_ROCM_REQUIRE_Q4_PREFILL_TILE8";
+constexpr const char *kPrefillK1024Tile4Disable =
+    "DS4_ROCM_DISABLE_Q4_PREFILL_K1024_TILE4";
+constexpr const char *kPrefillK1024Tile4SsdEnable =
+    "DS4_ROCM_ENABLE_Q4_PREFILL_K1024_TILE4_SSD";
+constexpr const char *kPrefillK1024Tile4Require =
+    "DS4_ROCM_REQUIRE_Q4_PREFILL_K1024_TILE4";
+constexpr const char *kPrefillWmmaEnable =
+    "DS4_ROCM_ENABLE_Q4_PREFILL_WMMA";
+constexpr const char *kPrefillWmmaSsdEnable =
+    "DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_SSD";
+constexpr const char *kPrefillWmmaDisable =
+    "DS4_ROCM_DISABLE_Q4_PREFILL_WMMA";
+constexpr const char *kPrefillWmmaRequire =
+    "DS4_ROCM_REQUIRE_Q4_PREFILL_WMMA";
+constexpr const char *kPrefillWmmaRowTile =
+    "DS4_ROCM_Q4_PREFILL_WMMA_ROW_TILE";
+constexpr const char *kPrefillWmmaK64 =
+    "DS4_ROCM_ENABLE_Q4_PREFILL_WMMA_K64";
+constexpr const char *kPrefillWmmaK128Disable =
+    "DS4_ROCM_DISABLE_Q4_PREFILL_WMMA_K128";
+constexpr const char *kPrefillQ8Wave32Enable =
+    "DS4_ROCM_ENABLE_Q4_PREFILL_Q8_K_WAVE32";
+constexpr const char *kPrefillQ8Wave32Disable =
+    "DS4_ROCM_DISABLE_Q4_PREFILL_Q8_K_WAVE32";
+constexpr const char *kPrefillQ8Wave32Require =
+    "DS4_ROCM_REQUIRE_Q4_PREFILL_Q8_K_WAVE32";
+constexpr const char *kQbF16Enable =
+    "DS4_ROCM_ENABLE_Q4_ATTN_Q_B_F16_CACHE";
+constexpr const char *kQbF16Disable =
+    "DS4_ROCM_DISABLE_Q4_ATTN_Q_B_F16_CACHE";
+constexpr const char *kQbF16Require =
+    "DS4_ROCM_REQUIRE_Q4_ATTN_Q_B_F16_CACHE";
+constexpr const char *kQbF16MinTokens =
+    "DS4_ROCM_Q4_ATTN_Q_B_F16_CACHE_MIN_TOKENS";
+constexpr const char *kQbF16OutputEnable =
+    "DS4_ROCM_ENABLE_Q4_ATTN_Q_B_F16_OUTPUT";
+constexpr const char *kGroupedDecodeEnable =
+    "DS4_ROCM_ENABLE_Q4_GROUPED_ATTN_A";
+constexpr const char *kGroupedDecodeDisable =
+    "DS4_ROCM_DISABLE_Q4_GROUPED_ATTN_A";
+constexpr const char *kGroupedDecodeRequire =
+    "DS4_ROCM_REQUIRE_Q4_GROUPED_ATTN_A";
+constexpr const char *kGroupedDecodeStats =
+    "DS4_ROCM_Q4_GROUPED_ATTN_A_STATS";
+
+struct block_q4_K_test {
+    uint16_t d;
+    uint16_t dmin;
+    uint8_t scales[12];
+    uint8_t qs[kQkK / 2u];
+};
+
+struct block_q8_K_test {
+    float d;
+    int8_t qs[kQkK];
+    int16_t bsums[kQkK / 16u];
+};
+
+struct block_q8_0_test {
+    uint16_t d;
+    int8_t qs[32];
+};
+
+static_assert(sizeof(block_q4_K_test) == 144u,
+              "Q4_K fixture must match the raw GGUF layout");
+static_assert(sizeof(block_q8_K_test) == 292u,
+              "Q8_K oracle must match the ROCm activation layout");
+static_assert(sizeof(block_q8_0_test) == 34u,
+              "Q8_0 fixture must match the raw GGUF layout");
+
+struct tensor_owner {
+    ds4_gpu_tensor *ptr = nullptr;
+
+    explicit tensor_owner(uint64_t bytes) : ptr(ds4_gpu_tensor_alloc(bytes)) {}
+    explicit tensor_owner(ds4_gpu_tensor *owned) : ptr(owned) {}
+    ~tensor_owner() { ds4_gpu_tensor_free(ptr); }
+
+    tensor_owner(const tensor_owner &) = delete;
+    tensor_owner &operator=(const tensor_owner &) = delete;
+};
+
+struct aligned_model {
+    uint8_t *data = nullptr;
+    uint64_t size = 0;
+    uint64_t weight0_offset = 0;
+    uint64_t weight1_offset = 0;
+    uint64_t attn_a_offset = 0;
+    uint64_t decode_attn_a_offset = 0;
+    uint64_t attn_b_offset = 0;
+    uint64_t attn_b_q8_offset = 0;
+    uint64_t tail_k1024_offset = 0;
+    uint64_t tail_k1024_pair_offset = 0;
+    uint64_t q_b_k1024_offset = 0;
+    uint64_t decode_attn_b_offset = 0;
+
+    ~aligned_model() { std::free(data); }
+
+    aligned_model(const aligned_model &) = delete;
+    aligned_model &operator=(const aligned_model &) = delete;
+    aligned_model() = default;
+};
+
+struct env_snapshot {
+    const char *name;
+    bool was_set;
+    std::string value;
+
+    explicit env_snapshot(const char *key)
+        : name(key), was_set(std::getenv(key) != nullptr),
+          value(was_set ? std::getenv(key) : "") {}
+    ~env_snapshot() {
+        if (was_set) {
+            (void)setenv(name, value.c_str(), 1);
+        } else {
+            (void)unsetenv(name);
+        }
+    }
+
+    env_snapshot(const env_snapshot &) = delete;
+    env_snapshot &operator=(const env_snapshot &) = delete;
+};
+
+uint64_t round_up(uint64_t value, uint64_t alignment) {
+    return (value + alignment - 1u) / alignment * alignment;
+}
+
+uint32_t lcg_next(uint32_t &state) {
+    state = state * 1664525u + 1013904223u;
+    return state;
+}
+
+float fp16_to_float(uint16_t h) {
+    const uint32_t sign = (uint32_t)(h & 0x8000u) << 16u;
+    uint32_t exp = (h >> 10u) & 0x1fu;
+    uint32_t mant = h & 0x3ffu;
+    uint32_t bits;
+    if (exp == 0u) {
+        if (mant == 0u) {
+            bits = sign;
+        } else {
+            int shift = 0;
+            while ((mant & 0x400u) == 0u) {
+                mant <<= 1u;
+                shift++;
+            }
+            mant &= 0x3ffu;
+            bits = sign | (uint32_t)(127 - 14 - shift) << 23u | mant << 13u;
+        }
+    } else if (exp == 31u) {
+        bits = sign | 0x7f800000u | mant << 13u;
+    } else {
+        bits = sign | (exp + 112u) << 23u | mant << 13u;
+    }
+    float out;
+    std::memcpy(&out, &bits, sizeof(out));
+    return out;
+}
+
+uint16_t float_to_fp16(float value) {
+    uint32_t bits;
+    std::memcpy(&bits, &value, sizeof(bits));
+    const uint32_t sign = bits >> 31u;
+    int32_t exp = (int32_t)((bits >> 23u) & 0xffu) - 127 + 15;
+    uint32_t mant = bits & 0x7fffffu;
+    if (exp >= 31) return (uint16_t)((sign << 15u) | 0x7c00u);
+    if (exp <= 0) {
+        if (exp < -10) return (uint16_t)(sign << 15u);
+        mant |= 0x800000u;
+        const uint32_t shift = (uint32_t)(14 - exp);
+        uint32_t rounded = mant >> shift;
+        const uint32_t halfway = 1u << (shift - 1u);
+        if ((mant & halfway) &&
+            ((mant & (halfway - 1u)) || (rounded & 1u))) {
+            rounded++;
+        }
+        return (uint16_t)((sign << 15u) | rounded);
+    }
+    uint32_t rounded = mant + 0x0fffu + ((mant >> 13u) & 1u);
+    if (rounded & 0x800000u) {
+        rounded = 0u;
+        exp++;
+        if (exp >= 31) return (uint16_t)((sign << 15u) | 0x7c00u);
+    }
+    return (uint16_t)((sign << 15u) | (uint32_t)exp << 10u |
+                      rounded >> 13u);
+}
+
+void q4_scale_min(uint32_t j, const uint8_t *scales,
+                  uint8_t *scale, uint8_t *minimum) {
+    if (j < 4u) {
+        *scale = scales[j] & 63u;
+        *minimum = scales[j + 4u] & 63u;
+    } else {
+        *scale = (scales[j + 4u] & 0x0fu) |
+                 (uint8_t)((scales[j - 4u] >> 6u) << 4u);
+        *minimum = (scales[j + 4u] >> 4u) |
+                   (uint8_t)((scales[j] >> 6u) << 4u);
+    }
+}
+
+void fill_q4_rows(block_q4_K_test *rows, uint32_t n_rows,
+                  uint32_t in_dim, uint32_t seed) {
+    uint32_t state = seed;
+    const uint32_t blocks_per_row = in_dim / kQkK;
+    for (uint32_t row = 0; row < n_rows; row++) {
+        for (uint32_t b = 0; b < blocks_per_row; b++) {
+            block_q4_K_test &block = rows[(uint64_t)row * blocks_per_row + b];
+            const float d = 0.0025f +
+                0.00025f * (float)(1u + (lcg_next(state) % 23u));
+            const float dmin = 0.0010f +
+                0.00020f * (float)(1u + (lcg_next(state) % 19u));
+            block.d = float_to_fp16(d);
+            block.dmin = float_to_fp16(dmin);
+            for (uint8_t &v : block.scales) v = (uint8_t)(lcg_next(state) >> 24u);
+            for (uint8_t &v : block.qs) v = (uint8_t)(lcg_next(state) >> 24u);
+        }
+    }
+}
+
+void fill_q8_0_rows(block_q8_0_test *rows, uint32_t n_rows,
+                    uint32_t in_dim, uint32_t seed) {
+    uint32_t state = seed;
+    const uint32_t blocks_per_row = in_dim / 32u;
+    for (uint32_t row = 0; row < n_rows; row++) {
+        for (uint32_t b = 0; b < blocks_per_row; b++) {
+            block_q8_0_test &block =
+                rows[(uint64_t)row * blocks_per_row + b];
+            const float scale = 0.0015f +
+                0.000125f * (float)(1u + (lcg_next(state) % 29u));
+            block.d = float_to_fp16(scale);
+            for (int8_t &q : block.qs) {
+                q = (int8_t)((int)(lcg_next(state) % 255u) - 127);
+            }
+        }
+    }
+}
+
+bool make_model(aligned_model *model) {
+    constexpr uint64_t page = 4096u;
+    const uint64_t row_bytes = (kK / kQkK) * sizeof(block_q4_K_test);
+    const uint64_t weight0_bytes = kM0 * row_bytes;
+    const uint64_t weight1_bytes = kM1 * row_bytes;
+    const uint64_t attn_a_row_bytes =
+        (kAttnGroupDim / kQkK) * sizeof(block_q4_K_test);
+    const uint64_t attn_a_bytes =
+        (uint64_t)kAttnGroups * kAttnRank * attn_a_row_bytes;
+    const uint64_t decode_attn_a_row_bytes =
+        (kDecodeAttnGroupDim / kQkK) * sizeof(block_q4_K_test);
+    const uint64_t decode_attn_a_group_bytes =
+        (uint64_t)kDecodeAttnRank * decode_attn_a_row_bytes;
+    const uint64_t decode_attn_a_bytes =
+        (uint64_t)kDecodeAttnGroups * decode_attn_a_group_bytes;
+    const uint64_t attn_b_row_bytes =
+        (kAttnLowDim / kQkK) * sizeof(block_q4_K_test);
+    const uint64_t attn_b_bytes =
+        (uint64_t)kAttnOutDim * attn_b_row_bytes;
+    const uint64_t tail_row_bytes =
+        (kTailK / kQkK) * sizeof(block_q4_K_test);
+    const uint64_t tail0_bytes = (uint64_t)kM0 * tail_row_bytes;
+    const uint64_t tail1_bytes = (uint64_t)kM1 * tail_row_bytes;
+    const uint64_t q_b_k1024_bytes =
+        (uint64_t)kQbOutDim * tail_row_bytes;
+    const uint64_t decode_attn_b_row_bytes =
+        (kDecodeAttnLowDim / kQkK) * sizeof(block_q4_K_test);
+    const uint64_t decode_attn_b_bytes =
+        (uint64_t)kDecodeAttnOutDim * decode_attn_b_row_bytes;
+    const uint64_t attn_b_q8_row_bytes =
+        (kAttnLowDim / 32u) * sizeof(block_q8_0_test);
+    const uint64_t attn_b_q8_bytes =
+        (uint64_t)kAttnOutDim * attn_b_q8_row_bytes;
+    model->weight0_offset = 0u;
+    model->weight1_offset = round_up(weight0_bytes, page);
+    model->attn_a_offset = round_up(
+        model->weight1_offset + weight1_bytes, page);
+    model->decode_attn_a_offset = round_up(
+        model->attn_a_offset + attn_a_bytes, page);
+    model->attn_b_offset = round_up(
+        model->decode_attn_a_offset + decode_attn_a_bytes, page);
+    model->tail_k1024_offset = round_up(
+        model->attn_b_offset + attn_b_bytes, page);
+    model->tail_k1024_pair_offset = round_up(
+        model->tail_k1024_offset + tail0_bytes, page);
+    model->attn_b_q8_offset = round_up(
+        model->tail_k1024_pair_offset + tail1_bytes, page);
+    model->q_b_k1024_offset = round_up(
+        model->attn_b_q8_offset + attn_b_q8_bytes, page);
+    model->decode_attn_b_offset = round_up(
+        model->q_b_k1024_offset + q_b_k1024_bytes, page);
+    model->size = round_up(
+        model->decode_attn_b_offset + decode_attn_b_bytes, page);
+    void *storage = nullptr;
+    if (posix_memalign(&storage, (size_t)page, (size_t)model->size) != 0) {
+        return false;
+    }
+    model->data = static_cast<uint8_t *>(storage);
+    std::memset(model->data, 0xa5, (size_t)model->size);
+    fill_q4_rows(reinterpret_cast<block_q4_K_test *>(
+                     model->data + model->weight0_offset),
+                 kM0, kK, 0x41c64e6du);
+    fill_q4_rows(reinterpret_cast<block_q4_K_test *>(
+                     model->data + model->weight1_offset),
+                 kM1, kK, 0x9e3779b9u);
+    fill_q4_rows(reinterpret_cast<block_q4_K_test *>(
+                     model->data + model->attn_a_offset),
+                 kAttnGroups * kAttnRank, kAttnGroupDim, 0x243f6a88u);
+    for (uint32_t group = 0; group < kDecodeAttnGroups; group++) {
+        fill_q4_rows(reinterpret_cast<block_q4_K_test *>(
+                         model->data + model->decode_attn_a_offset +
+                         (uint64_t)group * decode_attn_a_group_bytes),
+                     kDecodeAttnRank, kDecodeAttnGroupDim,
+                     0xd1b54a35u ^ (group * 0x9e3779b9u));
+    }
+    fill_q4_rows(reinterpret_cast<block_q4_K_test *>(
+                     model->data + model->attn_b_offset),
+                 kAttnOutDim, kAttnLowDim, 0x85a308d3u);
+    fill_q4_rows(reinterpret_cast<block_q4_K_test *>(
+                     model->data + model->tail_k1024_offset),
+                 kM0, kTailK, 0x13198a2eu);
+    fill_q4_rows(reinterpret_cast<block_q4_K_test *>(
+                     model->data + model->tail_k1024_pair_offset),
+                 kM1, kTailK, 0xa4093822u);
+    fill_q4_rows(reinterpret_cast<block_q4_K_test *>(
+                     model->data + model->q_b_k1024_offset),
+                 kQbOutDim, kTailK, 0x082efa98u);
+    fill_q4_rows(reinterpret_cast<block_q4_K_test *>(
+                     model->data + model->decode_attn_b_offset),
+                 kDecodeAttnOutDim, kDecodeAttnLowDim, 0x452821e6u);
+    fill_q8_0_rows(reinterpret_cast<block_q8_0_test *>(
+                       model->data + model->attn_b_q8_offset),
+                   kAttnOutDim, kAttnLowDim, 0x03707344u);
+    return true;
+}
+
+void fill_activation(std::vector<float> *x, uint32_t n_tokens,
+                     uint32_t in_dim = kK) {
+    x->resize((uint64_t)n_tokens * in_dim);
+    for (uint32_t token = 0; token < n_tokens; token++) {
+        for (uint32_t b = 0; b < in_dim / kQkK; b++) {
+            float *block =
+                x->data() + (uint64_t)token * in_dim + b * kQkK;
+            for (uint32_t i = 0; i < kQkK; i++) {
+                const int q = (int)((i * 73u + token * 37u + b * 19u) % 241u) - 120;
+                block[i] = (float)q / 32.0f;
+            }
+            // A unique, exactly representable maximum makes the CPU and GPU
+            // quantizers select the same signed scale without tie ambiguity.
+            block[0] = ((token + b) & 1u) ? 127.0f / 32.0f
+                                          : -127.0f / 32.0f;
+        }
+    }
+}
+
+/* Exercise the canonical max-selection edge cases: all-zero blocks and equal
+ * opposite-sign maxima spanning both the per-lane and cross-lane tree. */
+void fill_q8_wave32_activation(std::vector<float> *x, uint32_t n_tokens,
+                               uint32_t in_dim) {
+    x->resize((uint64_t)n_tokens * in_dim);
+    for (uint32_t token = 0; token < n_tokens; token++) {
+        for (uint32_t b = 0; b < in_dim / kQkK; b++) {
+            float *block = x->data() + (uint64_t)token * in_dim + b * kQkK;
+            if (((token + b) % 7u) == 0u) {
+                std::fill(block, block + kQkK, 0.0f);
+                continue;
+            }
+            for (uint32_t i = 0; i < kQkK; i++) {
+                const int value =
+                    (int)((i * 29u + token * 17u + b * 11u) % 97u) - 48;
+                block[i] = (float)value / 16.0f;
+            }
+            const float first = ((token + b) & 1u) ? 4.0f : -4.0f;
+            uint32_t first_index = 5u;
+            uint32_t second_index = 224u;
+            switch ((token + b) % 3u) {
+                case 1u:
+                    first_index = 0u;
+                    second_index = 128u;
+                    break;
+                case 2u:
+                    first_index = 1u;
+                    second_index = 16u;
+                    break;
+                default:
+                    break;
+            }
+            block[first_index] = first;
+            block[second_index] = -first;
+        }
+    }
+}
+
+void quantize_q8_K_cpu(const float *x, block_q8_K_test *out) {
+    float abs_part[kQkK];
+    float val_part[kQkK];
+    for (uint32_t i = 0; i < kQkK; i++) {
+        abs_part[i] = std::fabs(x[i]);
+        val_part[i] = x[i];
+    }
+    /* Mirror the canonical GPU reduction literally.  A linear scan chooses a
+     * different signed maximum when equal magnitudes occur, which changes the
+     * raw Q8_K bytes even though the resulting dot product is equivalent. */
+    for (uint32_t stride = kQkK >> 1u; stride != 0u; stride >>= 1u) {
+        for (uint32_t i = 0; i < stride; i++) {
+            if (abs_part[i + stride] > abs_part[i]) {
+                abs_part[i] = abs_part[i + stride];
+                val_part[i] = val_part[i + stride];
+            }
+        }
+    }
+    const float amax = abs_part[0];
+    const float maxv = val_part[0];
+    if (amax == 0.0f) {
+        std::memset(out, 0, sizeof(*out));
+        return;
+    }
+    const float iscale = -127.0f / maxv;
+    for (uint32_t i = 0; i < kQkK; i++) {
+        int q = (int)std::lrint(iscale * x[i]);
+        q = std::max(-128, std::min(127, q));
+        out->qs[i] = (int8_t)q;
+    }
+    for (uint32_t group = 0; group < kQkK / 16u; group++) {
+        int sum = 0;
+        for (uint32_t i = 0; i < 16u; i++) {
+            sum += out->qs[group * 16u + i];
+        }
+        out->bsums[group] = (int16_t)sum;
+    }
+    out->d = 1.0f / iscale;
+}
+
+float dot_q4_q8_raw(const block_q4_K_test &weight,
+                    const block_q8_K_test &activation) {
+    int isum = 0;
+    int summs = 0;
+    for (uint32_t j = 0; j < 8u; j++) {
+        uint8_t scale = 0;
+        uint8_t minimum = 0;
+        q4_scale_min(j, weight.scales, &scale, &minimum);
+        summs += (int)minimum *
+                 ((int)activation.bsums[2u * j] +
+                  (int)activation.bsums[2u * j + 1u]);
+        const uint32_t byte_offset = (j >> 1u) * 32u;
+        const uint32_t shift = (j & 1u) ? 4u : 0u;
+        int group_dot = 0;
+        for (uint32_t i = 0; i < 32u; i++) {
+            const int q4 = (weight.qs[byte_offset + i] >> shift) & 0x0f;
+            group_dot += q4 * (int)activation.qs[j * 32u + i];
+        }
+        isum += (int)scale * group_dot;
+    }
+    const float d = fp16_to_float(weight.d);
+    const float dmin = fp16_to_float(weight.dmin);
+    return activation.d * d * (float)isum -
+           activation.d * dmin * (float)summs;
+}
+
+std::vector<float> dense_reference(const uint8_t *weight_base,
+                                   const std::vector<float> &x,
+                                   uint32_t out_dim,
+                                   uint32_t n_tokens,
+                                   uint32_t in_dim = kK) {
+    const auto *weights = reinterpret_cast<const block_q4_K_test *>(weight_base);
+    const uint32_t blocks_per_row = in_dim / kQkK;
+    std::vector<block_q8_K_test> xq((uint64_t)n_tokens * blocks_per_row);
+    for (uint32_t token = 0; token < n_tokens; token++) {
+        for (uint32_t b = 0; b < blocks_per_row; b++) {
+            quantize_q8_K_cpu(
+                x.data() + (uint64_t)token * in_dim + b * kQkK,
+                &xq[(uint64_t)token * blocks_per_row + b]);
+        }
+    }
+    std::vector<float> result((uint64_t)n_tokens * out_dim, 0.0f);
+    for (uint32_t token = 0; token < n_tokens; token++) {
+        for (uint32_t row = 0; row < out_dim; row++) {
+            // Mirror the kernel's b=lane; b+=8 walk and width-8 shuffle tree.
+            // This makes raw-bit diagnostics meaningful even at the real
+            // K=4096 shape while the tolerance remains the promotion gate.
+            float lane_sum[8] = {};
+            for (uint32_t lane = 0; lane < 8u; lane++) {
+                for (uint32_t b = lane; b < blocks_per_row; b += 8u) {
+                    lane_sum[lane] += dot_q4_q8_raw(
+                        weights[(uint64_t)row * blocks_per_row + b],
+                        xq[(uint64_t)token * blocks_per_row + b]);
+                }
+            }
+            for (uint32_t offset = 4u; offset > 0u; offset >>= 1u) {
+                for (uint32_t lane = 0; lane + offset < 8u; lane++) {
+                    lane_sum[lane] += lane_sum[lane + offset];
+                }
+            }
+            result[(uint64_t)token * out_dim + row] = lane_sum[0];
+        }
+    }
+    return result;
+}
+
+bool close_to_cpu(const std::vector<float> &got,
+                  const std::vector<float> &expected,
+                  const char *label) {
+    uint64_t raw_mismatches = 0;
+    uint64_t tolerance_failures = 0;
+    float max_abs = 0.0f;
+    float max_rel = 0.0f;
+    size_t worst = 0;
+    for (size_t i = 0; i < got.size(); i++) {
+        if (std::memcmp(&got[i], &expected[i], sizeof(float)) != 0) {
+            raw_mismatches++;
+        }
+        const float diff = std::fabs(got[i] - expected[i]);
+        const float rel = diff / std::max(1.0f, std::fabs(expected[i]));
+        if (diff > max_abs) {
+            max_abs = diff;
+            worst = i;
+        }
+        max_rel = std::max(max_rel, rel);
+        const float limit = kCpuAbsTolerance +
+                            kCpuRelTolerance * std::fabs(expected[i]);
+        if (!std::isfinite(got[i]) || diff > limit) tolerance_failures++;
+    }
+    std::fprintf(stderr,
+                 "%s: raw_mismatches=%llu/%zu max_abs=%g max_rel=%g "
+                 "worst=%zu tolerance(abs=%g rel=%g) %s\n",
+                 label, (unsigned long long)raw_mismatches, got.size(),
+                 max_abs, max_rel, worst, kCpuAbsTolerance,
+                 kCpuRelTolerance, tolerance_failures == 0 ? "PASS" : "FAIL");
+    if (tolerance_failures != 0 && worst < got.size()) {
+        std::fprintf(stderr, "  worst got=%g cpu=%g delta=%g\n",
+                     got[worst], expected[worst], got[worst] - expected[worst]);
+    }
+    return tolerance_failures == 0;
+}
+
+bool close_with_tolerance(const std::vector<float> &got,
+                          const std::vector<float> &expected,
+                          float abs_tolerance,
+                          float rel_tolerance,
+                          const char *label,
+                          bool gate = true) {
+    if (got.size() != expected.size()) return false;
+    uint64_t failures = 0;
+    uint64_t nonfinite = 0;
+    float max_abs = 0.0f;
+    float max_rel = 0.0f;
+    size_t worst = 0;
+    for (size_t i = 0; i < got.size(); i++) {
+        if (!std::isfinite(got[i]) || !std::isfinite(expected[i])) {
+            failures++;
+            nonfinite++;
+            continue;
+        }
+        const float diff = std::fabs(got[i] - expected[i]);
+        const float rel = diff / std::max(1.0f, std::fabs(expected[i]));
+        if (diff > max_abs) {
+            max_abs = diff;
+            worst = i;
+        }
+        max_rel = std::max(max_rel, rel);
+        if (diff > abs_tolerance + rel_tolerance * std::fabs(expected[i])) {
+            failures++;
+        }
+    }
+    std::fprintf(stderr,
+                 "%s: failures=%llu/%zu nonfinite=%llu max_abs=%g "
+                 "max_rel=%g worst=%zu tolerance(abs=%g rel=%g) %s\n",
+                 label, (unsigned long long)failures, got.size(),
+                 (unsigned long long)nonfinite, max_abs, max_rel, worst,
+                 abs_tolerance, rel_tolerance,
+                 failures == 0u ? "PASS" :
+                 (gate || nonfinite != 0u ? "FAIL" : "DIAGNOSTIC"));
+    return nonfinite == 0u && (failures == 0u || !gate);
+}
+
+bool bitwise_equal(const std::vector<float> &got,
+                   const std::vector<float> &expected,
+                   const char *label) {
+    uint64_t mismatches = 0;
+    size_t first = 0;
+    for (size_t i = 0; i < got.size(); i++) {
+        if (std::memcmp(&got[i], &expected[i], sizeof(float)) != 0) {
+            if (mismatches == 0) first = i;
+            mismatches++;
+        }
+    }
+    std::fprintf(stderr, "%s: raw_mismatches=%llu/%zu %s\n",
+                 label, (unsigned long long)mismatches, got.size(),
+                 mismatches == 0 ? "PASS" : "FAIL");
+    if (mismatches != 0) {
+        uint32_t got_bits = 0;
+        uint32_t expected_bits = 0;
+        std::memcpy(&got_bits, &got[first], sizeof(got_bits));
+        std::memcpy(&expected_bits, &expected[first], sizeof(expected_bits));
+        std::fprintf(stderr,
+                     "  first=%zu got=%g/0x%08x expected=%g/0x%08x\n",
+                     first, got[first], got_bits, expected[first], expected_bits);
+    }
+    return mismatches == 0;
+}
+
+bool write_tensor(ds4_gpu_tensor *tensor, const std::vector<float> &values) {
+    return tensor && ds4_gpu_tensor_write(
+        tensor, 0, values.data(), values.size() * sizeof(float)) != 0;
+}
+
+bool read_tensor(const ds4_gpu_tensor *tensor, std::vector<float> *values) {
+    return tensor && ds4_gpu_tensor_read(
+        tensor, 0, values->data(), values->size() * sizeof(float)) != 0;
+}
+
+std::vector<float> sentinel_values(size_t count) {
+    std::vector<float> result(count);
+    for (size_t i = 0; i < count; i++) {
+        const uint32_t bits = 0x4b000000u + (uint32_t)i;
+        std::memcpy(&result[i], &bits, sizeof(bits));
+    }
+    return result;
+}
+
+bool run_dense_case(const aligned_model &model, uint32_t n_tokens,
+                    uint64_t offset, uint32_t out_dim, const char *label) {
+    std::vector<float> x;
+    fill_activation(&x, n_tokens);
+    tensor_owner x_gpu(x.size() * sizeof(float));
+    tensor_owner out_gpu((uint64_t)n_tokens * out_dim * sizeof(float));
+    if (!x_gpu.ptr || !out_gpu.ptr || !write_tensor(x_gpu.ptr, x)) {
+        std::fprintf(stderr, "%s: tensor allocation/write FAIL\n", label);
+        return false;
+    }
+    const int rc = ds4_gpu_matmul_quant_tensor(
+        out_gpu.ptr, model.data, model.size, offset, kQ4Type,
+        kK, out_dim, x_gpu.ptr, n_tokens);
+    std::vector<float> got((uint64_t)n_tokens * out_dim);
+    if (rc == 0 || !read_tensor(out_gpu.ptr, &got)) {
+        std::fprintf(stderr, "%s: dense dispatch rc=%d FAIL\n", label, rc);
+        return false;
+    }
+    const std::vector<float> cpu = dense_reference(
+        model.data + offset, x, out_dim, n_tokens);
+    return close_to_cpu(got, cpu, label);
+}
+
+bool output_guard_unchanged(const std::vector<float> &values,
+                            const std::vector<float> &sentinel,
+                            size_t logical_count,
+                            const char *label);
+
+bool run_pair_case(const aligned_model &model, uint32_t n_tokens,
+                   const char *label, bool reverse_outputs = false,
+                   uint32_t in_dim = kK) {
+    const uint32_t out0_dim = reverse_outputs ? kM1 : kM0;
+    const uint32_t out1_dim = reverse_outputs ? kM0 : kM1;
+    const uint64_t base0_offset = in_dim == kTailK
+                                ? model.tail_k1024_offset
+                                : model.weight0_offset;
+    const uint64_t base1_offset = in_dim == kTailK
+                                ? model.tail_k1024_pair_offset
+                                : model.weight1_offset;
+    const uint64_t weight0_offset = reverse_outputs
+                                  ? base1_offset : base0_offset;
+    const uint64_t weight1_offset = reverse_outputs
+                                  ? base0_offset : base1_offset;
+    const size_t count0 = (size_t)n_tokens * out0_dim;
+    const size_t count1 = (size_t)n_tokens * out1_dim;
+    const std::vector<float> sentinel0 =
+        sentinel_values(count0 + kOutputGuardFloats);
+    const std::vector<float> sentinel1 =
+        sentinel_values(count1 + kOutputGuardFloats);
+    std::vector<float> x;
+    fill_activation(&x, n_tokens, in_dim);
+    tensor_owner x_gpu(x.size() * sizeof(float));
+    tensor_owner dense0(count0 * sizeof(float));
+    tensor_owner dense1(count1 * sizeof(float));
+    tensor_owner pair0(sentinel0.size() * sizeof(float));
+    tensor_owner pair1(sentinel1.size() * sizeof(float));
+    if (!x_gpu.ptr || !dense0.ptr || !dense1.ptr || !pair0.ptr || !pair1.ptr ||
+        !write_tensor(x_gpu.ptr, x) ||
+        !write_tensor(pair0.ptr, sentinel0) ||
+        !write_tensor(pair1.ptr, sentinel1)) {
+        std::fprintf(stderr, "%s: tensor allocation/write FAIL\n", label);
+        return false;
+    }
+    const int dense_rc0 = ds4_gpu_matmul_quant_tensor(
+        dense0.ptr, model.data, model.size, weight0_offset, kQ4Type,
+        in_dim, out0_dim, x_gpu.ptr, n_tokens);
+    const int dense_rc1 = ds4_gpu_matmul_quant_tensor(
+        dense1.ptr, model.data, model.size, weight1_offset, kQ4Type,
+        in_dim, out1_dim, x_gpu.ptr, n_tokens);
+    const int pair_rc = ds4_gpu_matmul_q4_K_pair_tensor(
+        pair0.ptr, pair1.ptr, model.data, model.size,
+        weight0_offset, weight1_offset,
+        in_dim, out0_dim, out1_dim, x_gpu.ptr, n_tokens);
+    std::vector<float> dense0_host(count0);
+    std::vector<float> dense1_host(count1);
+    std::vector<float> pair0_host(sentinel0.size());
+    std::vector<float> pair1_host(sentinel1.size());
+    if (dense_rc0 == 0 || dense_rc1 == 0 || pair_rc <= 0 ||
+        !read_tensor(dense0.ptr, &dense0_host) ||
+        !read_tensor(dense1.ptr, &dense1_host) ||
+        !read_tensor(pair0.ptr, &pair0_host) ||
+        !read_tensor(pair1.ptr, &pair1_host)) {
+        std::fprintf(stderr,
+                     "%s: dispatch/read dense=(%d,%d) pair=%d FAIL\n",
+                     label, dense_rc0, dense_rc1, pair_rc);
+        return false;
+    }
+    const std::vector<float> cpu0 = dense_reference(
+        model.data + weight0_offset, x, out0_dim, n_tokens, in_dim);
+    const std::vector<float> cpu1 = dense_reference(
+        model.data + weight1_offset, x, out1_dim, n_tokens, in_dim);
+    bool ok = close_to_cpu(dense0_host, cpu0, "pair control dense0 vs CPU");
+    ok = close_to_cpu(dense1_host, cpu1, "pair control dense1 vs CPU") && ok;
+    ok = output_guard_unchanged(pair0_host, sentinel0, count0,
+                                "pair0 output canary") && ok;
+    ok = output_guard_unchanged(pair1_host, sentinel1, count1,
+                                "pair1 output canary") && ok;
+    pair0_host.resize(count0);
+    pair1_host.resize(count1);
+    ok = bitwise_equal(pair0_host, dense0_host, "pair0 vs standalone dense0") && ok;
+    ok = bitwise_equal(pair1_host, dense1_host, "pair1 vs standalone dense1") && ok;
+    std::fprintf(stderr, "%s: %s\n", label, ok ? "PASS" : "FAIL");
+    return ok;
+}
+
+bool unchanged_after_rejected_call(ds4_gpu_tensor *tensor,
+                                   const std::vector<float> &sentinel,
+                                   const char *label) {
+    std::vector<float> after(sentinel.size());
+    if (!read_tensor(tensor, &after)) {
+        std::fprintf(stderr, "%s: readback FAIL\n", label);
+        return false;
+    }
+    return bitwise_equal(after, sentinel, label);
+}
+
+bool output_guard_unchanged(const std::vector<float> &values,
+                            const std::vector<float> &sentinel,
+                            size_t logical_count,
+                            const char *label) {
+    if (values.size() != sentinel.size() ||
+        logical_count > values.size()) {
+        std::fprintf(stderr, "%s: invalid guard geometry FAIL\n", label);
+        return false;
+    }
+    uint64_t mismatches = 0;
+    size_t first = logical_count;
+    for (size_t i = logical_count; i < values.size(); i++) {
+        if (std::memcmp(&values[i], &sentinel[i], sizeof(float)) != 0) {
+            if (mismatches == 0) first = i;
+            mismatches++;
+        }
+    }
+    std::fprintf(stderr, "%s: mismatches=%llu/%zu %s\n",
+                 label, (unsigned long long)mismatches,
+                 values.size() - logical_count,
+                 mismatches == 0 ? "PASS" : "FAIL");
+    if (mismatches != 0) {
+        std::fprintf(stderr, "  first guard overwrite at float %zu\n", first);
+    }
+    return mismatches == 0;
+}
+
+bool output_body_overwritten(const std::vector<float> &values,
+                             const std::vector<float> &sentinel,
+                             size_t logical_count,
+                             const char *label) {
+    if (values.size() != sentinel.size() || logical_count > values.size()) {
+        std::fprintf(stderr, "%s: invalid body geometry FAIL\n", label);
+        return false;
+    }
+    uint64_t unchanged = 0;
+    size_t first = logical_count;
+    for (size_t i = 0; i < logical_count; i++) {
+        if (std::memcmp(&values[i], &sentinel[i], sizeof(float)) == 0) {
+            if (unchanged == 0) first = i;
+            unchanged++;
+        }
+    }
+    std::fprintf(stderr, "%s: unchanged=%llu/%zu %s\n",
+                 label, (unsigned long long)unchanged, logical_count,
+                 unchanged == 0 ? "PASS" : "FAIL");
+    if (unchanged != 0) {
+        std::fprintf(stderr, "  first unwritten output at float %zu\n", first);
+    }
+    return unchanged == 0;
+}
+
+bool output_body_finite(const std::vector<float> &values,
+                        size_t logical_count,
+                        const char *label) {
+    if (logical_count > values.size()) {
+        std::fprintf(stderr, "%s: invalid body geometry FAIL\n", label);
+        return false;
+    }
+    uint64_t nonfinite = 0u;
+    size_t first = logical_count;
+    for (size_t i = 0; i < logical_count; i++) {
+        if (!std::isfinite(values[i])) {
+            if (nonfinite == 0u) first = i;
+            nonfinite++;
+        }
+    }
+    std::fprintf(stderr, "%s: nonfinite=%llu/%zu %s\n",
+                 label, (unsigned long long)nonfinite, logical_count,
+                 nonfinite == 0u ? "PASS" : "FAIL");
+    if (nonfinite != 0u) {
+        std::fprintf(stderr, "  first non-finite output at float %zu\n", first);
+    }
+    return nonfinite == 0u;
+}
+
+bool run_prefill_parity_case(const aligned_model &model, uint32_t n_tokens,
+                             uint64_t offset, uint32_t out_dim,
+                             bool compare_cpu, const char *label,
+                             uint32_t in_dim = kK) {
+    std::vector<float> x;
+    fill_activation(&x, n_tokens, in_dim);
+    const size_t logical_count = (size_t)n_tokens * out_dim;
+    const size_t allocation_count = logical_count + kOutputGuardFloats;
+    const std::vector<float> sentinel = sentinel_values(allocation_count);
+
+    tensor_owner x_gpu(x.size() * sizeof(float));
+    tensor_owner legacy_gpu(allocation_count * sizeof(float));
+    tensor_owner candidate_gpu(allocation_count * sizeof(float));
+    if (!x_gpu.ptr || !legacy_gpu.ptr || !candidate_gpu.ptr ||
+        !write_tensor(x_gpu.ptr, x) ||
+        !write_tensor(legacy_gpu.ptr, sentinel) ||
+        !write_tensor(candidate_gpu.ptr, sentinel)) {
+        std::fprintf(stderr, "%s: tensor allocation/write FAIL\n", label);
+        return false;
+    }
+
+    env_snapshot enable(kPrefillEnable);
+    env_snapshot disable(kPrefillDisable);
+    env_snapshot require(kPrefillRequire);
+    env_snapshot k1024_tile4_disable(kPrefillK1024Tile4Disable);
+    env_snapshot k1024_tile4_ssd_enable(kPrefillK1024Tile4SsdEnable);
+    env_snapshot k1024_tile4_require(kPrefillK1024Tile4Require);
+    env_snapshot wmma_disable(kPrefillWmmaDisable);
+    const bool require_k1024_tile4 =
+        in_dim == kTailK && out_dim == kQbOutDim;
+
+    // The authoritative rollback remains the reference now that the tiled
+    // path is default-on.
+    (void)unsetenv(kPrefillEnable);
+    (void)setenv(kPrefillDisable, "1", 1);
+    (void)unsetenv(kPrefillRequire);
+    (void)unsetenv(kPrefillK1024Tile4SsdEnable);
+    (void)unsetenv(kPrefillK1024Tile4Require);
+    (void)setenv(kPrefillWmmaDisable, "1", 1);
+    const int legacy_rc = ds4_gpu_matmul_quant_tensor(
+        legacy_gpu.ptr, model.data, model.size, offset, kQ4Type,
+        in_dim, out_dim, x_gpu.ptr, n_tokens);
+
+    // TILE8 is default-on.  Leave the legacy ENABLE unset and use REQUIRE so
+    // a silently ineligible default cannot compare the legacy kernel with
+    // itself.
+    (void)unsetenv(kPrefillEnable);
+    (void)unsetenv(kPrefillDisable);
+    (void)setenv(kPrefillRequire, "1", 1);
+    (void)unsetenv(kPrefillK1024Tile4Disable);
+    (void)unsetenv(kPrefillK1024Tile4SsdEnable);
+    (void)setenv(kPrefillWmmaDisable, "1", 1);
+    if (require_k1024_tile4) {
+        (void)setenv(kPrefillK1024Tile4Require, "1", 1);
+    } else {
+        (void)unsetenv(kPrefillK1024Tile4Require);
+    }
+    const int candidate_rc = ds4_gpu_matmul_quant_tensor(
+        candidate_gpu.ptr, model.data, model.size, offset, kQ4Type,
+        in_dim, out_dim, x_gpu.ptr, n_tokens);
+
+    std::vector<float> legacy_all(allocation_count);
+    std::vector<float> candidate_all(allocation_count);
+    if (legacy_rc == 0 || candidate_rc == 0 ||
+        !read_tensor(legacy_gpu.ptr, &legacy_all) ||
+        !read_tensor(candidate_gpu.ptr, &candidate_all)) {
+        std::fprintf(stderr,
+                     "%s: dispatch/read legacy=%d candidate=%d FAIL\n",
+                     label, legacy_rc, candidate_rc);
+        return false;
+    }
+
+    bool ok = output_guard_unchanged(
+        legacy_all, sentinel, logical_count, "prefill legacy output canary");
+    ok = output_guard_unchanged(
+             candidate_all, sentinel, logical_count,
+             "prefill candidate output canary") && ok;
+
+    legacy_all.resize(logical_count);
+    candidate_all.resize(logical_count);
+    ok = bitwise_equal(candidate_all, legacy_all,
+                       "prefill candidate vs forced legacy") && ok;
+    if (compare_cpu) {
+        const std::vector<float> cpu = dense_reference(
+            model.data + offset, x, out_dim, n_tokens, in_dim);
+        ok = close_to_cpu(legacy_all, cpu,
+                          "prefill forced legacy vs CPU") && ok;
+        ok = close_to_cpu(candidate_all, cpu,
+                          "prefill candidate vs CPU") && ok;
+    }
+    std::fprintf(stderr,
+                 "%s: legacy_rc=%d candidate_rc=%d logical=%zu guard=%zu %s\n",
+                 label, legacy_rc, candidate_rc, logical_count,
+                 kOutputGuardFloats, ok ? "PASS" : "FAIL");
+    return ok;
+}
+
+bool run_q_b_f16_null_qhalf_case(const aligned_model &model) {
+    constexpr uint32_t n_tokens = 32u;
+    static_assert(kQbHeads * kQbHeadDim == kQbOutDim,
+                  "q_b test head geometry must cover the projection");
+    const size_t logical_count = (size_t)n_tokens * kQbOutDim;
+    const size_t allocation_count = logical_count + kOutputGuardFloats;
+    const size_t q_half_guard = kOutputGuardFloats;
+    const size_t q_half_count = logical_count + q_half_guard;
+    std::vector<float> x;
+    fill_activation(&x, n_tokens, kTailK);
+    const std::vector<float> sentinel = sentinel_values(allocation_count);
+    const std::vector<uint16_t> q_half_sentinel(q_half_count, 0x7e55u);
+    tensor_owner x_gpu(x.size() * sizeof(float));
+    tensor_owner provided_out(allocation_count * sizeof(float));
+    tensor_owner scratch_out(allocation_count * sizeof(float));
+    tensor_owner reference_out(logical_count * sizeof(float));
+    tensor_owner q_half_gpu(q_half_count * sizeof(uint16_t));
+    if (!x_gpu.ptr || !provided_out.ptr || !scratch_out.ptr ||
+        !reference_out.ptr || !q_half_gpu.ptr ||
+        !write_tensor(x_gpu.ptr, x) ||
+        !write_tensor(provided_out.ptr, sentinel) ||
+        !write_tensor(scratch_out.ptr, sentinel) ||
+        !ds4_gpu_tensor_write(q_half_gpu.ptr, 0, q_half_sentinel.data(),
+                              q_half_sentinel.size() * sizeof(uint16_t))) {
+        std::fprintf(stderr,
+                     "q_b F16 null-q_half: tensor allocation/write FAIL\n");
+        return false;
+    }
+
+    env_snapshot enable(kQbF16Enable);
+    env_snapshot disable(kQbF16Disable);
+    env_snapshot require(kQbF16Require);
+    env_snapshot min_tokens(kQbF16MinTokens);
+    env_snapshot f16_output(kQbF16OutputEnable);
+    (void)setenv(kQbF16Enable, "1", 1);
+    (void)unsetenv(kQbF16Disable);
+    (void)setenv(kQbF16Require, "1", 1);
+    (void)setenv(kQbF16MinTokens, "32", 1);
+    (void)setenv(kQbF16OutputEnable, "1", 1);
+
+    const uint64_t weight_bytes =
+        (uint64_t)kQbOutDim * (kTailK / kQkK) *
+        sizeof(block_q4_K_test);
+    const ds4_gpu_q4_attn_q_b_f16_sidecar_desc desc = {
+        model.q_b_k1024_offset,
+        weight_bytes,
+        kTailK,
+        kQbOutDim,
+        kQ4Type,
+        0u,
+    };
+    uint64_t prepared_bytes = 0;
+    const int prepare_rc = ds4_gpu_prepare_q4_attn_q_b_f16_sidecars(
+        model.data, model.size, &desc, 1u, n_tokens, 0u,
+        &prepared_bytes);
+
+    /* The release default must keep writing C=F32 and must not touch q_half,
+     * even when a large staging tensor is supplied by a test caller. */
+    (void)setenv(kQbF16OutputEnable, "0", 1);
+    int default_rc = 0;
+    if (prepare_rc > 0) {
+        default_rc = ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor(
+            provided_out.ptr, q_half_gpu.ptr, model.data, model.size,
+            model.q_b_k1024_offset, kQ4Type, kTailK, kQbOutDim,
+            x_gpu.ptr, n_tokens, kQbHeads, kQbHeadDim, kQbRot,
+            17u, 0u, false, 10000.0f, 1.0f, 0.0f, 1.0f,
+            32.0f, 1.0f, 1.0e-6f);
+    }
+    std::vector<float> default_host(allocation_count);
+    std::vector<uint16_t> default_half_host(q_half_count);
+    const bool default_read_ok = default_rc > 0 &&
+        read_tensor(provided_out.ptr, &default_host) &&
+        ds4_gpu_tensor_read(q_half_gpu.ptr, 0, default_half_host.data(),
+                            default_half_host.size() * sizeof(uint16_t));
+    bool ok = prepare_rc > 0 && default_read_ok;
+    if (default_read_ok) {
+        ok = output_guard_unchanged(
+                 default_host, sentinel, logical_count,
+                 "q_b default-F32 output canary") && ok;
+        uint64_t untouched_or_nonfinite = 0;
+        for (size_t i = 0; i < logical_count; i++) {
+            if (!std::isfinite(default_host[i]) ||
+                std::memcmp(&default_host[i], &sentinel[i],
+                            sizeof(float)) == 0) {
+                untouched_or_nonfinite++;
+            }
+        }
+        uint64_t half_mismatches = 0;
+        for (size_t i = 0; i < q_half_count; i++) {
+            if (default_half_host[i] != q_half_sentinel[i]) {
+                half_mismatches++;
+            }
+        }
+        std::fprintf(stderr,
+                     "q_b default-F32 writes finite output: failures=%llu/%zu; "
+                     "q_half mismatches=%llu/%zu %s\n",
+                     (unsigned long long)untouched_or_nonfinite,
+                     logical_count,
+                     (unsigned long long)half_mismatches, q_half_count,
+                     untouched_or_nonfinite == 0 && half_mismatches == 0
+                         ? "PASS" : "FAIL");
+        ok = untouched_or_nonfinite == 0 && half_mismatches == 0 && ok;
+    }
+
+    (void)setenv(kQbF16OutputEnable, "1", 1);
+    if (!write_tensor(provided_out.ptr, sentinel) ||
+        !ds4_gpu_tensor_write(q_half_gpu.ptr, 0, q_half_sentinel.data(),
+                              q_half_sentinel.size() * sizeof(uint16_t))) {
+        ok = false;
+    }
+    int provided_rc = 0;
+    if (prepare_rc > 0) {
+        provided_rc = ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor(
+            provided_out.ptr, q_half_gpu.ptr, model.data, model.size,
+            model.q_b_k1024_offset, kQ4Type, kTailK, kQbOutDim,
+            x_gpu.ptr, n_tokens, kQbHeads, kQbHeadDim, kQbRot,
+            17u, 0u, false, 10000.0f, 1.0f, 0.0f, 1.0f,
+            32.0f, 1.0f, 1.0e-6f);
+    }
+
+    std::vector<float> provided_host(allocation_count);
+    std::vector<uint16_t> q_half_host(q_half_count);
+    const bool provided_read_ok = provided_rc > 0 &&
+        read_tensor(provided_out.ptr, &provided_host) &&
+        ds4_gpu_tensor_read(q_half_gpu.ptr, 0, q_half_host.data(),
+                            q_half_host.size() * sizeof(uint16_t));
+    ok = provided_read_ok && ok;
+    if (provided_read_ok) {
+        ok = output_guard_unchanged(
+                 provided_host, sentinel, logical_count,
+                 "q_b F16 provided-q_half output canary") && ok;
+        uint64_t nonfinite = 0;
+        for (size_t i = 0; i < logical_count; i++) {
+            if (!std::isfinite(provided_host[i])) nonfinite++;
+        }
+        uint64_t half_guard_mismatches = 0;
+        for (size_t i = logical_count; i < q_half_count; i++) {
+            if (q_half_host[i] != q_half_sentinel[i]) {
+                half_guard_mismatches++;
+            }
+        }
+        std::fprintf(stderr,
+                     "q_b F16 provided-q_half: nonfinite=%llu/%zu; "
+                     "canary mismatches=%llu/%zu %s\n",
+                     (unsigned long long)nonfinite, logical_count,
+                     (unsigned long long)half_guard_mismatches, q_half_guard,
+                     nonfinite == 0 && half_guard_mismatches == 0
+                         ? "PASS" : "FAIL");
+        ok = nonfinite == 0 && half_guard_mismatches == 0 && ok;
+    }
+
+    /* Re-expand the accepted F16 projection exactly and feed the established
+     * F32 epilogue. This is a bitwise oracle for the fused half-input tail,
+     * with a nonzero production-shape projection rather than a zero smoke test. */
+    int reference_rc = 0;
+    std::vector<float> reference_input(logical_count);
+    if (provided_read_ok) {
+        for (size_t i = 0; i < logical_count; i++) {
+            reference_input[i] = fp16_to_float(q_half_host[i]);
+        }
+        if (write_tensor(reference_out.ptr, reference_input)) {
+            reference_rc = ds4_gpu_head_rms_norm_rope_tail_tensor(
+                reference_out.ptr, n_tokens, kQbHeads, kQbHeadDim, kQbRot,
+                17u, 0u, false, 10000.0f, 1.0f, 0.0f, 1.0f,
+                32.0f, 1.0f, 1.0e-6f);
+        }
+    }
+    std::vector<float> reference_host(logical_count);
+    if (reference_rc > 0 && read_tensor(reference_out.ptr, &reference_host)) {
+        provided_host.resize(logical_count);
+        ok = bitwise_equal(provided_host, reference_host,
+                           "q_b F16 fused tail vs expanded-F16 F32 tail") && ok;
+    } else {
+        std::fprintf(stderr,
+                     "q_b F16 expanded-F16 epilogue reference rc=%d FAIL\n",
+                     reference_rc);
+        ok = false;
+    }
+
+    /* Non-Apple graphs pass NULL q_half. The backend-owned Q_F16 region must
+     * produce the exact same fused result as an explicit staging tensor. */
+    int scratch_rc = 0;
+    if (prepare_rc > 0) {
+        scratch_rc = ds4_gpu_attn_q_b_f16_head_rms_rope_tail_tensor(
+            scratch_out.ptr, nullptr, model.data, model.size,
+            model.q_b_k1024_offset, kQ4Type, kTailK, kQbOutDim,
+            x_gpu.ptr, n_tokens, kQbHeads, kQbHeadDim, kQbRot,
+            17u, 0u, false, 10000.0f, 1.0f, 0.0f, 1.0f,
+            32.0f, 1.0f, 1.0e-6f);
+    }
+    std::vector<float> scratch_host(allocation_count);
+    if (scratch_rc > 0 && read_tensor(scratch_out.ptr, &scratch_host)) {
+        ok = output_guard_unchanged(
+                 scratch_host, sentinel, logical_count,
+                 "q_b F16 null-q_half output canary") && ok;
+        scratch_host.resize(logical_count);
+        if (provided_read_ok) {
+            ok = bitwise_equal(scratch_host, provided_host,
+                               "q_b F16 null vs provided q_half") && ok;
+        } else {
+            ok = false;
+        }
+    } else {
+        std::fprintf(stderr,
+                     "q_b F16 null-q_half dispatch/read rc=%d FAIL\n",
+                     scratch_rc);
+        ok = false;
+    }
+
+    const int release_rc = ds4_gpu_release_q4_attn_q_b_f16_sidecars();
+    ok = release_rc != 0 && ok;
+    std::fprintf(stderr,
+                 "q_b F16 q_half staging: prepare=%d default=%d provided=%d "
+                 "scratch=%d "
+                 "prepared=%.2f MiB release=%d %s\n",
+                 prepare_rc, default_rc, provided_rc, scratch_rc,
+                 (double)prepared_bytes / 1048576.0, release_rc,
+                 ok ? "PASS" : "FAIL");
+    return ok;
+}
+
+bool run_prefill_wmma_row_tile_policy_oracle() {
+    struct row_tile_case {
+        const char *label;
+        const char *value;
+        uint32_t out_dim;
+        uint32_t expected;
+    };
+    const row_tile_case cases[] = {
+        {"automatic small", nullptr, 1023u, 64u},
+        {"automatic normal lower boundary", nullptr, 1024u, 128u},
+        {"automatic normal upper boundary", nullptr, 8191u, 128u},
+        {"automatic large", nullptr, 8192u, 256u},
+        {"explicit 64", "64", 32768u, 64u},
+        {"explicit 128", "128", 32768u, 128u},
+        {"explicit 256", "256", 1024u, 256u},
+        {"non-whitelisted integer", "257", 1024u, 128u},
+        {"negative integer", "-1", 1024u, 128u},
+        {"negative wrapped whitelist", "-18446744073709551552", 1024u,
+         128u},
+        {"malformed", "128x", 1024u, 128u},
+        {"empty", "", 8192u, 256u},
+    };
+
+    env_snapshot row_tile(kPrefillWmmaRowTile);
+    bool ok = true;
+    for (const row_tile_case &test : cases) {
+        const int env_rc = test.value
+            ? setenv(kPrefillWmmaRowTile, test.value, 1)
+            : unsetenv(kPrefillWmmaRowTile);
+        const uint32_t got = env_rc == 0
+            ? ds4_rocm_test_q4_prefill_wmma_row_tile(test.out_dim)
+            : 0u;
+        if (env_rc != 0 || got != test.expected) {
+            std::fprintf(stderr,
+                         "Q4 WMMA row-tile policy %s: expected=%u got=%u "
+                         "env_rc=%d FAIL\n",
+                         test.label, test.expected, got, env_rc);
+            ok = false;
+        }
+    }
+    std::fprintf(stderr, "Q4 WMMA row-tile policy: %s\n",
+                 ok ? "PASS" : "FAIL");
+    return ok;
+}
+
+bool run_prefill_wmma_requested_policy_oracle() {
+    struct policy_case {
+        const char *label;
+        int ssd_streaming;
+        int enabled;
+        int ssd_enabled;
+        int disabled;
+        int required;
+        int standalone_expected;
+        int attention_a_expected;
+        int attention_b_expected;
+    };
+    const policy_case cases[] = {
+        {"resident unset keeps A automatic", 0, -1, 0, 0, 0, 1, 1, 0},
+        {"resident ENABLE=0 compatibility opt-out", 0, 0, 0, 0, 0,
+         0, 0, 0},
+        {"resident ENABLE=1 opts attention A only", 0, 1, 0, 0, 0,
+         1, 1, 0},
+        {"resident DISABLE opts out", 0, -1, 0, 1, 0, 0, 0, 0},
+        {"resident REQUIRE requests both attention stages", 0, 0, 0, 0, 1,
+         1, 1, 1},
+        {"DISABLE+REQUIRE reaches strict rejection", 0, -1, 0, 1, 1,
+         1, 1, 1},
+        {"SSD default stays conservative", 1, -1, 0, 0, 0, 0, 0, 0},
+        {"generic ENABLE does not bypass SSD gate", 1, 1, 0, 0, 0,
+         0, 0, 0},
+        {"SSD gate opts attention A only", 1, -1, 1, 0, 0, 1, 1, 0},
+        {"SSD DISABLE opts out", 1, -1, 1, 1, 0, 0, 0, 0},
+        {"SSD REQUIRE requests both attention stages", 1, -1, 0, 0, 1,
+         1, 1, 1},
+    };
+
+    bool ok = true;
+    for (const policy_case &test : cases) {
+        const int got = ds4_rocm_test_q4_prefill_wmma_requested_policy(
+            test.ssd_streaming, test.enabled, test.ssd_enabled,
+            test.disabled, test.required);
+        if (got != test.standalone_expected) {
+            std::fprintf(stderr,
+                         "Q4 standalone WMMA request policy %s: "
+                         "expected=%d got=%d FAIL\n",
+                         test.label, test.standalone_expected, got);
+            ok = false;
+        }
+        const int a_got =
+            ds4_rocm_test_q4_prefill_wmma_attention_a_requested_policy(
+                test.ssd_streaming, test.enabled, test.ssd_enabled,
+                test.disabled, test.required);
+        if (a_got != test.attention_a_expected) {
+            std::fprintf(stderr,
+                         "Q4 attention-output A WMMA policy %s: "
+                         "expected=%d got=%d FAIL\n",
+                         test.label, test.attention_a_expected, a_got);
+            ok = false;
+        }
+        const int b_got =
+            ds4_rocm_test_q4_prefill_wmma_attention_b_requested_policy(
+                test.ssd_streaming, test.enabled, test.ssd_enabled,
+                test.disabled, test.required);
+        if (b_got != test.attention_b_expected) {
+            std::fprintf(stderr,
+                         "Q4 attention-output B WMMA policy %s: "
+                         "expected=%d got=%d FAIL\n",
+                         test.label, test.attention_b_expected, b_got);
+            ok = false;
+        }
+    }
+    const int optional_q8_yield =
+        ds4_rocm_test_q4_prefill_wmma_yields_to_q8_wave32(1, 0);
+    const int absent_q8_yield =
+        ds4_rocm_test_q4_prefill_wmma_yields_to_q8_wave32(0, 0);
+    const int strict_wmma_yield =
+        ds4_rocm_test_q4_prefill_wmma_yields_to_q8_wave32(1, 1);
+    if (optional_q8_yield != 1 || absent_q8_yield != 0 ||
+        strict_wmma_yield != 0) {
+        std::fprintf(stderr,
+                     "Q4 WMMA/Q8 precedence: optional=%d absent=%d "
+                     "strict=%d FAIL\n",
+                     optional_q8_yield, absent_q8_yield,
+                     strict_wmma_yield);
+        ok = false;
+    }
+    const int k64_unset =
+        ds4_rocm_test_q4_prefill_wmma_k64_control_policy(-1);
+    const int k64_false =
+        ds4_rocm_test_q4_prefill_wmma_k64_control_policy(0);
+    const int k64_true =
+        ds4_rocm_test_q4_prefill_wmma_k64_control_policy(1);
+    if (k64_unset != 1 || k64_false != 0 || k64_true != 1) {
+        std::fprintf(stderr,
+                     "Q4 WMMA K64 default policy: unset=%d false=%d "
+                     "true=%d FAIL\n",
+                     k64_unset, k64_false, k64_true);
+        ok = false;
+    }
+    const int k128_unset =
+        ds4_rocm_test_q4_prefill_wmma_k128_policy(
+            -1, 1, 256u, 1);
+    const int k128_disabled =
+        ds4_rocm_test_q4_prefill_wmma_k128_policy(
+            1, 1, 256u, 1);
+    const int k128_false =
+        ds4_rocm_test_q4_prefill_wmma_k128_policy(
+            0, 1, 256u, 1);
+    const int k128_k32_rollback =
+        ds4_rocm_test_q4_prefill_wmma_k128_policy(
+            -1, 0, 256u, 1);
+    const int k128_rows128 =
+        ds4_rocm_test_q4_prefill_wmma_k128_policy(
+            -1, 1, 128u, 1);
+    const int k128_unaligned =
+        ds4_rocm_test_q4_prefill_wmma_k128_policy(
+            -1, 1, 256u, 0);
+    if (k128_unset != 1 || k128_disabled != 0 || k128_false != 1 ||
+        k128_k32_rollback != 0 || k128_rows128 != 0 ||
+        k128_unaligned != 0) {
+        std::fprintf(stderr,
+                     "Q4 WMMA K128 opt-out policy: unset=%d disabled=%d "
+                     "false=%d k32=%d rows128=%d unaligned=%d FAIL\n",
+                     k128_unset, k128_disabled, k128_false,
+                     k128_k32_rollback, k128_rows128, k128_unaligned);
+        ok = false;
+    }
+    std::fprintf(stderr,
+                 "ROCm Q4 WMMA request policy oracle: cases=%zu "
+                 "q8_yield=%d/%d/%d k64=%d/%d/%d "
+                 "k128=%d/%d/%d/%d/%d/%d %s\n",
+                 sizeof(cases) / sizeof(cases[0]), optional_q8_yield,
+                 absent_q8_yield, strict_wmma_yield,
+                 k64_unset, k64_false, k64_true,
+                 k128_unset, k128_disabled, k128_false,
+                 k128_k32_rollback, k128_rows128, k128_unaligned,
+                 ok ? "PASS" : "FAIL");
+    return ok;
+}
+
+bool run_prefill_k1024_tile4_policy_oracle() {
+    struct policy_case {
+        const char *label;
+        int ssd_streaming;
+        int device_resident;
+        int ssd_enabled;
+        int disabled;
+        int required;
+        int expected;
+    };
+    const policy_case cases[] = {
+        {"resident default remains automatic", 0, 0, 0, 0, 0, 1},
+        {"resident rollback", 0, 1, 0, 1, 0, 0},
+        {"resident DISABLE dominates REQUIRE", 0, 1, 1, 1, 1, -1},
+        {"SSD default stays conservative", 1, 1, 0, 0, 0, 0},
+        {"SSD opt-in rejects a nonresident range", 1, 0, 1, 0, 0, 0},
+        {"SSD opt-in accepts a resident range", 1, 1, 1, 0, 0, 1},
+        {"SSD REQUIRE rejects a nonresident range", 1, 0, 0, 0, 1, -1},
+        {"SSD REQUIRE requests a resident range", 1, 1, 0, 0, 1, 1},
+        {"SSD DISABLE dominates ENABLE", 1, 1, 1, 1, 0, 0},
+        {"SSD DISABLE dominates REQUIRE", 1, 1, 1, 1, 1, -1},
+    };
+
+    bool ok = true;
+    for (const policy_case &test : cases) {
+        const int got = ds4_rocm_test_q4_prefill_k1024_tile4_policy(
+            test.ssd_streaming, test.device_resident, test.ssd_enabled,
+            test.disabled, test.required);
+        if (got != test.expected) {
+            std::fprintf(stderr,
+                         "K1024 TILE4 policy %s: expected=%d got=%d FAIL\n",
+                         test.label, test.expected, got);
+            ok = false;
+        }
+    }
+    std::fprintf(stderr,
+                 "ROCm Q4 K1024 TILE4 policy oracle: cases=%zu %s\n",
+                 sizeof(cases) / sizeof(cases[0]), ok ? "PASS" : "FAIL");
+    return ok;
+}
+
+bool run_prefill_q8_wave32_policy_oracle() {
+    struct policy_case {
+        const char *label;
+        int scope;
+        int compatible;
+        int enabled;
+        int disabled;
+        int required;
+        int expected;
+    };
+    const policy_case cases[] = {
+        {"default remains canonical", 1, 1, 0, 0, 0, 0},
+        {"ENABLE selects compatible prefill", 1, 1, 1, 0, 0, 1},
+        {"ENABLE falls back outside prefill", 0, 1, 1, 0, 0, 0},
+        {"ENABLE falls back on incompatible runtime", 1, 0, 1, 0, 0, 0},
+        {"DISABLE dominates ENABLE", 1, 1, 1, 1, 0, 0},
+        {"REQUIRE is an opt-in", 1, 1, 0, 0, 1, 1},
+        {"REQUIRE rejects non-prefill", 0, 1, 0, 0, 1, -1},
+        {"REQUIRE rejects incompatible runtime", 1, 0, 0, 0, 1, -1},
+        {"DISABLE dominates REQUIRE", 1, 1, 1, 1, 1, -1},
+    };
+    bool ok = true;
+    for (const policy_case &test : cases) {
+        const int got = ds4_rocm_test_q4_prefill_q8_wave32_policy(
+            test.scope, test.compatible, test.enabled, test.disabled,
+            test.required);
+        if (got != test.expected) {
+            std::fprintf(stderr,
+                         "Q8_K wave32 policy %s: expected=%d got=%d FAIL\n",
+                         test.label, test.expected, got);
+            ok = false;
+        }
+    }
+    struct q_b_policy_case {
+        const char *label;
+        uint32_t weight_type;
+        uint32_t n_tok;
+        int required;
+        int f16_required;
+        int expected;
+    };
+    const q_b_policy_case q_b_cases[] = {
+        {"Q4 prefill REQUIRE yields", kQ4Type, 32u, 1, 0, 1},
+        {"Q4 long prefill REQUIRE yields", kQ4Type, 4097u, 1, 0, 1},
+        {"dual REQUIRE conflicts", kQ4Type, 32u, 1, 1, -1},
+        {"Q4 decode does not claim prefill", kQ4Type, 8u, 1, 1, 0},
+        {"Q8 prefill is unrelated", kQ8Type, 32u, 1, 1, 0},
+        {"Q4 prefill without Q8 REQUIRE keeps F16 policy",
+         kQ4Type, 32u, 0, 1, 0},
+    };
+    for (const q_b_policy_case &test : q_b_cases) {
+        const int got =
+            ds4_rocm_test_q4_attn_q_b_yield_to_q8_wave32_policy(
+                test.weight_type, test.n_tok, test.required,
+                test.f16_required);
+        if (got != test.expected) {
+            std::fprintf(stderr,
+                         "Q8_K wave32 q_b yield policy %s: "
+                         "expected=%d got=%d FAIL\n",
+                         test.label, test.expected, got);
+            ok = false;
+        }
+    }
+    std::fprintf(stderr,
+                 "ROCm Q4 Q8_K wave32 policy oracle: selector=%zu q_b=%zu "
+                 "%s\n",
+                 sizeof(cases) / sizeof(cases[0]),
+                 sizeof(q_b_cases) / sizeof(q_b_cases[0]),
+                 ok ? "PASS" : "FAIL");
+    return ok;
+}
+
+bool run_pair_pre_enqueue_policy_oracle() {
+    struct policy_case {
+        const char *label;
+        int prefill_scope;
+        int tile8_required;
+        int q8_wave32_required;
+        int expected;
+    };
+    const policy_case cases[] = {
+        {"optional prefill rejection falls back", 1, 0, 0, 0},
+        {"TILE8 REQUIRE rejection fails closed", 1, 1, 0, -1},
+        {"Q8 wave32 REQUIRE can use dense fallback", 1, 0, 1, 0},
+        {"TILE8 remains strict with dual REQUIRE", 1, 1, 1, -1},
+        {"decode ignores prefill TILE8 REQUIRE", 0, 1, 0, 0},
+        {"optional decode rejection falls back", 0, 0, 0, 0},
+    };
+
+    bool ok = true;
+    for (const policy_case &test : cases) {
+        const int got = ds4_rocm_test_q4_pair_pre_enqueue_failure_policy(
+            test.prefill_scope, test.tile8_required,
+            test.q8_wave32_required);
+        if (got != test.expected) {
+            std::fprintf(stderr,
+                         "Q4 pair pre-enqueue policy %s: "
+                         "expected=%d got=%d FAIL\n",
+                         test.label, test.expected, got);
+            ok = false;
+        }
+    }
+
+    /* Exercise the public selector without a GPU: null tensors reach the
+     * validation rejection only when the pair is selected. */
+    env_snapshot tile8_disable(kPrefillDisable);
+    env_snapshot tile8_require(kPrefillRequire);
+    env_snapshot dense_pair_enable("DS4_ROCM_ENABLE_Q4_DENSE_PAIR");
+    env_snapshot dense_pair_disable("DS4_ROCM_DISABLE_Q4_DENSE_PAIR");
+    env_snapshot wmma_enable(kPrefillWmmaEnable);
+    env_snapshot wmma_disable(kPrefillWmmaDisable);
+    env_snapshot wmma_require(kPrefillWmmaRequire);
+    env_snapshot wmma_row_tile(kPrefillWmmaRowTile);
+    env_snapshot wmma_k64(kPrefillWmmaK64);
+    env_snapshot q8_enable(kPrefillQ8Wave32Enable);
+    env_snapshot q8_disable(kPrefillQ8Wave32Disable);
+    env_snapshot q8_require(kPrefillQ8Wave32Require);
+    (void)unsetenv(kPrefillDisable);
+    (void)setenv(kPrefillRequire, "1", 1);
+    (void)unsetenv("DS4_ROCM_ENABLE_Q4_DENSE_PAIR");
+    (void)unsetenv("DS4_ROCM_DISABLE_Q4_DENSE_PAIR");
+    (void)unsetenv(kPrefillWmmaEnable);
+    (void)unsetenv(kPrefillWmmaDisable);
+    (void)unsetenv(kPrefillWmmaRequire);
+    (void)unsetenv(kPrefillWmmaRowTile);
+    (void)setenv(kPrefillWmmaK64, "0", 1);
+    (void)unsetenv(kPrefillQ8Wave32Enable);
+    (void)unsetenv(kPrefillQ8Wave32Disable);
+    (void)unsetenv(kPrefillQ8Wave32Require);
+    const int required_validation = ds4_gpu_matmul_q4_K_pair_tensor(
+        nullptr, nullptr, nullptr, 0u, 0u, 0u,
+        kK, kM0, kM1, nullptr, 9u);
+
+    (void)unsetenv(kPrefillRequire);
+    (void)setenv(kPrefillDisable, "1", 1);
+    const int opt_out = ds4_gpu_matmul_q4_K_pair_tensor(
+        nullptr, nullptr, nullptr, 0u, 0u, 0u,
+        kK, kM0, kM1, nullptr, 9u);
+
+    (void)unsetenv(kPrefillDisable);
+    (void)setenv(kPrefillRequire, "1", 1);
+    (void)setenv("DS4_ROCM_ENABLE_Q4_DENSE_PAIR", "1", 1);
+    const int decode = ds4_gpu_matmul_q4_K_pair_tensor(
+        nullptr, nullptr, nullptr, 0u, 0u, 0u,
+        kK, kM0, kM1, nullptr, 1u);
+    if (required_validation != -1 || opt_out != 0 || decode != 0) {
+        std::fprintf(stderr,
+                     "Q4 pair public pre-enqueue policy: "
+                     "required=%d opt_out=%d decode=%d FAIL\n",
+                     required_validation, opt_out, decode);
+        ok = false;
+    }
+    std::fprintf(stderr,
+                 "ROCm Q4 pair pre-enqueue policy oracle: cases=%zu "
+                 "required=%d opt_out=%d decode=%d %s\n",
+                 sizeof(cases) / sizeof(cases[0]), required_validation,
+                 opt_out, decode, ok ? "PASS" : "FAIL");
+    return ok;
+}
+
+bool run_prefill_q8_wave32_oracle(const aligned_model &model) {
+#if DS4_TEST_HAS_HIP_RUNTIME
+    int device = 0;
+    hipDeviceProp_t properties = {};
+    if (hipGetDevice(&device) != hipSuccess ||
+        hipGetDeviceProperties(&properties, device) != hipSuccess ||
+        properties.warpSize != 32 ||
+        std::strncmp(properties.gcnArchName, "gfx1151", 7u) != 0) {
+        std::fprintf(stderr,
+                     "ROCm Q4 Q8_K wave32 oracle: SKIP "
+                     "(requires gfx1151 wave32)\n");
+        return true;
+    }
+
+    bool ok = true;
+    constexpr uint32_t n_tokens = 9u;
+    const uint32_t dimensions[] = {256u, 1024u, 4096u};
+    for (uint32_t in_dim : dimensions) {
+        std::vector<float> x;
+        fill_q8_wave32_activation(&x, n_tokens, in_dim);
+        const size_t block_count =
+            (size_t)n_tokens * (in_dim / kQkK);
+        const size_t bytes = block_count * sizeof(block_q8_K_test);
+        tensor_owner x_gpu(x.size() * sizeof(float));
+        tensor_owner legacy_gpu(bytes);
+        tensor_owner wave_gpu(bytes);
+        if (!x_gpu.ptr || !legacy_gpu.ptr || !wave_gpu.ptr ||
+            !write_tensor(x_gpu.ptr, x)) {
+            std::fprintf(stderr,
+                         "Q8_K wave32 raw K=%u allocation/write FAIL\n",
+                         in_dim);
+            ok = false;
+            continue;
+        }
+        const int legacy_rc = ds4_rocm_test_q8_K_quantize_tensor(
+            legacy_gpu.ptr, x_gpu.ptr, in_dim, n_tokens, 0);
+        const int wave_rc = ds4_rocm_test_q8_K_quantize_tensor(
+            wave_gpu.ptr, x_gpu.ptr, in_dim, n_tokens, 1);
+        std::vector<block_q8_K_test> legacy(block_count);
+        std::vector<block_q8_K_test> wave(block_count);
+        std::vector<block_q8_K_test> cpu(block_count);
+        const bool read_ok = legacy_rc != 0 && wave_rc != 0 &&
+            ds4_gpu_tensor_read(legacy_gpu.ptr, 0, legacy.data(), bytes) != 0 &&
+            ds4_gpu_tensor_read(wave_gpu.ptr, 0, wave.data(), bytes) != 0;
+        for (size_t i = 0; i < block_count; i++) {
+            quantize_q8_K_cpu(x.data() + i * kQkK, &cpu[i]);
+        }
+        const bool pair_equal = read_ok &&
+            std::memcmp(legacy.data(), wave.data(), bytes) == 0;
+        const bool cpu_equal = read_ok &&
+            std::memcmp(legacy.data(), cpu.data(), bytes) == 0;
+        std::fprintf(stderr,
+                     "Q8_K wave32 raw K=%u blocks=%zu legacy=%d wave=%d "
+                     "pair=%s cpu=%s %s\n",
+                     in_dim, block_count, legacy_rc, wave_rc,
+                     pair_equal ? "bitwise" : "MISMATCH",
+                     cpu_equal ? "bitwise" : "MISMATCH",
+                     pair_equal && cpu_equal ? "PASS" : "FAIL");
+        ok = pair_equal && cpu_equal && ok;
+    }
+
+    env_snapshot tile8_disable(kPrefillDisable);
+    env_snapshot tile8_require(kPrefillRequire);
+    env_snapshot wmma_disable(kPrefillWmmaDisable);
+    env_snapshot q8_enable(kPrefillQ8Wave32Enable);
+    env_snapshot q8_disable(kPrefillQ8Wave32Disable);
+    env_snapshot q8_require(kPrefillQ8Wave32Require);
+    (void)unsetenv(kPrefillDisable);
+    (void)setenv(kPrefillRequire, "1", 1);
+    (void)setenv(kPrefillWmmaDisable, "1", 1);
+    (void)unsetenv(kPrefillQ8Wave32Enable);
+    (void)unsetenv(kPrefillQ8Wave32Disable);
+    (void)setenv(kPrefillQ8Wave32Require, "1", 1);
+
+    std::vector<float> x;
+    fill_q8_wave32_activation(&x, n_tokens, kK);
+    tensor_owner x_gpu(x.size() * sizeof(float));
+    tensor_owner out_gpu((uint64_t)n_tokens * kM0 * sizeof(float));
+    bool dispatch_ok = x_gpu.ptr && out_gpu.ptr && write_tensor(x_gpu.ptr, x);
+    ds4_rocm_test_q4_prefill_q8_wave32_reset();
+    const int rc = dispatch_ok ? ds4_gpu_matmul_quant_tensor(
+        out_gpu.ptr, model.data, model.size, model.weight0_offset, kQ4Type,
+        kK, kM0, x_gpu.ptr, n_tokens) : 0;
+    const uint64_t calls =
+        ds4_rocm_test_q4_prefill_q8_wave32_get_calls();
+    std::vector<float> got((uint64_t)n_tokens * kM0);
+    dispatch_ok = dispatch_ok && rc != 0 && calls == 1u &&
+                  read_tensor(out_gpu.ptr, &got);
+    if (dispatch_ok) {
+        const std::vector<float> cpu = dense_reference(
+            model.data + model.weight0_offset, x, kM0, n_tokens);
+        dispatch_ok = close_to_cpu(
+            got, cpu, "Q8_K wave32 REQUIRE public dispatch vs CPU");
+    }
+
+    /* Q8 REQUIRE owns only activation quantization.  With TILE8 opted out,
+     * the pair must yield so the graph can issue two dense calls, each of
+     * which still attests the wave32 quantizer. */
+    tensor_owner fallback0((uint64_t)n_tokens * kM0 * sizeof(float));
+    tensor_owner fallback1((uint64_t)n_tokens * kM1 * sizeof(float));
+    (void)setenv(kPrefillDisable, "1", 1);
+    (void)unsetenv(kPrefillRequire);
+    (void)unsetenv(kPrefillQ8Wave32Disable);
+    ds4_rocm_test_q4_prefill_q8_wave32_reset();
+    const int pair_fallback_rc = fallback0.ptr && fallback1.ptr
+        ? ds4_gpu_matmul_q4_K_pair_tensor(
+              fallback0.ptr, fallback1.ptr, model.data, model.size,
+              model.weight0_offset, model.weight1_offset,
+              kK, kM0, kM1, x_gpu.ptr, n_tokens)
+        : -1;
+    const uint64_t pair_fallback_calls =
+        ds4_rocm_test_q4_prefill_q8_wave32_get_calls();
+    const int fallback_rc0 = pair_fallback_rc == 0
+        ? ds4_gpu_matmul_quant_tensor(
+              fallback0.ptr, model.data, model.size, model.weight0_offset,
+              kQ4Type, kK, kM0, x_gpu.ptr, n_tokens)
+        : 0;
+    const int fallback_rc1 = fallback_rc0 != 0
+        ? ds4_gpu_matmul_quant_tensor(
+              fallback1.ptr, model.data, model.size, model.weight1_offset,
+              kQ4Type, kK, kM1, x_gpu.ptr, n_tokens)
+        : 0;
+    const uint64_t dense_fallback_calls =
+        ds4_rocm_test_q4_prefill_q8_wave32_get_calls();
+    const bool pair_fallback = pair_fallback_rc == 0 &&
+        pair_fallback_calls == 0u && fallback_rc0 != 0 &&
+        fallback_rc1 != 0 && dense_fallback_calls == 2u;
+
+    (void)unsetenv(kPrefillDisable);
+    (void)setenv(kPrefillRequire, "1", 1);
+    (void)setenv(kPrefillQ8Wave32Disable, "1", 1);
+    ds4_rocm_test_q4_prefill_q8_wave32_reset();
+    const int rejected_rc = ds4_gpu_matmul_quant_tensor(
+        out_gpu.ptr, model.data, model.size, model.weight0_offset, kQ4Type,
+        kK, kM0, x_gpu.ptr, n_tokens);
+    const uint64_t rejected_calls =
+        ds4_rocm_test_q4_prefill_q8_wave32_get_calls();
+    const bool rejected = rejected_rc == 0 && rejected_calls == 0u;
+    std::fprintf(stderr,
+                 "ROCm Q4 Q8_K wave32 dispatch: rc=%d calls=%llu "
+                 "pair_fallback=%d/%llu dense_fallback=%d,%d/%llu "
+                 "disable+require=%d/%llu %s\n",
+                 rc, (unsigned long long)calls, pair_fallback_rc,
+                 (unsigned long long)pair_fallback_calls,
+                 fallback_rc0, fallback_rc1,
+                 (unsigned long long)dense_fallback_calls,
+                 rejected_rc,
+                 (unsigned long long)rejected_calls,
+                 dispatch_ok && pair_fallback && rejected ? "PASS" : "FAIL");
+    return dispatch_ok && pair_fallback && rejected && ok;
+#else
+    (void)model;
+    return true;
+#endif
+}
+
+bool run_prefill_k1024_tile4_ssd_case(const aligned_model &model) {
+    constexpr uint32_t n_tokens = 9u;
+    const size_t logical_count = (size_t)n_tokens * kQbOutDim;
+    const size_t allocation_count = logical_count + kOutputGuardFloats;
+    const std::vector<float> sentinel = sentinel_values(allocation_count);
+    std::vector<float> x;
+    fill_activation(&x, n_tokens, kTailK);
+    tensor_owner x_gpu(x.size() * sizeof(float));
+    tensor_owner tile8_gpu(allocation_count * sizeof(float));
+    tensor_owner tile4_gpu(allocation_count * sizeof(float));
+    if (!x_gpu.ptr || !tile8_gpu.ptr || !tile4_gpu.ptr ||
+        !write_tensor(x_gpu.ptr, x) ||
+        !write_tensor(tile8_gpu.ptr, sentinel) ||
+        !write_tensor(tile4_gpu.ptr, sentinel)) {
+        std::fprintf(stderr, "prefill K1024 TILE4 SSD: setup FAIL\n");
+        return false;
+    }
+
+    env_snapshot tile8_disable(kPrefillDisable);
+    env_snapshot tile8_require(kPrefillRequire);
+    env_snapshot tile4_enable(kPrefillK1024Tile4SsdEnable);
+    env_snapshot tile4_disable(kPrefillK1024Tile4Disable);
+    env_snapshot tile4_require(kPrefillK1024Tile4Require);
+    env_snapshot cache_limit("DS4_ROCM_STREAM_MODEL_CACHE_GB");
+    (void)unsetenv(kPrefillDisable);
+    (void)setenv(kPrefillRequire, "1", 1);
+    (void)unsetenv(kPrefillK1024Tile4SsdEnable);
+    (void)unsetenv(kPrefillK1024Tile4Disable);
+    (void)unsetenv(kPrefillK1024Tile4Require);
+    (void)setenv("DS4_ROCM_STREAM_MODEL_CACHE_GB", "1", 1);
+
+    FILE *model_file = std::tmpfile();
+    void *ssd_model_map = MAP_FAILED;
+    bool model_map_switched = false;
+    bool ok = model_file != nullptr && model.size <= (uint64_t)SIZE_MAX;
+    if (ok) {
+        ok = std::fwrite(model.data, 1u, (size_t)model.size, model_file) ==
+                 (size_t)model.size &&
+             std::fflush(model_file) == 0;
+    }
+    const int model_fd = ok ? fileno(model_file) : -1;
+    if (ok && model_fd >= 0) {
+        ssd_model_map = mmap(nullptr, (size_t)model.size, PROT_READ,
+                             MAP_PRIVATE, model_fd, 0);
+        ok = ssd_model_map != MAP_FAILED;
+    } else {
+        ok = false;
+    }
+    if (ok) {
+        ok = ds4_gpu_synchronize() != 0 &&
+             ds4_gpu_set_model_map(ssd_model_map, model.size) != 0;
+        model_map_switched = ok;
+    }
+    if (ok) ok = ds4_gpu_set_model_fd(model_fd) != 0;
+
+    int tile8_rc = 0;
+    int tile4_rc = 0;
+    int rejected_rc = 1;
+    if (ok) {
+        /* Switching modes releases prior synthetic resident ranges.  The
+         * default call reloads q_b from the file into the normal SSD device
+         * cache but must retain TILE8 because the new SSD candidate is off. */
+        ds4_gpu_set_ssd_streaming(true);
+        tile8_rc = ds4_gpu_matmul_quant_tensor(
+            tile8_gpu.ptr, ssd_model_map, model.size,
+            model.q_b_k1024_offset,
+            kQ4Type, kTailK, kQbOutDim, x_gpu.ptr, n_tokens);
+
+        /* REQUIRE is also an opt-in.  The second call reuses the already
+         * cached device range and cannot false-green through TILE8. */
+        (void)setenv(kPrefillK1024Tile4SsdEnable, "1", 1);
+        (void)setenv(kPrefillK1024Tile4Require, "1", 1);
+        tile4_rc = ds4_gpu_matmul_quant_tensor(
+            tile4_gpu.ptr, ssd_model_map, model.size,
+            model.q_b_k1024_offset,
+            kQ4Type, kTailK, kQbOutDim, x_gpu.ptr, n_tokens);
+    }
+
+    std::vector<float> tile8_host(allocation_count);
+    std::vector<float> tile4_host(allocation_count);
+    const bool read_ok = tile8_rc != 0 && tile4_rc != 0 &&
+        read_tensor(tile8_gpu.ptr, &tile8_host) &&
+        read_tensor(tile4_gpu.ptr, &tile4_host);
+    bool parity_ok = read_ok;
+    if (read_ok) {
+        parity_ok = output_body_overwritten(
+            tile8_host, sentinel, logical_count,
+            "prefill K1024 TILE8 SSD output body");
+        parity_ok = output_body_overwritten(
+                        tile4_host, sentinel, logical_count,
+                        "prefill K1024 TILE4 SSD output body") &&
+                    parity_ok;
+        parity_ok = output_guard_unchanged(
+            tile8_host, sentinel, logical_count,
+            "prefill K1024 TILE8 SSD output canary");
+        parity_ok = output_guard_unchanged(
+                        tile4_host, sentinel, logical_count,
+                        "prefill K1024 TILE4 SSD output canary") &&
+                    parity_ok;
+        tile8_host.resize(logical_count);
+        tile4_host.resize(logical_count);
+        parity_ok = bitwise_equal(
+                        tile4_host, tile8_host,
+                        "prefill K1024 TILE4 SSD vs TILE8 SSD") &&
+                    parity_ok;
+    }
+
+    if (ok) {
+        (void)setenv(kPrefillK1024Tile4Disable, "1", 1);
+        if (write_tensor(tile4_gpu.ptr, sentinel)) {
+            rejected_rc = ds4_gpu_matmul_quant_tensor(
+                tile4_gpu.ptr, ssd_model_map, model.size,
+                model.q_b_k1024_offset, kQ4Type, kTailK, kQbOutDim,
+                x_gpu.ptr, n_tokens);
+        }
+    }
+    const bool rejected_ok = rejected_rc == 0 &&
+        unchanged_after_rejected_call(
+            tile4_gpu.ptr, sentinel,
+            "prefill K1024 TILE4 SSD DISABLE+REQUIRE preserves output");
+
+    /* Even a failed candidate may follow an accepted baseline launch.  Drain
+     * it before the mode transition releases the backing range cache. */
+    (void)ds4_gpu_synchronize();
+    ds4_gpu_set_ssd_streaming(false);
+    (void)ds4_gpu_set_model_fd(-1);
+    bool model_map_restored = !model_map_switched;
+    if (model_map_switched &&
+        !ds4_gpu_set_model_map(model.data, model.size)) {
+        std::fprintf(stderr,
+                     "prefill K1024 TILE4 SSD: model-map restore FAIL\n");
+        ok = false;
+    } else if (model_map_switched) {
+        model_map_restored = true;
+    }
+    if (ssd_model_map != MAP_FAILED && model_map_restored) {
+        (void)munmap(ssd_model_map, (size_t)model.size);
+    }
+    if (model_file) std::fclose(model_file);
+    ok = ok && parity_ok && rejected_ok;
+    std::fprintf(stderr,
+                 "prefill q_b K1024 TILE4 SSD: tile8=%d tile4=%d "
+                 "rejected=%d %s\n",
+                 tile8_rc, tile4_rc, rejected_rc, ok ? "PASS" : "FAIL");
+    return ok;
+}
+
+bool run_prefill_gate_guards(const aligned_model &model) {
+    constexpr uint32_t n_tokens = 9u;
+    std::vector<float> x;
+    fill_activation(&x, n_tokens);
+    const size_t output_count = (size_t)n_tokens * kM1 + kOutputGuardFloats;
+    const std::vector<float> sentinel = sentinel_values(output_count);
+    tensor_owner x_gpu(x.size() * sizeof(float));
+    tensor_owner out_gpu(output_count * sizeof(float));
+    if (!x_gpu.ptr || !out_gpu.ptr || !write_tensor(x_gpu.ptr, x) ||
+        !write_tensor(out_gpu.ptr, sentinel)) {
+        std::fprintf(stderr, "prefill gate guards: setup FAIL\n");
+        return false;
+    }
+
+    env_snapshot enable(kPrefillEnable);
+    env_snapshot disable(kPrefillDisable);
+    env_snapshot require(kPrefillRequire);
+
+    /* REQUIRE with neither ENABLE nor DISABLE proves that TILE8 is selected
+     * by the default policy. */
+    (void)unsetenv(kPrefillEnable);
+    (void)unsetenv(kPrefillDisable);
+    (void)setenv(kPrefillRequire, "1", 1);
+    const int default_rc = ds4_gpu_matmul_quant_tensor(
+        out_gpu.ptr, model.data, model.size, model.weight1_offset, kQ4Type,
+        kK, kM1, x_gpu.ptr, n_tokens);
+    bool ok = default_rc != 0;
+    if (default_rc == 0) {
+        std::fprintf(stderr,
+                     "prefill default-on REQUIRE: expected success got=%d FAIL\n",
+                     default_rc);
+    }
+    std::vector<float> default_out(output_count);
+    if (!read_tensor(out_gpu.ptr, &default_out)) return false;
+    ok = output_guard_unchanged(
+             default_out, sentinel, (size_t)n_tokens * kM1,
+             "prefill default-on output canary") && ok;
+    if (!write_tensor(out_gpu.ptr, sentinel)) return false;
+
+    (void)setenv(kPrefillEnable, "1", 1);
+    (void)setenv(kPrefillDisable, "1", 1);
+    (void)setenv(kPrefillRequire, "1", 1);
+    const int rc = ds4_gpu_matmul_quant_tensor(
+        out_gpu.ptr, model.data, model.size, model.weight1_offset, kQ4Type,
+        kK, kM1, x_gpu.ptr, n_tokens);
+    ok = rc == 0 && ok;
+    if (rc != 0) {
+        std::fprintf(stderr,
+                     "prefill DISABLE+REQUIRE: expected rc=0 got=%d FAIL\n",
+                     rc);
+    }
+    ok = unchanged_after_rejected_call(
+             out_gpu.ptr, sentinel,
+             "prefill DISABLE dominates REQUIRE and preserves output") && ok;
+    return ok;
+}
+
+bool run_prefill_pair_case(const aligned_model &model, uint32_t n_tokens,
+                           bool reverse_outputs, uint32_t in_dim = kK) {
+    const uint32_t out0_dim = reverse_outputs ? kM1 : kM0;
+    const uint32_t out1_dim = reverse_outputs ? kM0 : kM1;
+    const uint64_t base0_offset = in_dim == kTailK
+                                ? model.tail_k1024_offset
+                                : model.weight0_offset;
+    const uint64_t base1_offset = in_dim == kTailK
+                                ? model.tail_k1024_pair_offset
+                                : model.weight1_offset;
+    const uint64_t weight0_offset = reverse_outputs
+                                  ? base1_offset : base0_offset;
+    const uint64_t weight1_offset = reverse_outputs
+                                  ? base0_offset : base1_offset;
+    std::vector<float> x;
+    fill_activation(&x, n_tokens, in_dim);
+    const size_t count0 = (size_t)n_tokens * out0_dim;
+    const size_t count1 = (size_t)n_tokens * out1_dim;
+    const std::vector<float> sentinel0 =
+        sentinel_values(count0 + kOutputGuardFloats);
+    const std::vector<float> sentinel1 =
+        sentinel_values(count1 + kOutputGuardFloats);
+
+    tensor_owner x_gpu(x.size() * sizeof(float));
+    tensor_owner legacy0(sentinel0.size() * sizeof(float));
+    tensor_owner legacy1(sentinel1.size() * sizeof(float));
+    tensor_owner pair0(sentinel0.size() * sizeof(float));
+    tensor_owner pair1(sentinel1.size() * sizeof(float));
+    if (!x_gpu.ptr || !legacy0.ptr || !legacy1.ptr || !pair0.ptr ||
+        !pair1.ptr || !write_tensor(x_gpu.ptr, x) ||
+        !write_tensor(legacy0.ptr, sentinel0) ||
+        !write_tensor(legacy1.ptr, sentinel1) ||
+        !write_tensor(pair0.ptr, sentinel0) ||
+        !write_tensor(pair1.ptr, sentinel1)) {
+        std::fprintf(stderr, "prefill pair n_tok=%u reverse=%d: setup FAIL\n",
+                     n_tokens, reverse_outputs ? 1 : 0);
+        return false;
+    }
+
+    env_snapshot prefill_enable(kPrefillEnable);
+    env_snapshot prefill_disable(kPrefillDisable);
+    env_snapshot prefill_require(kPrefillRequire);
+    env_snapshot pair_enable("DS4_ROCM_ENABLE_Q4_DENSE_PAIR");
+    env_snapshot pair_disable("DS4_ROCM_DISABLE_Q4_DENSE_PAIR");
+
+    (void)unsetenv(kPrefillEnable);
+    (void)setenv(kPrefillDisable, "1", 1);
+    (void)unsetenv(kPrefillRequire);
+    const int legacy_rc0 = ds4_gpu_matmul_quant_tensor(
+        legacy0.ptr, model.data, model.size, weight0_offset, kQ4Type,
+        in_dim, out0_dim, x_gpu.ptr, n_tokens);
+    const int legacy_rc1 = ds4_gpu_matmul_quant_tensor(
+        legacy1.ptr, model.data, model.size, weight1_offset, kQ4Type,
+        in_dim, out1_dim, x_gpu.ptr, n_tokens);
+
+    // The prefill pair is a distinct path: it must not depend on the legacy
+    // decode-pair opt-in, whose <=8-token behavior is tested separately.
+    (void)unsetenv(kPrefillEnable);
+    (void)unsetenv(kPrefillDisable);
+    (void)setenv(kPrefillRequire, "1", 1);
+    (void)unsetenv("DS4_ROCM_ENABLE_Q4_DENSE_PAIR");
+    (void)unsetenv("DS4_ROCM_DISABLE_Q4_DENSE_PAIR");
+    const int pair_rc = ds4_gpu_matmul_q4_K_pair_tensor(
+        pair0.ptr, pair1.ptr, model.data, model.size,
+        weight0_offset, weight1_offset,
+        in_dim, out0_dim, out1_dim, x_gpu.ptr, n_tokens);
+
+    std::vector<float> legacy0_host(sentinel0.size());
+    std::vector<float> legacy1_host(sentinel1.size());
+    std::vector<float> pair0_host(sentinel0.size());
+    std::vector<float> pair1_host(sentinel1.size());
+    if (legacy_rc0 == 0 || legacy_rc1 == 0 || pair_rc <= 0 ||
+        !read_tensor(legacy0.ptr, &legacy0_host) ||
+        !read_tensor(legacy1.ptr, &legacy1_host) ||
+        !read_tensor(pair0.ptr, &pair0_host) ||
+        !read_tensor(pair1.ptr, &pair1_host)) {
+        std::fprintf(stderr,
+                     "prefill pair n_tok=%u reverse=%d: dispatch/read "
+                     "legacy=(%d,%d) pair=%d FAIL\n",
+                     n_tokens, reverse_outputs ? 1 : 0,
+                     legacy_rc0, legacy_rc1, pair_rc);
+        return false;
+    }
+
+    bool ok = output_guard_unchanged(
+        pair0_host, sentinel0, count0, "prefill pair0 output canary");
+    ok = output_guard_unchanged(
+             pair1_host, sentinel1, count1,
+             "prefill pair1 output canary") && ok;
+    pair0_host.resize(count0);
+    pair1_host.resize(count1);
+    legacy0_host.resize(count0);
+    legacy1_host.resize(count1);
+    ok = bitwise_equal(pair0_host, legacy0_host,
+                       "prefill pair0 vs forced legacy dense0") && ok;
+    ok = bitwise_equal(pair1_host, legacy1_host,
+                       "prefill pair1 vs forced legacy dense1") && ok;
+    std::fprintf(stderr,
+                 "prefill pair K=%u M=(%u,%u) n_tok=%u "
+                 "legacy=(%d,%d) pair=%d %s\n",
+                 in_dim, out0_dim, out1_dim, n_tokens,
+                 legacy_rc0, legacy_rc1, pair_rc, ok ? "PASS" : "FAIL");
+    return ok;
+}
+
+bool run_attention_rowwise_reference(const aligned_model &model,
+                                     const ds4_gpu_tensor *heads,
+                                     ds4_gpu_tensor *low,
+                                     ds4_gpu_tensor *out,
+                                     uint32_t n_tokens,
+                                     uint64_t out_b_offset,
+                                     uint32_t out_b_type) {
+    const uint64_t heads_group_bytes =
+        (uint64_t)kAttnGroupDim * sizeof(float);
+    const uint64_t heads_token_bytes =
+        (uint64_t)kAttnGroups * heads_group_bytes;
+    const uint64_t low_group_bytes =
+        (uint64_t)kAttnRank * sizeof(float);
+    const uint64_t low_token_bytes =
+        (uint64_t)kAttnLowDim * sizeof(float);
+    const uint64_t out_token_bytes =
+        (uint64_t)kAttnOutDim * sizeof(float);
+    const uint64_t row_a_bytes =
+        (kAttnGroupDim / kQkK) * sizeof(block_q4_K_test);
+    const uint64_t group_a_bytes = (uint64_t)kAttnRank * row_a_bytes;
+
+    for (uint32_t token = 0; token < n_tokens; token++) {
+        for (uint32_t group = 0; group < kAttnGroups; group++) {
+            tensor_owner heads_group(ds4_gpu_tensor_view(
+                heads,
+                (uint64_t)token * heads_token_bytes +
+                    (uint64_t)group * heads_group_bytes,
+                heads_group_bytes));
+            tensor_owner low_group(ds4_gpu_tensor_view(
+                low,
+                (uint64_t)token * low_token_bytes +
+                    (uint64_t)group * low_group_bytes,
+                low_group_bytes));
+            if (!heads_group.ptr || !low_group.ptr ||
+                ds4_gpu_matmul_quant_tensor(
+                    low_group.ptr, model.data, model.size,
+                    model.attn_a_offset + (uint64_t)group * group_a_bytes,
+                    kQ4Type, kAttnGroupDim, kAttnRank,
+                    heads_group.ptr, 1u) == 0) {
+                std::fprintf(stderr,
+                             "attention row reference A token=%u group=%u FAIL\n",
+                             token, group);
+                return false;
+            }
+        }
+        tensor_owner low_row(ds4_gpu_tensor_view(
+            low, (uint64_t)token * low_token_bytes, low_token_bytes));
+        tensor_owner out_row(ds4_gpu_tensor_view(
+            out, (uint64_t)token * out_token_bytes, out_token_bytes));
+        if (!low_row.ptr || !out_row.ptr ||
+            ds4_gpu_matmul_quant_tensor(
+                out_row.ptr, model.data, model.size, out_b_offset,
+                out_b_type, kAttnLowDim, kAttnOutDim,
+                low_row.ptr, 1u) == 0) {
+            std::fprintf(stderr,
+                         "attention row reference B token=%u FAIL\n", token);
+            return false;
+        }
+    }
+    return true;
+}
+
+bool run_attention_prefill_case(const aligned_model &model,
+                                uint32_t n_tokens,
+                                const char *label,
+                                uint32_t out_b_type = kQ4Type) {
+    if (out_b_type != kQ4Type && out_b_type != kQ8Type) {
+        std::fprintf(stderr, "%s: unsupported output-B type %u FAIL\n",
+                     label, out_b_type);
+        return false;
+    }
+    const uint64_t out_b_offset = out_b_type == kQ8Type
+        ? model.attn_b_q8_offset : model.attn_b_offset;
+    const size_t heads_count =
+        (size_t)n_tokens * kAttnGroups * kAttnGroupDim;
+    const size_t low_count = (size_t)n_tokens * kAttnLowDim;
+    const size_t out_count = (size_t)n_tokens * kAttnOutDim;
+    const size_t group_tmp_count = (size_t)n_tokens * kAttnGroupDim;
+    const size_t low_tmp_count = (size_t)n_tokens * kAttnRank;
+    std::vector<float> heads_host;
+    fill_activation(&heads_host, n_tokens * kAttnGroups);
+    const std::vector<float> low_sentinel =
+        sentinel_values(low_count + kOutputGuardFloats);
+    const std::vector<float> out_sentinel =
+        sentinel_values(out_count + kOutputGuardFloats);
+    const std::vector<float> group_tmp_sentinel =
+        sentinel_values(group_tmp_count + kOutputGuardFloats);
+    const std::vector<float> low_tmp_sentinel =
+        sentinel_values(low_tmp_count + kOutputGuardFloats);
+
+    tensor_owner heads_gpu(heads_count * sizeof(float));
+    tensor_owner reference_low(low_sentinel.size() * sizeof(float));
+    tensor_owner reference_out(out_sentinel.size() * sizeof(float));
+    tensor_owner candidate_low(low_sentinel.size() * sizeof(float));
+    tensor_owner candidate_out(out_sentinel.size() * sizeof(float));
+    tensor_owner group_tmp(group_tmp_sentinel.size() * sizeof(float));
+    tensor_owner low_tmp(low_tmp_sentinel.size() * sizeof(float));
+    if (!heads_gpu.ptr || !reference_low.ptr || !reference_out.ptr ||
+        !candidate_low.ptr || !candidate_out.ptr || !group_tmp.ptr ||
+        !low_tmp.ptr || !write_tensor(heads_gpu.ptr, heads_host) ||
+        !write_tensor(reference_low.ptr, low_sentinel) ||
+        !write_tensor(reference_out.ptr, out_sentinel) ||
+        !write_tensor(candidate_low.ptr, low_sentinel) ||
+        !write_tensor(candidate_out.ptr, out_sentinel) ||
+        !write_tensor(group_tmp.ptr, group_tmp_sentinel) ||
+        !write_tensor(low_tmp.ptr, low_tmp_sentinel)) {
+        std::fprintf(stderr, "%s: setup FAIL\n", label);
+        return false;
+    }
+
+    /* Register/copy A as one contiguous tensor before the row-wise oracle
+     * requests eight contained group views.  Reversing that order creates
+     * overlapping cache registrations (eight subranges followed by their
+     * superset) and can fail before either candidate kernel is launched. */
+    const uint64_t out_a_row_bytes =
+        (kAttnGroupDim / kQkK) * sizeof(block_q4_K_test);
+    const uint64_t out_a_bytes =
+        (uint64_t)kAttnGroups * kAttnRank * out_a_row_bytes;
+    if (!ds4_gpu_cache_model_range(
+            model.data, model.size, model.attn_a_offset, out_a_bytes,
+            "ROCm Q4 attention-prefill fixture A")) {
+        std::fprintf(stderr, "%s: attention-output-A preload FAIL\n", label);
+        return false;
+    }
+
+    env_snapshot enable(kPrefillEnable);
+    env_snapshot disable(kPrefillDisable);
+    env_snapshot require(kPrefillRequire);
+    (void)unsetenv(kPrefillEnable);
+    (void)setenv(kPrefillDisable, "1", 1);
+    (void)unsetenv(kPrefillRequire);
+    if (!run_attention_rowwise_reference(
+            model, heads_gpu.ptr, reference_low.ptr, reference_out.ptr,
+            n_tokens, out_b_offset, out_b_type)) {
+        std::fprintf(stderr, "%s: row-wise reference FAIL\n", label);
+        return false;
+    }
+
+    (void)unsetenv(kPrefillEnable);
+    (void)unsetenv(kPrefillDisable);
+    (void)setenv(kPrefillRequire, "1", 1);
+    const int candidate_rc = ds4_gpu_attention_output_q4_K_batch_tensor(
+        candidate_out.ptr, candidate_low.ptr, group_tmp.ptr, low_tmp.ptr,
+        model.data, model.size, model.attn_a_offset, out_b_offset,
+        out_b_type, kAttnGroupDim, kAttnRank, kAttnGroups, kAttnOutDim,
+        heads_gpu.ptr, n_tokens);
+
+    std::vector<float> reference_low_host(low_sentinel.size());
+    std::vector<float> reference_out_host(out_sentinel.size());
+    std::vector<float> candidate_low_host(low_sentinel.size());
+    std::vector<float> candidate_out_host(out_sentinel.size());
+    std::vector<float> group_tmp_host(group_tmp_sentinel.size());
+    std::vector<float> low_tmp_host(low_tmp_sentinel.size());
+    if (candidate_rc != 1 ||
+        !read_tensor(reference_low.ptr, &reference_low_host) ||
+        !read_tensor(reference_out.ptr, &reference_out_host) ||
+        !read_tensor(candidate_low.ptr, &candidate_low_host) ||
+        !read_tensor(candidate_out.ptr, &candidate_out_host) ||
+        !read_tensor(group_tmp.ptr, &group_tmp_host) ||
+        !read_tensor(low_tmp.ptr, &low_tmp_host)) {
+        std::fprintf(stderr, "%s: candidate dispatch/read rc=%d FAIL\n",
+                     label, candidate_rc);
+        return false;
+    }
+
+    bool ok = output_guard_unchanged(
+        candidate_low_host, low_sentinel, low_count,
+        "attention candidate low canary");
+    ok = output_guard_unchanged(
+             candidate_out_host, out_sentinel, out_count,
+             "attention candidate out canary") && ok;
+    ok = output_guard_unchanged(
+             group_tmp_host, group_tmp_sentinel, group_tmp_count,
+             "attention group scratch canary") && ok;
+    ok = output_guard_unchanged(
+             low_tmp_host, low_tmp_sentinel, low_tmp_count,
+             "attention low scratch canary") && ok;
+    reference_low_host.resize(low_count);
+    reference_out_host.resize(out_count);
+    candidate_low_host.resize(low_count);
+    candidate_out_host.resize(out_count);
+    ok = bitwise_equal(candidate_low_host, reference_low_host,
+                       "attention candidate low vs 8x row-wise A") && ok;
+    ok = bitwise_equal(candidate_out_host, reference_out_host,
+                       "attention candidate out vs row-wise B") && ok;
+
+    // The batch API promises -1 for a REQUIRE diagnostic so the graph does
+    // not replay its row fallback after a forced-candidate failure.
+    if (!write_tensor(candidate_low.ptr, low_sentinel) ||
+        !write_tensor(candidate_out.ptr, out_sentinel) ||
+        !write_tensor(group_tmp.ptr, group_tmp_sentinel) ||
+        !write_tensor(low_tmp.ptr, low_tmp_sentinel)) {
+        return false;
+    }
+    (void)setenv(kPrefillDisable, "1", 1);
+    const int rejected_rc = ds4_gpu_attention_output_q4_K_batch_tensor(
+        candidate_out.ptr, candidate_low.ptr, group_tmp.ptr, low_tmp.ptr,
+        model.data, model.size, model.attn_a_offset, out_b_offset,
+        out_b_type, kAttnGroupDim, kAttnRank, kAttnGroups, kAttnOutDim,
+        heads_gpu.ptr, n_tokens);
+    if (rejected_rc != -1) {
+        std::fprintf(stderr,
+                     "%s: DISABLE+REQUIRE expected rc=-1 got=%d FAIL\n",
+                     label, rejected_rc);
+        ok = false;
+    }
+    ok = unchanged_after_rejected_call(
+             candidate_low.ptr, low_sentinel,
+             "attention rejected call preserves low") && ok;
+    ok = unchanged_after_rejected_call(
+             candidate_out.ptr, out_sentinel,
+             "attention rejected call preserves out") && ok;
+    ok = unchanged_after_rejected_call(
+             group_tmp.ptr, group_tmp_sentinel,
+             "attention rejected call preserves group scratch") && ok;
+    ok = unchanged_after_rejected_call(
+             low_tmp.ptr, low_tmp_sentinel,
+             "attention rejected call preserves low scratch") && ok;
+    std::fprintf(stderr,
+                 "%s: candidate_rc=%d rejected_rc=%d %s\n",
+                 label, candidate_rc, rejected_rc, ok ? "PASS" : "FAIL");
+    return ok;
+}
+
+bool run_grouped_attention_decode_case(const aligned_model &model) {
+    const uint64_t row_bytes =
+        (kDecodeAttnGroupDim / kQkK) * sizeof(block_q4_K_test);
+    const uint64_t group_weight_bytes =
+        (uint64_t)kDecodeAttnRank * row_bytes;
+    const size_t heads_count =
+        (size_t)kDecodeAttnGroups * kDecodeAttnGroupDim;
+    const size_t logical_count = kDecodeAttnLowDim;
+    const size_t allocation_count = logical_count + kOutputGuardFloats;
+    std::vector<float> heads_host;
+    fill_activation(&heads_host, kDecodeAttnGroups, kDecodeAttnGroupDim);
+    const std::vector<float> sentinel = sentinel_values(allocation_count);
+
+    tensor_owner heads_gpu(heads_count * sizeof(float));
+    tensor_owner legacy_gpu(allocation_count * sizeof(float));
+    tensor_owner candidate_gpu(allocation_count * sizeof(float));
+    if (!heads_gpu.ptr || !legacy_gpu.ptr || !candidate_gpu.ptr ||
+        !write_tensor(heads_gpu.ptr, heads_host) ||
+        !write_tensor(legacy_gpu.ptr, sentinel) ||
+        !write_tensor(candidate_gpu.ptr, sentinel)) {
+        std::fprintf(stderr, "grouped attention-A decode: setup FAIL\n");
+        return false;
+    }
+
+    /* Eight standalone decode calls are the bitwise oracle.  Per-group seeds
+     * and activation rows make a wrong weight/input group immediately visible. */
+    for (uint32_t group = 0; group < kDecodeAttnGroups; group++) {
+        tensor_owner head_group(ds4_gpu_tensor_view(
+            heads_gpu.ptr,
+            (uint64_t)group * kDecodeAttnGroupDim * sizeof(float),
+            (uint64_t)kDecodeAttnGroupDim * sizeof(float)));
+        tensor_owner low_group(ds4_gpu_tensor_view(
+            legacy_gpu.ptr,
+            (uint64_t)group * kDecodeAttnRank * sizeof(float),
+            (uint64_t)kDecodeAttnRank * sizeof(float)));
+        if (!head_group.ptr || !low_group.ptr ||
+            ds4_gpu_matmul_quant_tensor(
+                low_group.ptr, model.data, model.size,
+                model.decode_attn_a_offset +
+                    (uint64_t)group * group_weight_bytes,
+                kQ4Type, kDecodeAttnGroupDim, kDecodeAttnRank,
+                head_group.ptr, 1u) == 0) {
+            std::fprintf(stderr,
+                         "grouped attention-A legacy group=%u FAIL\n", group);
+            return false;
+        }
+    }
+
+    env_snapshot enable(kGroupedDecodeEnable);
+    env_snapshot disable(kGroupedDecodeDisable);
+    env_snapshot require(kGroupedDecodeRequire);
+    env_snapshot stats(kGroupedDecodeStats);
+    (void)setenv(kGroupedDecodeStats, "1", 1);
+    (void)unsetenv(kGroupedDecodeEnable);
+    (void)unsetenv(kGroupedDecodeDisable);
+    (void)unsetenv(kGroupedDecodeRequire);
+    ds4_gpu_set_ssd_streaming(false);
+
+    const int default_rc = ds4_gpu_attention_output_low_q4_K_slice_tensor(
+        candidate_gpu.ptr, model.data, model.size,
+        model.decode_attn_a_offset, kDecodeAttnGroupDim, kDecodeAttnRank,
+        0u, kDecodeAttnGroups, heads_gpu.ptr, 1);
+    std::vector<float> resident_default_host(allocation_count);
+    std::vector<float> resident_legacy_host(allocation_count);
+    bool ok = default_rc == 1 &&
+              read_tensor(candidate_gpu.ptr, &resident_default_host) &&
+              read_tensor(legacy_gpu.ptr, &resident_legacy_host);
+    if (!ok) {
+        std::fprintf(stderr,
+                     "grouped attention-A resident production default: "
+                     "expected rc=1 got=%d/readback FAIL\n",
+                     default_rc);
+    }
+    if (default_rc == 1) {
+        ok = output_guard_unchanged(
+                 resident_default_host, sentinel, logical_count,
+                 "grouped attention-A resident default output canary") && ok;
+        ok = bitwise_equal(
+                 resident_default_host, resident_legacy_host,
+                 "grouped attention-A resident default vs 8 legacy calls") && ok;
+    }
+    if (!write_tensor(candidate_gpu.ptr, sentinel)) {
+        std::fprintf(stderr,
+                     "grouped attention-A gate reset: tensor write FAIL\n");
+        return false;
+    }
+
+    /* The batch fallback passes one row at a time through the same low-level
+     * API.  It must not inherit the automatic one-token decode policy. */
+    const int batch_row_default_rc =
+        ds4_gpu_attention_output_low_q4_K_slice_tensor(
+            candidate_gpu.ptr, model.data, model.size,
+            model.decode_attn_a_offset, kDecodeAttnGroupDim, kDecodeAttnRank,
+            0u, kDecodeAttnGroups, heads_gpu.ptr, 0);
+    if (batch_row_default_rc != 0) {
+        std::fprintf(stderr,
+                     "grouped attention-A batch-row default: "
+                     "expected rc=0 got=%d FAIL\n",
+                     batch_row_default_rc);
+        ok = false;
+    }
+    ok = unchanged_after_rejected_call(
+             candidate_gpu.ptr, sentinel,
+             "grouped attention-A batch-row context preserves output") && ok;
+
+    /* The production shape is implicit only for a fully resident model.  This
+     * toggles policy state without opening or reading an SSD-backed model. */
+    ds4_gpu_set_ssd_streaming(true);
+    const int streaming_default_rc =
+        ds4_gpu_attention_output_low_q4_K_slice_tensor(
+            candidate_gpu.ptr, model.data, model.size,
+            model.decode_attn_a_offset, kDecodeAttnGroupDim, kDecodeAttnRank,
+            0u, kDecodeAttnGroups, heads_gpu.ptr, 1);
+    ds4_gpu_set_ssd_streaming(false);
+    if (streaming_default_rc != 0) {
+        std::fprintf(stderr,
+                     "grouped attention-A streaming default: "
+                     "expected rc=0 got=%d FAIL\n",
+                     streaming_default_rc);
+        ok = false;
+    }
+    ok = unchanged_after_rejected_call(
+             candidate_gpu.ptr, sentinel,
+             "grouped attention-A streaming mode preserves output") && ok;
+
+    (void)setenv(kGroupedDecodeDisable, "1", 1);
+    const int rollback_rc =
+        ds4_gpu_attention_output_low_q4_K_slice_tensor(
+            candidate_gpu.ptr, model.data, model.size,
+            model.decode_attn_a_offset, kDecodeAttnGroupDim, kDecodeAttnRank,
+            0u, kDecodeAttnGroups, heads_gpu.ptr, 1);
+    if (rollback_rc != 0) {
+        std::fprintf(stderr,
+                     "grouped attention-A resident rollback: "
+                     "expected rc=0 got=%d FAIL\n",
+                     rollback_rc);
+        ok = false;
+    }
+    ok = unchanged_after_rejected_call(
+             candidate_gpu.ptr, sentinel,
+             "grouped attention-A DISABLE rolls back resident default") && ok;
+
+    (void)setenv(kGroupedDecodeEnable, "1", 1);
+    const int disabled_enabled_rc =
+        ds4_gpu_attention_output_low_q4_K_slice_tensor(
+            candidate_gpu.ptr, model.data, model.size,
+            model.decode_attn_a_offset, kDecodeAttnGroupDim, kDecodeAttnRank,
+            0u, kDecodeAttnGroups, heads_gpu.ptr, 1);
+    if (disabled_enabled_rc != 0) {
+        std::fprintf(stderr,
+                     "grouped attention-A ENABLE+DISABLE: expected rc=0 got=%d FAIL\n",
+                     disabled_enabled_rc);
+        ok = false;
+    }
+    ok = unchanged_after_rejected_call(
+             candidate_gpu.ptr, sentinel,
+             "grouped attention-A DISABLE dominates ENABLE") && ok;
+
+    (void)setenv(kGroupedDecodeRequire, "1", 1);
+    const int disabled_rc =
+        ds4_gpu_attention_output_low_q4_K_slice_tensor(
+            candidate_gpu.ptr, model.data, model.size,
+            model.decode_attn_a_offset, kDecodeAttnGroupDim, kDecodeAttnRank,
+            0u, kDecodeAttnGroups, heads_gpu.ptr, 1);
+    if (disabled_rc != -1) {
+        std::fprintf(stderr,
+                     "grouped attention-A DISABLE+REQUIRE: expected rc=-1 got=%d FAIL\n",
+                     disabled_rc);
+        ok = false;
+    }
+    ok = unchanged_after_rejected_call(
+             candidate_gpu.ptr, sentinel,
+             "grouped attention-A DISABLE dominates REQUIRE") && ok;
+
+    (void)unsetenv(kGroupedDecodeDisable);
+    const int invalid_rc = ds4_gpu_attention_output_low_q4_K_slice_tensor(
+        candidate_gpu.ptr, model.data, model.size, model.size - 16u,
+        kDecodeAttnGroupDim, kDecodeAttnRank, 0u, kDecodeAttnGroups,
+        heads_gpu.ptr, 1);
+    if (invalid_rc != -1) {
+        std::fprintf(stderr,
+                     "grouped attention-A REQUIRE range guard: expected rc=-1 got=%d FAIL\n",
+                     invalid_rc);
+        ok = false;
+    }
+    ok = unchanged_after_rejected_call(
+             candidate_gpu.ptr, sentinel,
+             "grouped attention-A rejected range preserves output") && ok;
+
+    const int candidate_rc = ds4_gpu_attention_output_low_q4_K_slice_tensor(
+        candidate_gpu.ptr, model.data, model.size,
+        model.decode_attn_a_offset, kDecodeAttnGroupDim, kDecodeAttnRank,
+        0u, kDecodeAttnGroups, heads_gpu.ptr, 1);
+    std::vector<float> legacy_host(allocation_count);
+    std::vector<float> candidate_host(allocation_count);
+    if (candidate_rc != 1 || !read_tensor(legacy_gpu.ptr, &legacy_host) ||
+        !read_tensor(candidate_gpu.ptr, &candidate_host)) {
+        std::fprintf(stderr,
+                     "grouped attention-A candidate dispatch/read rc=%d FAIL\n",
+                     candidate_rc);
+        return false;
+    }
+    ok = output_guard_unchanged(
+             legacy_host, sentinel, logical_count,
+             "grouped attention-A legacy output canary") && ok;
+    ok = output_guard_unchanged(
+             candidate_host, sentinel, logical_count,
+             "grouped attention-A candidate output canary") && ok;
+    legacy_host.resize(logical_count);
+    candidate_host.resize(logical_count);
+    ok = bitwise_equal(candidate_host, legacy_host,
+                       "grouped attention-A candidate vs 8 legacy calls") && ok;
+
+    /* A non-zero weight-group origin consumes a compact input/output slice.
+     * Reuse groups 3 and 4 from the full fixture to verify both the weight
+     * skip and local grouped layout. */
+    constexpr uint32_t subset_group0 = 3u;
+    constexpr uint32_t subset_group_cnt = 2u;
+    const size_t subset_logical_count =
+        (size_t)subset_group_cnt * kDecodeAttnRank;
+    const std::vector<float> subset_sentinel =
+        sentinel_values(subset_logical_count + kOutputGuardFloats);
+    tensor_owner subset_heads(ds4_gpu_tensor_view(
+        heads_gpu.ptr,
+        (uint64_t)subset_group0 * kDecodeAttnGroupDim * sizeof(float),
+        (uint64_t)subset_group_cnt * kDecodeAttnGroupDim * sizeof(float)));
+    tensor_owner subset_legacy(subset_sentinel.size() * sizeof(float));
+    tensor_owner subset_candidate(subset_sentinel.size() * sizeof(float));
+    if (!subset_heads.ptr || !subset_legacy.ptr || !subset_candidate.ptr ||
+        !write_tensor(subset_legacy.ptr, subset_sentinel) ||
+        !write_tensor(subset_candidate.ptr, subset_sentinel)) {
+        std::fprintf(stderr, "grouped attention-A subset: setup FAIL\n");
+        return false;
+    }
+    for (uint32_t i = 0; i < subset_group_cnt; i++) {
+        tensor_owner head_group(ds4_gpu_tensor_view(
+            subset_heads.ptr,
+            (uint64_t)i * kDecodeAttnGroupDim * sizeof(float),
+            (uint64_t)kDecodeAttnGroupDim * sizeof(float)));
+        tensor_owner low_group(ds4_gpu_tensor_view(
+            subset_legacy.ptr,
+            (uint64_t)i * kDecodeAttnRank * sizeof(float),
+            (uint64_t)kDecodeAttnRank * sizeof(float)));
+        if (!head_group.ptr || !low_group.ptr ||
+            ds4_gpu_matmul_quant_tensor(
+                low_group.ptr, model.data, model.size,
+                model.decode_attn_a_offset +
+                    (uint64_t)(subset_group0 + i) * group_weight_bytes,
+                kQ4Type, kDecodeAttnGroupDim, kDecodeAttnRank,
+                head_group.ptr, 1u) == 0) {
+            std::fprintf(stderr,
+                         "grouped attention-A subset legacy group=%u FAIL\n",
+                         subset_group0 + i);
+            return false;
+        }
+    }
+
+    /* A slice is deliberately outside the implicit production scope.  It
+     * must fall back while the environment is clean, then dispatch when the
+     * existing explicit ENABLE override is restored. */
+    (void)unsetenv(kGroupedDecodeEnable);
+    (void)unsetenv(kGroupedDecodeRequire);
+    const int subset_default_rc =
+        ds4_gpu_attention_output_low_q4_K_slice_tensor(
+            subset_candidate.ptr, model.data, model.size,
+            model.decode_attn_a_offset, kDecodeAttnGroupDim, kDecodeAttnRank,
+            subset_group0, subset_group_cnt, subset_heads.ptr, 1);
+    if (subset_default_rc != 0) {
+        std::fprintf(stderr,
+                     "grouped attention-A non-standard default: "
+                     "expected rc=0 got=%d FAIL\n",
+                     subset_default_rc);
+        ok = false;
+    }
+    ok = unchanged_after_rejected_call(
+             subset_candidate.ptr, subset_sentinel,
+             "grouped attention-A non-standard default preserves output") && ok;
+    (void)setenv(kGroupedDecodeEnable, "1", 1);
+    const int subset_rc = ds4_gpu_attention_output_low_q4_K_slice_tensor(
+        subset_candidate.ptr, model.data, model.size,
+        model.decode_attn_a_offset, kDecodeAttnGroupDim, kDecodeAttnRank,
+        subset_group0, subset_group_cnt, subset_heads.ptr, 1);
+    std::vector<float> subset_legacy_host(subset_sentinel.size());
+    std::vector<float> subset_candidate_host(subset_sentinel.size());
+    if (subset_rc != 1 ||
+        !read_tensor(subset_legacy.ptr, &subset_legacy_host) ||
+        !read_tensor(subset_candidate.ptr, &subset_candidate_host)) {
+        std::fprintf(stderr,
+                     "grouped attention-A subset dispatch/read rc=%d FAIL\n",
+                     subset_rc);
+        return false;
+    }
+    ok = output_guard_unchanged(
+             subset_legacy_host, subset_sentinel, subset_logical_count,
+             "grouped attention-A subset legacy canary") && ok;
+    ok = output_guard_unchanged(
+             subset_candidate_host, subset_sentinel, subset_logical_count,
+             "grouped attention-A subset candidate canary") && ok;
+    subset_legacy_host.resize(subset_logical_count);
+    subset_candidate_host.resize(subset_logical_count);
+    ok = bitwise_equal(
+             subset_candidate_host, subset_legacy_host,
+             "grouped attention-A subset group0=3 count=2 vs legacy") && ok;
+    std::fprintf(stderr,
+                 "grouped attention-A decode groups=8 K=4096 rank=1024: "
+                 "resident_default=%d batch_row_default=%d "
+                 "streaming_default=%d rollback=%d "
+                 "disabled_enabled=%d disabled_required=%d invalid=%d "
+                 "candidate=%d subset_default=%d subset_enabled=%d "
+                 "stats_expected=calls:10,dispatches:3,groups:18,"
+                 "fallbacks:5,failures:2 %s\n",
+                 default_rc, batch_row_default_rc,
+                 streaming_default_rc, rollback_rc,
+                 disabled_enabled_rc, disabled_rc, invalid_rc,
+                 candidate_rc, subset_default_rc, subset_rc,
+                 ok ? "PASS" : "FAIL");
+    return ok;
+}
+
+bool run_dense_guards(const aligned_model &model) {
+    std::vector<float> x;
+    fill_activation(&x, 1u);
+    tensor_owner x_gpu(x.size() * sizeof(float));
+    tensor_owner out_gpu((uint64_t)kM0 * sizeof(float));
+    const std::vector<float> sentinel = sentinel_values(kM0);
+    if (!x_gpu.ptr || !out_gpu.ptr || !write_tensor(x_gpu.ptr, x) ||
+        !write_tensor(out_gpu.ptr, sentinel)) {
+        std::fprintf(stderr, "dense guards: setup FAIL\n");
+        return false;
+    }
+    const int bad_k_rc = ds4_gpu_matmul_quant_tensor(
+        out_gpu.ptr, model.data, model.size, model.weight0_offset, kQ4Type,
+        kK - 1u, kM0, x_gpu.ptr, 1u);
+    bool ok = bad_k_rc == 0 && unchanged_after_rejected_call(
+        out_gpu.ptr, sentinel, "dense K%256 guard preserves output");
+    if (bad_k_rc != 0) {
+        std::fprintf(stderr, "dense K%%256 guard: expected rc=0 got=%d FAIL\n",
+                     bad_k_rc);
+    }
+    if (!write_tensor(out_gpu.ptr, sentinel)) return false;
+    const int range_rc = ds4_gpu_matmul_quant_tensor(
+        out_gpu.ptr, model.data, model.size, model.size - 16u, kQ4Type,
+        kK, kM0, x_gpu.ptr, 1u);
+    ok = (range_rc == 0) && unchanged_after_rejected_call(
+        out_gpu.ptr, sentinel, "dense model-range guard preserves output") && ok;
+    if (range_rc != 0) {
+        std::fprintf(stderr, "dense model-range guard: expected rc=0 got=%d FAIL\n",
+                     range_rc);
+    }
+    return ok;
+}
+
+bool run_pair_guards(const aligned_model &model) {
+    constexpr uint32_t n_tokens = 9u;
+    std::vector<float> x;
+    fill_activation(&x, n_tokens);
+    tensor_owner x_gpu(x.size() * sizeof(float));
+    tensor_owner out0((uint64_t)n_tokens * kM0 * sizeof(float));
+    tensor_owner out1((uint64_t)n_tokens * kM1 * sizeof(float));
+    const std::vector<float> sentinel0 = sentinel_values((uint64_t)n_tokens * kM0);
+    const std::vector<float> sentinel1 = sentinel_values((uint64_t)n_tokens * kM1);
+    if (!x_gpu.ptr || !out0.ptr || !out1.ptr || !write_tensor(x_gpu.ptr, x) ||
+        !write_tensor(out0.ptr, sentinel0) || !write_tensor(out1.ptr, sentinel1)) {
+        std::fprintf(stderr, "pair guards: setup FAIL\n");
+        return false;
+    }
+    env_snapshot prefill_enable(kPrefillEnable);
+    env_snapshot prefill_disable(kPrefillDisable);
+    env_snapshot prefill_require(kPrefillRequire);
+    env_snapshot wmma_enable(kPrefillWmmaEnable);
+    env_snapshot wmma_disable(kPrefillWmmaDisable);
+    env_snapshot wmma_require(kPrefillWmmaRequire);
+    env_snapshot q8_enable(kPrefillQ8Wave32Enable);
+    env_snapshot q8_disable(kPrefillQ8Wave32Disable);
+    env_snapshot q8_require(kPrefillQ8Wave32Require);
+    (void)unsetenv(kPrefillEnable);
+    (void)setenv(kPrefillDisable, "1", 1);
+    (void)unsetenv(kPrefillRequire);
+    (void)unsetenv(kPrefillWmmaEnable);
+    (void)unsetenv(kPrefillWmmaDisable);
+    (void)unsetenv(kPrefillWmmaRequire);
+    (void)unsetenv(kPrefillQ8Wave32Enable);
+    (void)unsetenv(kPrefillQ8Wave32Disable);
+    (void)unsetenv(kPrefillQ8Wave32Require);
+    const int rc = ds4_gpu_matmul_q4_K_pair_tensor(
+        out0.ptr, out1.ptr, model.data, model.size,
+        model.weight0_offset, model.weight1_offset,
+        kK, kM0, kM1, x_gpu.ptr, n_tokens);
+    bool ok = rc == 0;
+    if (rc != 0) {
+        std::fprintf(stderr, "pair n_tok=9 guard: expected rc=0 got=%d FAIL\n", rc);
+    }
+    ok = unchanged_after_rejected_call(
+             out0.ptr, sentinel0, "pair n_tok=9 preserves out0") && ok;
+    ok = unchanged_after_rejected_call(
+             out1.ptr, sentinel1, "pair n_tok=9 preserves out1") && ok;
+
+    (void)unsetenv(kPrefillDisable);
+    (void)setenv(kPrefillRequire, "1", 1);
+    if (!write_tensor(out0.ptr, sentinel0) ||
+        !write_tensor(out1.ptr, sentinel1)) {
+        return false;
+    }
+    const int validation_rc = ds4_gpu_matmul_q4_K_pair_tensor(
+        out0.ptr, out1.ptr, model.data, model.size,
+        model.weight0_offset, model.weight1_offset,
+        kK - 1u, kM0, kM1, x_gpu.ptr, n_tokens);
+    ok = validation_rc == -1 && unchanged_after_rejected_call(
+        out0.ptr, sentinel0,
+        "required pair validation preserves out0") && ok;
+    ok = unchanged_after_rejected_call(
+             out1.ptr, sentinel1,
+             "required pair validation preserves out1") && ok;
+    if (validation_rc != -1) {
+        std::fprintf(stderr,
+                     "required pair validation: expected rc=-1 got=%d FAIL\n",
+                     validation_rc);
+    }
+
+    if (!write_tensor(out0.ptr, sentinel0) ||
+        !write_tensor(out1.ptr, sentinel1)) {
+        return false;
+    }
+    const int range_rc = ds4_gpu_matmul_q4_K_pair_tensor(
+        out0.ptr, out1.ptr, model.data, model.size,
+        model.size - 16u, model.weight1_offset,
+        kK, kM0, kM1, x_gpu.ptr, n_tokens);
+    ok = range_rc == -1 && unchanged_after_rejected_call(
+        out0.ptr, sentinel0, "required pair range preserves out0") && ok;
+    ok = unchanged_after_rejected_call(
+             out1.ptr, sentinel1,
+             "required pair range preserves out1") && ok;
+    if (range_rc != -1) {
+        std::fprintf(stderr,
+                     "required pair range: expected rc=-1 got=%d FAIL\n",
+                     range_rc);
+    }
+
+    const size_t strict_shared_count =
+        (size_t)n_tokens * ((size_t)kM0 + kM1);
+    const std::vector<float> strict_shared_sentinel =
+        sentinel_values(strict_shared_count);
+    tensor_owner strict_shared(strict_shared_count * sizeof(float));
+    tensor_owner strict_overlap0(ds4_gpu_tensor_view(
+        strict_shared.ptr, 0u,
+        (uint64_t)n_tokens * kM0 * sizeof(float)));
+    tensor_owner strict_overlap1(ds4_gpu_tensor_view(
+        strict_shared.ptr,
+        ((uint64_t)n_tokens * kM0 - 1u) * sizeof(float),
+        (uint64_t)n_tokens * kM1 * sizeof(float)));
+    if (!strict_shared.ptr || !strict_overlap0.ptr || !strict_overlap1.ptr ||
+        !write_tensor(strict_shared.ptr, strict_shared_sentinel)) {
+        std::fprintf(stderr, "required pair overlap guard: setup FAIL\n");
+        return false;
+    }
+    const int strict_overlap_rc = ds4_gpu_matmul_q4_K_pair_tensor(
+        strict_overlap0.ptr, strict_overlap1.ptr, model.data, model.size,
+        model.weight0_offset, model.weight1_offset,
+        kK, kM0, kM1, x_gpu.ptr, n_tokens);
+    ok = strict_overlap_rc == -1 && unchanged_after_rejected_call(
+        strict_shared.ptr, strict_shared_sentinel,
+        "required pair partial-overlap preserves storage") && ok;
+    if (strict_overlap_rc != -1) {
+        std::fprintf(stderr,
+                     "required pair partial-overlap: "
+                     "expected rc=-1 got=%d FAIL\n",
+                     strict_overlap_rc);
+    }
+
+    (void)setenv(kPrefillDisable, "1", 1);
+    (void)unsetenv(kPrefillRequire);
+    const size_t shared_count = (size_t)kM0 + kM1;
+    const std::vector<float> shared_sentinel = sentinel_values(shared_count);
+    tensor_owner shared(shared_count * sizeof(float));
+    tensor_owner overlap0(ds4_gpu_tensor_view(
+        shared.ptr, 0u, (uint64_t)kM0 * sizeof(float)));
+    tensor_owner overlap1(ds4_gpu_tensor_view(
+        shared.ptr, (uint64_t)(kM0 - 1u) * sizeof(float),
+        (uint64_t)kM1 * sizeof(float)));
+    if (!shared.ptr || !overlap0.ptr || !overlap1.ptr ||
+        !write_tensor(shared.ptr, shared_sentinel)) {
+        std::fprintf(stderr, "pair overlap guard: setup FAIL\n");
+        return false;
+    }
+    const int overlap_rc = ds4_gpu_matmul_q4_K_pair_tensor(
+        overlap0.ptr, overlap1.ptr, model.data, model.size,
+        model.weight0_offset, model.weight1_offset,
+        kK, kM0, kM1, x_gpu.ptr, 1u);
+    ok = overlap_rc == 0 && unchanged_after_rejected_call(
+        shared.ptr, shared_sentinel,
+        "pair partial-overlap guard preserves storage") && ok;
+    if (overlap_rc != 0) {
+        std::fprintf(stderr,
+                     "pair partial-overlap guard: expected rc=0 got=%d FAIL\n",
+                     overlap_rc);
+    }
+    return ok;
+}
+
+bool run_pair_opt_in_guards(const aligned_model &model) {
+    constexpr uint32_t n_tokens = 1u;
+    std::vector<float> x;
+    fill_activation(&x, n_tokens);
+    tensor_owner x_gpu(x.size() * sizeof(float));
+    tensor_owner out0((uint64_t)n_tokens * kM0 * sizeof(float));
+    tensor_owner out1((uint64_t)n_tokens * kM1 * sizeof(float));
+    const std::vector<float> sentinel0 = sentinel_values(kM0);
+    const std::vector<float> sentinel1 = sentinel_values(kM1);
+    if (!x_gpu.ptr || !out0.ptr || !out1.ptr || !write_tensor(x_gpu.ptr, x)) {
+        std::fprintf(stderr, "pair opt-in guards: setup FAIL\n");
+        return false;
+    }
+
+    auto rejected_call = [&](const char *label) {
+        if (!write_tensor(out0.ptr, sentinel0) ||
+            !write_tensor(out1.ptr, sentinel1)) {
+            return false;
+        }
+        const int rc = ds4_gpu_matmul_q4_K_pair_tensor(
+            out0.ptr, out1.ptr, model.data, model.size,
+            model.weight0_offset, model.weight1_offset,
+            kK, kM0, kM1, x_gpu.ptr, n_tokens);
+        bool guard_ok = rc == 0;
+        if (rc != 0) {
+            std::fprintf(stderr, "%s: expected rc=0 got=%d FAIL\n", label, rc);
+        }
+        guard_ok = unchanged_after_rejected_call(out0.ptr, sentinel0, label) &&
+                   guard_ok;
+        guard_ok = unchanged_after_rejected_call(out1.ptr, sentinel1, label) &&
+                   guard_ok;
+        return guard_ok;
+    };
+
+    env_snapshot enable("DS4_ROCM_ENABLE_Q4_DENSE_PAIR");
+    env_snapshot disable("DS4_ROCM_DISABLE_Q4_DENSE_PAIR");
+    (void)unsetenv("DS4_ROCM_ENABLE_Q4_DENSE_PAIR");
+    (void)unsetenv("DS4_ROCM_DISABLE_Q4_DENSE_PAIR");
+    bool ok = rejected_call("pair disabled-by-default preserves outputs");
+    (void)setenv("DS4_ROCM_ENABLE_Q4_DENSE_PAIR", "1", 1);
+    (void)setenv("DS4_ROCM_DISABLE_Q4_DENSE_PAIR", "1", 1);
+    ok = rejected_call("pair DISABLE dominates ENABLE") && ok;
+    return ok;
+}
+
+bool run_prefill_wmma_smoke(const aligned_model &model) {
+#if DS4_TEST_HAS_HIP_RUNTIME
+    hipDeviceProp_t properties{};
+    if (hipGetDeviceProperties(&properties, 0) != hipSuccess ||
+        properties.warpSize != 32 ||
+        std::strncmp(properties.gcnArchName, "gfx1151", 7u) != 0) {
+        std::fprintf(stderr,
+                     "ROCm Q4 direct-WMMA prefill: SKIP "
+                     "(requires gfx1151 wave32)\n");
+        return true;
+    }
+#else
+    (void)model;
+    return true;
+#endif
+
+    constexpr uint32_t n_tokens = 257u;
+    const size_t logical_count = (size_t)n_tokens * kM0;
+    /* A broken N-tail store could write the remaining 63 tokens of the final
+     * tile, while a simultaneous M-tail failure in the forced K128 rowtile
+     * could address the remaining 255 rows.  Cover both predicates failing
+     * together, not just the normal API canary. */
+    constexpr size_t wmma_guard_floats =
+        (64u - 1u) * kM0 + (256u - 1u);
+    const size_t allocation_count = logical_count + wmma_guard_floats;
+    const std::vector<float> sentinel = sentinel_values(allocation_count);
+    std::vector<float> x;
+    fill_activation(&x, n_tokens, kK);
+    tensor_owner x_gpu(x.size() * sizeof(float));
+    tensor_owner tile8_gpu(allocation_count * sizeof(float));
+    tensor_owner wmma_gpu(allocation_count * sizeof(float));
+    tensor_owner k64_gpu(allocation_count * sizeof(float));
+    tensor_owner k128_gpu(allocation_count * sizeof(float));
+    tensor_owner k128_rollback_gpu(allocation_count * sizeof(float));
+    if (!x_gpu.ptr || !tile8_gpu.ptr || !wmma_gpu.ptr || !k64_gpu.ptr ||
+        !k128_gpu.ptr || !k128_rollback_gpu.ptr ||
+        !write_tensor(x_gpu.ptr, x) || !write_tensor(tile8_gpu.ptr, sentinel) ||
+        !write_tensor(wmma_gpu.ptr, sentinel) ||
+        !write_tensor(k64_gpu.ptr, sentinel) ||
+        !write_tensor(k128_gpu.ptr, sentinel) ||
+        !write_tensor(k128_rollback_gpu.ptr, sentinel)) {
+        std::fprintf(stderr, "ROCm Q4 direct-WMMA prefill: setup FAIL\n");
+        return false;
+    }
+
+    env_snapshot tile8_enable(kPrefillEnable);
+    env_snapshot tile8_disable(kPrefillDisable);
+    env_snapshot tile8_require(kPrefillRequire);
+    env_snapshot tile4_require(kPrefillK1024Tile4Require);
+    env_snapshot wmma_enable(kPrefillWmmaEnable);
+    env_snapshot wmma_ssd_enable(kPrefillWmmaSsdEnable);
+    env_snapshot wmma_disable(kPrefillWmmaDisable);
+    env_snapshot wmma_require(kPrefillWmmaRequire);
+    env_snapshot wmma_row_tile(kPrefillWmmaRowTile);
+    env_snapshot wmma_k64(kPrefillWmmaK64);
+    env_snapshot wmma_k128_disable(kPrefillWmmaK128Disable);
+    env_snapshot q8_wave32_enable(kPrefillQ8Wave32Enable);
+    env_snapshot q8_wave32_disable(kPrefillQ8Wave32Disable);
+    env_snapshot q8_wave32_require(kPrefillQ8Wave32Require);
+
+    (void)unsetenv(kPrefillEnable);
+    (void)unsetenv(kPrefillDisable);
+    (void)setenv(kPrefillRequire, "1", 1);
+    (void)unsetenv(kPrefillK1024Tile4Require);
+    (void)unsetenv(kPrefillWmmaEnable);
+    (void)unsetenv(kPrefillWmmaSsdEnable);
+    (void)setenv(kPrefillWmmaDisable, "1", 1);
+    (void)unsetenv(kPrefillWmmaRequire);
+    (void)unsetenv(kPrefillWmmaRowTile);
+    (void)setenv(kPrefillWmmaK64, "0", 1);
+    (void)unsetenv(kPrefillWmmaK128Disable);
+    (void)unsetenv(kPrefillQ8Wave32Enable);
+    (void)unsetenv(kPrefillQ8Wave32Disable);
+    (void)unsetenv(kPrefillQ8Wave32Require);
+    ds4_rocm_test_q4_prefill_wmma_reset();
+    const int tile8_rc = ds4_gpu_matmul_quant_tensor(
+        tile8_gpu.ptr, model.data, model.size, model.weight0_offset, kQ4Type,
+        kK, kM0, x_gpu.ptr, n_tokens);
+    const uint64_t tile8_wmma_calls =
+        ds4_rocm_test_q4_prefill_wmma_get_calls();
+    const uint64_t tile8_k64_calls =
+        ds4_rocm_test_q4_prefill_wmma_k64_get_calls();
+
+    (void)unsetenv(kPrefillRequire);
+    (void)unsetenv(kPrefillWmmaEnable);
+    (void)unsetenv(kPrefillWmmaDisable);
+    (void)unsetenv(kPrefillWmmaRequire);
+    (void)setenv(kPrefillWmmaK64, "0", 1);
+    ds4_rocm_test_q4_prefill_wmma_reset();
+    const int wmma_rc = ds4_gpu_matmul_quant_tensor(
+        wmma_gpu.ptr, model.data, model.size, model.weight0_offset, kQ4Type,
+        kK, kM0, x_gpu.ptr, n_tokens);
+    const uint64_t wmma_calls =
+        ds4_rocm_test_q4_prefill_wmma_get_calls();
+    const uint64_t wmma_k64_calls =
+        ds4_rocm_test_q4_prefill_wmma_k64_get_calls();
+
+    /* K64 remains the automatic fallback for row geometries outside K128's
+     * 256-row scope. */
+    (void)unsetenv(kPrefillWmmaK64);
+    ds4_rocm_test_q4_prefill_wmma_reset();
+    const int k64_fallback_rc = ds4_gpu_matmul_quant_tensor(
+        k64_gpu.ptr, model.data, model.size, model.weight0_offset, kQ4Type,
+        kK, kM0, x_gpu.ptr, n_tokens);
+    const uint64_t k64_fallback_wmma_calls =
+        ds4_rocm_test_q4_prefill_wmma_get_calls();
+    const uint64_t k64_fallback_calls =
+        ds4_rocm_test_q4_prefill_wmma_k64_get_calls();
+
+    /* Force the 256-row geometry on this compact M-tail fixture so default-on
+     * K128 is covered without allocating the full q_b output.  The direct q_b
+     * benchmark exercises the natural shape. */
+    (void)setenv(kPrefillWmmaRowTile, "256", 1);
+    (void)unsetenv(kPrefillWmmaK128Disable);
+    ds4_rocm_test_q4_prefill_wmma_reset();
+    const int k128_rc = ds4_gpu_matmul_quant_tensor(
+        k128_gpu.ptr, model.data, model.size, model.weight0_offset, kQ4Type,
+        kK, kM0, x_gpu.ptr, n_tokens);
+    const uint64_t k128_wmma_calls =
+        ds4_rocm_test_q4_prefill_wmma_get_calls();
+    const uint64_t k128_k64_calls =
+        ds4_rocm_test_q4_prefill_wmma_k64_get_calls();
+    const uint64_t k128_calls =
+        ds4_rocm_test_q4_prefill_wmma_k128_get_calls();
+
+    /* A single opt-out must restore K64 on the same otherwise-eligible
+     * production geometry. */
+    (void)setenv(kPrefillWmmaK128Disable, "1", 1);
+    ds4_rocm_test_q4_prefill_wmma_reset();
+    const int k128_rollback_rc = ds4_gpu_matmul_quant_tensor(
+        k128_rollback_gpu.ptr, model.data, model.size, model.weight0_offset,
+        kQ4Type, kK, kM0, x_gpu.ptr, n_tokens);
+    const uint64_t k128_rollback_wmma_calls =
+        ds4_rocm_test_q4_prefill_wmma_get_calls();
+    const uint64_t k128_rollback_k64_calls =
+        ds4_rocm_test_q4_prefill_wmma_k64_get_calls();
+    const uint64_t k128_rollback_k128_calls =
+        ds4_rocm_test_q4_prefill_wmma_k128_get_calls();
+
+    std::vector<float> tile8(allocation_count);
+    std::vector<float> wmma(allocation_count);
+    std::vector<float> k64_fallback(allocation_count);
+    std::vector<float> k128(allocation_count);
+    std::vector<float> k128_rollback(allocation_count);
+    bool ok = tile8_rc != 0 && wmma_rc != 0 && k64_fallback_rc != 0 &&
+        k128_rc != 0 && k128_rollback_rc != 0 &&
+        tile8_wmma_calls == 0u && tile8_k64_calls == 0u &&
+        wmma_calls == 1u && wmma_k64_calls == 0u &&
+        k64_fallback_wmma_calls == 1u && k64_fallback_calls == 1u &&
+        k128_wmma_calls == 1u && k128_k64_calls == 0u &&
+        k128_calls == 1u && k128_rollback_wmma_calls == 1u &&
+        k128_rollback_k64_calls == 1u &&
+        k128_rollback_k128_calls == 0u &&
+        read_tensor(tile8_gpu.ptr, &tile8) &&
+        read_tensor(wmma_gpu.ptr, &wmma) &&
+        read_tensor(k64_gpu.ptr, &k64_fallback) &&
+        read_tensor(k128_gpu.ptr, &k128) &&
+        read_tensor(k128_rollback_gpu.ptr, &k128_rollback);
+    if (ok) {
+        ok = output_body_overwritten(tile8, sentinel, logical_count,
+                                     "direct-WMMA TILE8 output body") && ok;
+        ok = output_body_overwritten(wmma, sentinel, logical_count,
+                                     "direct-WMMA candidate output body") && ok;
+        ok = output_body_overwritten(
+                 k64_fallback, sentinel, logical_count,
+                 "direct-WMMA K64 fallback output body") && ok;
+        ok = output_body_overwritten(
+                 k128, sentinel, logical_count,
+                 "direct-WMMA default K128 output body") && ok;
+        ok = output_body_overwritten(
+                 k128_rollback, sentinel, logical_count,
+                 "direct-WMMA K128 opt-out output body") && ok;
+        ok = output_guard_unchanged(tile8, sentinel, logical_count,
+                                    "direct-WMMA TILE8 output canary") && ok;
+        ok = output_guard_unchanged(wmma, sentinel, logical_count,
+                                    "direct-WMMA candidate output canary") && ok;
+        ok = output_guard_unchanged(
+                 k64_fallback, sentinel, logical_count,
+                 "direct-WMMA K64 fallback output canary") && ok;
+        ok = output_guard_unchanged(
+                 k128, sentinel, logical_count,
+                 "direct-WMMA default K128 output canary") && ok;
+        ok = output_guard_unchanged(
+                 k128_rollback, sentinel, logical_count,
+                 "direct-WMMA K128 opt-out output canary") && ok;
+        tile8.resize(logical_count);
+        wmma.resize(logical_count);
+        k64_fallback.resize(logical_count);
+        k128.resize(logical_count);
+        k128_rollback.resize(logical_count);
+        ok = close_with_tolerance(wmma, tile8, 2.0f, 3.0e-2f,
+                                  "direct-WMMA vs TILE8 N/M tail") && ok;
+        ok = bitwise_equal(k64_fallback, wmma,
+                           "direct-WMMA K64 fallback vs K32 N/M tail") && ok;
+        ok = bitwise_equal(k128, k64_fallback,
+                           "direct-WMMA K128 vs K64 N/M tail") && ok;
+        ok = bitwise_equal(k128_rollback, k128,
+                           "direct-WMMA K128 opt-out vs default") && ok;
+    }
+
+    /* Return to the neutral K32 setting for the remaining policy cases. */
+    (void)setenv(kPrefillWmmaK64, "0", 1);
+    (void)unsetenv(kPrefillWmmaK128Disable);
+    (void)unsetenv(kPrefillWmmaRowTile);
+    (void)setenv(kPrefillWmmaDisable, "1", 1);
+    (void)unsetenv(kPrefillWmmaRequire);
+    if (!write_tensor(wmma_gpu.ptr, sentinel)) return false;
+    ds4_rocm_test_q4_prefill_wmma_reset();
+    const int opt_out_rc = ds4_gpu_matmul_quant_tensor(
+        wmma_gpu.ptr, model.data, model.size, model.weight0_offset, kQ4Type,
+        kK, kM0, x_gpu.ptr, n_tokens);
+    const uint64_t opt_out_wmma_calls =
+        ds4_rocm_test_q4_prefill_wmma_get_calls();
+    const uint64_t opt_out_k64_calls =
+        ds4_rocm_test_q4_prefill_wmma_k64_get_calls();
+    std::vector<float> opt_out(allocation_count);
+    const bool opt_out_read = opt_out_rc != 0 && opt_out_wmma_calls == 0u &&
+        opt_out_k64_calls == 0u &&
+        read_tensor(wmma_gpu.ptr, &opt_out);
+    ok = opt_out_read && ok;
+    if (opt_out_read) {
+        ok = output_guard_unchanged(
+                 opt_out, sentinel, logical_count,
+                 "direct-WMMA opt-out output canary") && ok;
+        opt_out.resize(logical_count);
+        ok = bitwise_equal(opt_out, tile8,
+                           "direct-WMMA opt-out vs TILE8") && ok;
+    }
+
+    (void)setenv(kPrefillWmmaRequire, "1", 1);
+    if (!write_tensor(wmma_gpu.ptr, sentinel)) return false;
+    ds4_rocm_test_q4_prefill_wmma_reset();
+    const int rejected_rc = ds4_gpu_matmul_quant_tensor(
+        wmma_gpu.ptr, model.data, model.size, model.weight0_offset, kQ4Type,
+        kK, kM0, x_gpu.ptr, n_tokens);
+    const uint64_t rejected_wmma_calls =
+        ds4_rocm_test_q4_prefill_wmma_get_calls();
+    const uint64_t rejected_k64_calls =
+        ds4_rocm_test_q4_prefill_wmma_k64_get_calls();
+    ok = rejected_rc == 0 && rejected_wmma_calls == 0u &&
+         rejected_k64_calls == 0u &&
+         unchanged_after_rejected_call(
+             wmma_gpu.ptr, sentinel,
+             "direct-WMMA DISABLE+REQUIRE preserves output") && ok;
+    std::fprintf(stderr,
+                 "ROCm Q4 direct-WMMA prefill: tile8=%d/%llu/%llu "
+                 "K32=%d/%llu/%llu K64-fallback=%d/%llu/%llu "
+                 "K128-default=%d/%llu/%llu/%llu "
+                 "K128-optout=%d/%llu/%llu/%llu "
+                 "opt_out=%d/%llu/%llu "
+                 "rejected=%d/%llu/%llu %s\n",
+                 tile8_rc, (unsigned long long)tile8_wmma_calls,
+                 (unsigned long long)tile8_k64_calls,
+                 wmma_rc, (unsigned long long)wmma_calls,
+                 (unsigned long long)wmma_k64_calls,
+                 k64_fallback_rc,
+                 (unsigned long long)k64_fallback_wmma_calls,
+                 (unsigned long long)k64_fallback_calls,
+                 k128_rc, (unsigned long long)k128_wmma_calls,
+                 (unsigned long long)k128_k64_calls,
+                 (unsigned long long)k128_calls,
+                 k128_rollback_rc,
+                 (unsigned long long)k128_rollback_wmma_calls,
+                 (unsigned long long)k128_rollback_k64_calls,
+                 (unsigned long long)k128_rollback_k128_calls,
+                 opt_out_rc, (unsigned long long)opt_out_wmma_calls,
+                 (unsigned long long)opt_out_k64_calls,
+                 rejected_rc, (unsigned long long)rejected_wmma_calls,
+                 (unsigned long long)rejected_k64_calls,
+                 ok ? "PASS" : "FAIL");
+    return ok;
+}
+
+bool run_attention_output_wmma_smoke(const aligned_model &model) {
+#if DS4_TEST_HAS_HIP_RUNTIME
+    hipDeviceProp_t properties{};
+    if (hipGetDeviceProperties(&properties, 0) != hipSuccess ||
+        properties.warpSize != 32 ||
+        std::strncmp(properties.gcnArchName, "gfx1151", 7u) != 0) {
+        std::fprintf(stderr,
+                     "ROCm Q4 production output direct-WMMA: SKIP "
+                     "(requires gfx1151 wave32)\n");
+        return true;
+    }
+#else
+    (void)model;
+    return true;
+#endif
+
+    constexpr uint32_t n_tokens = 257u;
+    const size_t heads_count =
+        (size_t)n_tokens * kDecodeAttnGroups * kDecodeAttnGroupDim;
+    const size_t low_count = (size_t)n_tokens * kDecodeAttnLowDim;
+    const size_t out_count = (size_t)n_tokens * kDecodeAttnOutDim;
+    constexpr size_t low_guard = (64u - 1u) * kDecodeAttnLowDim;
+    constexpr size_t out_guard = (64u - 1u) * kDecodeAttnOutDim;
+    std::vector<float> heads_host;
+    fill_activation(
+        &heads_host, n_tokens * kDecodeAttnGroups, kDecodeAttnGroupDim);
+    const std::vector<float> low_sentinel =
+        sentinel_values(low_count + low_guard);
+    const std::vector<float> out_sentinel =
+        sentinel_values(out_count + out_guard);
+
+    tensor_owner heads_gpu(heads_count * sizeof(float));
+    tensor_owner tile8_low(low_sentinel.size() * sizeof(float));
+    tensor_owner tile8_out(out_sentinel.size() * sizeof(float));
+    tensor_owner candidate_low(low_sentinel.size() * sizeof(float));
+    tensor_owner candidate_out(out_sentinel.size() * sizeof(float));
+    tensor_owner replay_out(out_sentinel.size() * sizeof(float));
+    if (!heads_gpu.ptr || !tile8_low.ptr || !tile8_out.ptr ||
+        !candidate_low.ptr || !candidate_out.ptr || !replay_out.ptr ||
+        !write_tensor(heads_gpu.ptr, heads_host) ||
+        !write_tensor(tile8_low.ptr, low_sentinel) ||
+        !write_tensor(tile8_out.ptr, out_sentinel)) {
+        std::fprintf(stderr,
+                     "ROCm Q4 production output direct-WMMA: setup FAIL\n");
+        return false;
+    }
+
+    env_snapshot tile8_enable(kPrefillEnable);
+    env_snapshot tile8_disable(kPrefillDisable);
+    env_snapshot tile8_require(kPrefillRequire);
+    env_snapshot tile4_require(kPrefillK1024Tile4Require);
+    env_snapshot wmma_enable(kPrefillWmmaEnable);
+    env_snapshot wmma_ssd_enable(kPrefillWmmaSsdEnable);
+    env_snapshot wmma_disable(kPrefillWmmaDisable);
+    env_snapshot wmma_require(kPrefillWmmaRequire);
+    env_snapshot wmma_row_tile(kPrefillWmmaRowTile);
+    env_snapshot wmma_k64(kPrefillWmmaK64);
+    env_snapshot q8_wave32_enable(kPrefillQ8Wave32Enable);
+    env_snapshot q8_wave32_disable(kPrefillQ8Wave32Disable);
+    env_snapshot q8_wave32_require(kPrefillQ8Wave32Require);
+
+    struct batch_result {
+        int rc = 0;
+        uint64_t wmma_calls = 0u;
+        uint64_t k64_calls = 0u;
+        bool write_ok = false;
+        bool low_read = false;
+        bool out_read = false;
+    };
+
+    struct replay_result {
+        int rc = 0;
+        uint64_t wmma_calls = 0u;
+        uint64_t k64_calls = 0u;
+        bool write_ok = false;
+        bool out_read = false;
+    };
+
+    auto clear_controls = [&]() {
+        (void)unsetenv(kPrefillEnable);
+        (void)unsetenv(kPrefillDisable);
+        (void)unsetenv(kPrefillRequire);
+        (void)unsetenv(kPrefillK1024Tile4Require);
+        (void)unsetenv(kPrefillWmmaEnable);
+        (void)unsetenv(kPrefillWmmaSsdEnable);
+        (void)unsetenv(kPrefillWmmaDisable);
+        (void)unsetenv(kPrefillWmmaRequire);
+        (void)unsetenv(kPrefillWmmaRowTile);
+        (void)unsetenv(kPrefillWmmaK64);
+        (void)unsetenv(kPrefillQ8Wave32Enable);
+        (void)unsetenv(kPrefillQ8Wave32Disable);
+        (void)unsetenv(kPrefillQ8Wave32Require);
+    };
+
+    auto run_batch = [&](ds4_gpu_tensor *low, ds4_gpu_tensor *out,
+                         std::vector<float> *low_host,
+                         std::vector<float> *out_host) {
+        batch_result result;
+        ds4_rocm_test_q4_prefill_wmma_reset();
+        result.write_ok = write_tensor(low, low_sentinel) &&
+                          write_tensor(out, out_sentinel);
+        if (result.write_ok) {
+            result.rc = ds4_gpu_attention_output_q4_K_batch_tensor(
+                out, low, nullptr, nullptr,
+                model.data, model.size, model.decode_attn_a_offset,
+                model.decode_attn_b_offset, kQ4Type, kDecodeAttnGroupDim,
+                kDecodeAttnRank, kDecodeAttnGroups, kDecodeAttnOutDim,
+                heads_gpu.ptr, n_tokens);
+        }
+        result.wmma_calls = ds4_rocm_test_q4_prefill_wmma_get_calls();
+        result.k64_calls = ds4_rocm_test_q4_prefill_wmma_k64_get_calls();
+        if (result.write_ok) {
+            result.low_read = read_tensor(low, low_host);
+            result.out_read = read_tensor(out, out_host);
+        }
+        return result;
+    };
+
+    auto batch_ready = [&](const batch_result &result,
+                           uint64_t expected_wmma,
+                           uint64_t expected_k64,
+                           const char *label) {
+        const bool ready = result.write_ok && result.rc == 1 &&
+            result.wmma_calls == expected_wmma &&
+            result.k64_calls == expected_k64 &&
+            result.low_read && result.out_read;
+        std::fprintf(stderr,
+                     "%s: rc=%d WMMA=%llu/%llu K64=%llu/%llu "
+                     "write=%d read=%d/%d %s\n",
+                     label, result.rc,
+                     (unsigned long long)result.wmma_calls,
+                     (unsigned long long)expected_wmma,
+                     (unsigned long long)result.k64_calls,
+                     (unsigned long long)expected_k64,
+                     result.write_ok ? 1 : 0,
+                     result.low_read ? 1 : 0,
+                     result.out_read ? 1 : 0,
+                     ready ? "PASS" : "FAIL");
+        return ready;
+    };
+
+    auto batch_memory_ok = [&](const batch_result &result,
+                               const std::vector<float> &low_host,
+                               const std::vector<float> &out_host,
+                               const char *label) {
+        if (!result.low_read || !result.out_read) return false;
+        const std::string low_body = std::string(label) + " low body";
+        const std::string out_body = std::string(label) + " out body";
+        const std::string low_finite = std::string(label) + " low finite";
+        const std::string out_finite = std::string(label) + " out finite";
+        const std::string low_guard = std::string(label) + " low N-tail";
+        const std::string out_guard = std::string(label) + " out N-tail";
+        bool memory_ok = output_body_overwritten(
+            low_host, low_sentinel, low_count, low_body.c_str());
+        memory_ok = output_body_overwritten(
+            out_host, out_sentinel, out_count, out_body.c_str()) && memory_ok;
+        memory_ok = output_body_finite(
+            low_host, low_count, low_finite.c_str()) && memory_ok;
+        memory_ok = output_body_finite(
+            out_host, out_count, out_finite.c_str()) && memory_ok;
+        memory_ok = output_guard_unchanged(
+            low_host, low_sentinel, low_count, low_guard.c_str()) && memory_ok;
+        memory_ok = output_guard_unchanged(
+            out_host, out_sentinel, out_count, out_guard.c_str()) && memory_ok;
+        return memory_ok;
+    };
+
+    auto run_tile8_replay = [&](const ds4_gpu_tensor *low,
+                                std::vector<float> *out_host) {
+        replay_result result;
+        clear_controls();
+        (void)setenv(kPrefillRequire, "1", 1);
+        (void)setenv(kPrefillWmmaDisable, "1", 1);
+        ds4_rocm_test_q4_prefill_wmma_reset();
+        result.write_ok = write_tensor(replay_out.ptr, out_sentinel);
+        if (result.write_ok) {
+            result.rc = ds4_gpu_matmul_quant_tensor(
+                replay_out.ptr, model.data, model.size,
+                model.decode_attn_b_offset, kQ4Type,
+                kDecodeAttnLowDim, kDecodeAttnOutDim, low, n_tokens);
+        }
+        result.wmma_calls = ds4_rocm_test_q4_prefill_wmma_get_calls();
+        result.k64_calls = ds4_rocm_test_q4_prefill_wmma_k64_get_calls();
+        if (result.write_ok) {
+            result.out_read = read_tensor(replay_out.ptr, out_host);
+        }
+        return result;
+    };
+
+    auto replay_ready = [&](const replay_result &result,
+                            const std::vector<float> &out_host,
+                            const char *label) {
+        bool ready = result.write_ok && result.rc != 0 &&
+            result.wmma_calls == 0u && result.k64_calls == 0u &&
+            result.out_read;
+        std::fprintf(stderr,
+                     "%s: rc=%d WMMA=%llu K64=%llu write=%d read=%d %s\n",
+                     label, result.rc,
+                     (unsigned long long)result.wmma_calls,
+                     (unsigned long long)result.k64_calls,
+                     result.write_ok ? 1 : 0,
+                     result.out_read ? 1 : 0,
+                     ready ? "PASS" : "FAIL");
+        if (result.out_read) {
+            const std::string body = std::string(label) + " body";
+            const std::string finite = std::string(label) + " finite";
+            const std::string guard = std::string(label) + " N-tail";
+            ready = output_body_overwritten(
+                out_host, out_sentinel, out_count, body.c_str()) && ready;
+            ready = output_body_finite(
+                out_host, out_count, finite.c_str()) && ready;
+            ready = output_guard_unchanged(
+                out_host, out_sentinel, out_count, guard.c_str()) && ready;
+        }
+        return ready;
+    };
+
+    ds4_gpu_set_ssd_streaming(false);
+
+    /* Forced TILE8 is the exact baseline. */
+    clear_controls();
+    (void)setenv(kPrefillRequire, "1", 1);
+    (void)setenv(kPrefillWmmaDisable, "1", 1);
+    std::vector<float> tile8_low_host(low_sentinel.size());
+    std::vector<float> tile8_out_host(out_sentinel.size());
+    const batch_result tile8_result = run_batch(
+        tile8_low.ptr, tile8_out.ptr, &tile8_low_host, &tile8_out_host);
+
+    /* The production default keeps validated WMMA on A and exact TILE8 on B. */
+    clear_controls();
+    std::vector<float> default_low_host(low_sentinel.size());
+    std::vector<float> default_out_host(out_sentinel.size());
+    const batch_result default_result = run_batch(
+        candidate_low.ptr, candidate_out.ptr,
+        &default_low_host, &default_out_host);
+
+    /* ENABLE opts only A into direct WMMA.  Pin K32 as the rollback arm. */
+    clear_controls();
+    (void)setenv(kPrefillWmmaEnable, "1", 1);
+    (void)setenv(kPrefillWmmaK64, "0", 1);
+    std::vector<float> k32_low_host(low_sentinel.size());
+    std::vector<float> k32_out_host(out_sentinel.size());
+    const batch_result k32_result = run_batch(
+        candidate_low.ptr, candidate_out.ptr, &k32_low_host, &k32_out_host);
+
+    std::vector<float> k32_replay_host(out_sentinel.size());
+    replay_result k32_replay_result;
+    if (k32_result.rc == 1 && k32_result.low_read) {
+        k32_replay_result = run_tile8_replay(
+            candidate_low.ptr, &k32_replay_host);
+    }
+
+    /* Run K64 regardless of every earlier comparison result. */
+    clear_controls();
+    (void)setenv(kPrefillWmmaEnable, "1", 1);
+    std::vector<float> k64_low_host(low_sentinel.size());
+    std::vector<float> k64_out_host(out_sentinel.size());
+    const batch_result k64_result = run_batch(
+        candidate_low.ptr, candidate_out.ptr, &k64_low_host, &k64_out_host);
+
+    /* REQUIRE is the only policy that opts both A and B into direct WMMA. */
+    clear_controls();
+    (void)setenv(kPrefillWmmaRequire, "1", 1);
+    (void)setenv(kPrefillWmmaK64, "0", 1);
+    std::vector<float> strict_k32_low_host(low_sentinel.size());
+    std::vector<float> strict_k32_out_host(out_sentinel.size());
+    const batch_result strict_k32_result = run_batch(
+        candidate_low.ptr, candidate_out.ptr,
+        &strict_k32_low_host, &strict_k32_out_host);
+
+    clear_controls();
+    (void)setenv(kPrefillWmmaRequire, "1", 1);
+    std::vector<float> strict_k64_low_host(low_sentinel.size());
+    std::vector<float> strict_k64_out_host(out_sentinel.size());
+    const batch_result strict_k64_result = run_batch(
+        candidate_low.ptr, candidate_out.ptr,
+        &strict_k64_low_host, &strict_k64_out_host);
+
+    const bool tile8_ready = batch_ready(
+        tile8_result, 0u, 0u, "production forced TILE8");
+    const bool default_ready = batch_ready(
+        default_result, 1u, 1u,
+        "production clean-default A-K64/B-TILE8");
+    const bool k32_ready = batch_ready(
+        k32_result, 1u, 0u, "production ENABLE A-K32/B-TILE8");
+    const bool k64_ready = batch_ready(
+        k64_result, 1u, 1u, "production ENABLE A-K64/B-TILE8");
+    const bool strict_k32_ready = batch_ready(
+        strict_k32_result, 2u, 0u, "production REQUIRE A+B K32");
+    const bool strict_k64_ready = batch_ready(
+        strict_k64_result, 2u, 2u, "production REQUIRE A+B K64");
+
+    const bool tile8_memory = batch_memory_ok(
+        tile8_result, tile8_low_host, tile8_out_host,
+        "production forced TILE8");
+    const bool default_memory = batch_memory_ok(
+        default_result, default_low_host, default_out_host,
+        "production clean-default A-K64/B-TILE8");
+    const bool k32_memory = batch_memory_ok(
+        k32_result, k32_low_host, k32_out_host,
+        "production ENABLE A-K32/B-TILE8");
+    const bool k64_memory = batch_memory_ok(
+        k64_result, k64_low_host, k64_out_host,
+        "production ENABLE A-K64/B-TILE8");
+    const bool strict_k32_memory = batch_memory_ok(
+        strict_k32_result, strict_k32_low_host, strict_k32_out_host,
+        "production REQUIRE A+B K32");
+    const bool strict_k64_memory = batch_memory_ok(
+        strict_k64_result, strict_k64_low_host, strict_k64_out_host,
+        "production REQUIRE A+B K64");
+    const bool k32_replay_ready = replay_ready(
+        k32_replay_result, k32_replay_host,
+        "production K32-low standalone B-TILE8 replay");
+
+    tile8_low_host.resize(low_count);
+    tile8_out_host.resize(out_count);
+    default_low_host.resize(low_count);
+    default_out_host.resize(out_count);
+    k32_low_host.resize(low_count);
+    k32_out_host.resize(out_count);
+    k32_replay_host.resize(out_count);
+    k64_low_host.resize(low_count);
+    k64_out_host.resize(out_count);
+    strict_k32_low_host.resize(low_count);
+    strict_k32_out_host.resize(out_count);
+    strict_k64_low_host.resize(low_count);
+    strict_k64_out_host.resize(out_count);
+
+    bool comparisons_ok = true;
+    if (tile8_result.low_read && default_result.low_read) {
+        comparisons_ok = close_with_tolerance(
+            default_low_host, tile8_low_host,
+            2.0f, 3.0e-2f,
+            "production clean-default A-WMMA vs forced TILE8") &&
+            comparisons_ok;
+    } else {
+        comparisons_ok = false;
+    }
+    if (tile8_result.out_read && default_result.out_read) {
+        comparisons_ok = close_with_tolerance(
+            default_out_host, tile8_out_host,
+            16.0f, 8.0e-2f,
+            "production clean-default A-WMMA/B-TILE8 vs all-TILE8",
+            false) && comparisons_ok;
+    } else {
+        comparisons_ok = false;
+    }
+    if (tile8_result.low_read && k32_result.low_read) {
+        comparisons_ok = close_with_tolerance(
+            k32_low_host, tile8_low_host, 2.0f, 3.0e-2f,
+            "production ENABLE output-A K32 vs TILE8") && comparisons_ok;
+    } else {
+        comparisons_ok = false;
+    }
+    if (k32_result.out_read && k32_replay_result.out_read) {
+        comparisons_ok = bitwise_equal(
+            k32_out_host, k32_replay_host,
+            "production ENABLE K32 B vs standalone TILE8 replay") &&
+            comparisons_ok;
+    } else {
+        comparisons_ok = false;
+    }
+    if (k32_result.low_read && k64_result.low_read) {
+        comparisons_ok = bitwise_equal(
+            k64_low_host, k32_low_host,
+            "production ENABLE output-A K64 vs K32") && comparisons_ok;
+    } else {
+        comparisons_ok = false;
+    }
+    if (k32_result.out_read && k64_result.out_read) {
+        comparisons_ok = bitwise_equal(
+            k64_out_host, k32_out_host,
+            "production ENABLE A-K64/B-TILE8 vs A-K32/B-TILE8") &&
+            comparisons_ok;
+    } else {
+        comparisons_ok = false;
+    }
+    if (default_result.low_read && k64_result.low_read) {
+        comparisons_ok = bitwise_equal(
+            default_low_host, k64_low_host,
+            "production clean-default A vs explicit ENABLE A-K64") &&
+            comparisons_ok;
+    } else {
+        comparisons_ok = false;
+    }
+    if (default_result.out_read && k64_result.out_read) {
+        comparisons_ok = bitwise_equal(
+            default_out_host, k64_out_host,
+            "production clean-default B vs explicit ENABLE B-TILE8") &&
+            comparisons_ok;
+    } else {
+        comparisons_ok = false;
+    }
+    if (strict_k32_result.low_read && strict_k64_result.low_read) {
+        comparisons_ok = bitwise_equal(
+            strict_k64_low_host, strict_k32_low_host,
+            "production REQUIRE output-A K64 vs K32") && comparisons_ok;
+    } else {
+        comparisons_ok = false;
+    }
+    if (strict_k32_result.out_read && strict_k64_result.out_read) {
+        comparisons_ok = bitwise_equal(
+            strict_k64_out_host, strict_k32_out_host,
+            "production REQUIRE output A+B K64 vs K32") && comparisons_ok;
+    } else {
+        comparisons_ok = false;
+    }
+
+    const bool ok = tile8_ready && default_ready && k32_ready && k64_ready &&
+        strict_k32_ready && strict_k64_ready && tile8_memory &&
+        default_memory && k32_memory && k64_memory && strict_k32_memory &&
+        strict_k64_memory && k32_replay_ready && comparisons_ok;
+    std::fprintf(stderr,
+                 "ROCm Q4 production output WMMA policy: "
+                 "tile8=%d/%llu/%llu default=%d/%llu/%llu "
+                 "enable-k32=%d/%llu/%llu enable-k64=%d/%llu/%llu "
+                 "require-k32=%d/%llu/%llu require-k64=%d/%llu/%llu %s\n",
+                 tile8_result.rc,
+                 (unsigned long long)tile8_result.wmma_calls,
+                 (unsigned long long)tile8_result.k64_calls,
+                 default_result.rc,
+                 (unsigned long long)default_result.wmma_calls,
+                 (unsigned long long)default_result.k64_calls,
+                 k32_result.rc,
+                 (unsigned long long)k32_result.wmma_calls,
+                 (unsigned long long)k32_result.k64_calls,
+                 k64_result.rc,
+                 (unsigned long long)k64_result.wmma_calls,
+                 (unsigned long long)k64_result.k64_calls,
+                 strict_k32_result.rc,
+                 (unsigned long long)strict_k32_result.wmma_calls,
+                 (unsigned long long)strict_k32_result.k64_calls,
+                 strict_k64_result.rc,
+                 (unsigned long long)strict_k64_result.wmma_calls,
+                 (unsigned long long)strict_k64_result.k64_calls,
+                 ok ? "PASS" : "FAIL");
+    return ok;
+}
+
+int detect_rocm_device() {
+#if DS4_TEST_HAS_HIP_RUNTIME
+    int count = 0;
+    const hipError_t err = hipGetDeviceCount(&count);
+    if (err != hipSuccess || count <= 0) {
+        std::fprintf(stderr,
+                     "ROCm Q4 dense/pair/prefill: SKIP "
+                     "(HIP runtime has no visible device: %s)\n",
+                     err == hipSuccess ? "device count is zero" : hipGetErrorString(err));
+        return 0;
+    }
+    return count;
+#else
+    std::fprintf(stderr,
+                 "ROCm Q4 dense/pair/prefill: SKIP "
+                 "(compiled without HIP runtime headers)\n");
+    return 0;
+#endif
+}
+
+}  // namespace
+
+int main(int argc, char **argv) {
+    bool run_dense = true;
+    bool run_pair = true;
+    bool run_prefill = true;
+    bool run_grouped_decode = true;
+    bool run_prefill_long = false;
+    bool run_policy_only = false;
+    if (argc == 2 && std::strcmp(argv[1], "--dense") == 0) {
+        run_pair = false;
+        run_prefill = false;
+        run_grouped_decode = false;
+    } else if (argc == 2 && std::strcmp(argv[1], "--pair") == 0) {
+        run_dense = false;
+        run_prefill = false;
+        run_grouped_decode = false;
+    } else if (argc == 2 && std::strcmp(argv[1], "--prefill") == 0) {
+        run_dense = false;
+        run_pair = false;
+        run_grouped_decode = false;
+    } else if (argc == 2 &&
+               std::strcmp(argv[1], "--grouped-decode") == 0) {
+        run_dense = false;
+        run_pair = false;
+        run_prefill = false;
+    } else if (argc == 2 &&
+               std::strcmp(argv[1], "--prefill-long") == 0) {
+        run_dense = false;
+        run_pair = false;
+        run_grouped_decode = false;
+        run_prefill_long = true;
+    } else if (argc == 2 && std::strcmp(argv[1], "--policy") == 0) {
+        run_dense = false;
+        run_pair = false;
+        run_prefill = false;
+        run_grouped_decode = false;
+        run_policy_only = true;
+    } else if (argc > 1 &&
+               !(argc == 2 && std::strcmp(argv[1], "--all") == 0)) {
+        std::fprintf(stderr,
+                     "usage: %s [--all|--dense|--pair|--grouped-decode|"
+                     "--prefill|--prefill-long|--policy]\n",
+                     argv[0]);
+        return 2;
+    }
+
+    env_snapshot prefill_enable(kPrefillEnable);
+    env_snapshot prefill_disable(kPrefillDisable);
+    env_snapshot prefill_require(kPrefillRequire);
+    env_snapshot tile4_ssd_enable(kPrefillK1024Tile4SsdEnable);
+    env_snapshot tile4_disable(kPrefillK1024Tile4Disable);
+    env_snapshot tile4_require(kPrefillK1024Tile4Require);
+    env_snapshot wmma_enable(kPrefillWmmaEnable);
+    env_snapshot wmma_ssd_enable(kPrefillWmmaSsdEnable);
+    env_snapshot wmma_disable(kPrefillWmmaDisable);
+    env_snapshot wmma_require(kPrefillWmmaRequire);
+    env_snapshot wmma_row_tile(kPrefillWmmaRowTile);
+    env_snapshot wmma_k64_global(kPrefillWmmaK64);
+    env_snapshot wmma_k128_disable_global(kPrefillWmmaK128Disable);
+    env_snapshot q8_wave32_enable(kPrefillQ8Wave32Enable);
+    env_snapshot q8_wave32_disable(kPrefillQ8Wave32Disable);
+    env_snapshot q8_wave32_require(kPrefillQ8Wave32Require);
+    env_snapshot grouped_enable(kGroupedDecodeEnable);
+    env_snapshot grouped_disable(kGroupedDecodeDisable);
+    env_snapshot grouped_require(kGroupedDecodeRequire);
+    env_snapshot grouped_stats(kGroupedDecodeStats);
+    (void)unsetenv(kPrefillEnable);
+    (void)unsetenv(kPrefillDisable);
+    (void)unsetenv(kPrefillRequire);
+    (void)unsetenv(kPrefillK1024Tile4SsdEnable);
+    (void)unsetenv(kPrefillK1024Tile4Disable);
+    (void)unsetenv(kPrefillK1024Tile4Require);
+    (void)unsetenv(kPrefillWmmaEnable);
+    (void)unsetenv(kPrefillWmmaSsdEnable);
+    (void)unsetenv(kPrefillWmmaDisable);
+    (void)unsetenv(kPrefillWmmaRequire);
+    (void)unsetenv(kPrefillWmmaRowTile);
+    /* Neutralize wider K64/K128 staging for every non-staging oracle. */
+    (void)setenv(kPrefillWmmaK64, "0", 1);
+    (void)unsetenv(kPrefillWmmaK128Disable);
+    (void)unsetenv(kPrefillQ8Wave32Enable);
+    (void)unsetenv(kPrefillQ8Wave32Disable);
+    (void)unsetenv(kPrefillQ8Wave32Require);
+    (void)unsetenv(kGroupedDecodeEnable);
+    (void)unsetenv(kGroupedDecodeDisable);
+    (void)unsetenv(kGroupedDecodeRequire);
+    (void)unsetenv(kGroupedDecodeStats);
+
+    const bool policy_ok = run_prefill_wmma_requested_policy_oracle() &&
+                           run_prefill_wmma_row_tile_policy_oracle() &&
+                           run_prefill_k1024_tile4_policy_oracle() &&
+                           run_prefill_q8_wave32_policy_oracle() &&
+                           run_pair_pre_enqueue_policy_oracle();
+    if (run_policy_only || !policy_ok) return policy_ok ? 0 : 1;
+
+    if (detect_rocm_device() <= 0) {
+        const char *require_device =
+            std::getenv("DS4_TEST_REQUIRE_ROCM_DEVICE");
+        const bool required = require_device && require_device[0] != '\0' &&
+                              std::strcmp(require_device, "0") != 0;
+        return required ? 1 : kSkip;
+    }
+    if (!ds4_gpu_init()) {
+        std::fprintf(stderr,
+                     "ROCm Q4 dense/pair/prefill: FAIL "
+                     "(device is visible but ds4_gpu_init failed)\n");
+        return 1;
+    }
+
+    aligned_model model;
+    bool ok = policy_ok && make_model(&model);
+    if (!ok) {
+        std::fprintf(stderr,
+                     "ROCm Q4 dense/pair/prefill: fixture allocation FAIL\n");
+    } else if (!ds4_gpu_set_model_map(model.data, model.size)) {
+        std::fprintf(stderr,
+                     "ROCm Q4 dense/pair/prefill: model-map registration FAIL\n");
+        ok = false;
+    }
+
+    const bool model_ready = ok;
+    if (model_ready && run_dense) {
+        std::fprintf(stderr, "ROCm Q4 dense oracle (raw GGUF Q4_K x Q8_K):\n");
+        ok = run_dense_case(model, 1u, model.weight0_offset, kM0,
+                            "dense n_tok=1") && ok;
+        ok = run_dense_case(model, 3u, model.weight1_offset, kM1,
+                            "dense n_tok=3") && ok;
+        ok = run_dense_case(model, 9u, model.weight1_offset, kM1,
+                            "dense n_tok=9") && ok;
+        ok = run_dense_case(model, 128u, model.weight1_offset, kM1,
+                            "dense n_tok=128") && ok;
+        ok = run_dense_guards(model) && ok;
+    }
+    if (model_ready && run_pair) {
+        std::fprintf(stderr, "ROCm Q4 pair parity (pair vs two dense):\n");
+        env_snapshot enable("DS4_ROCM_ENABLE_Q4_DENSE_PAIR");
+        env_snapshot disable("DS4_ROCM_DISABLE_Q4_DENSE_PAIR");
+        (void)setenv("DS4_ROCM_ENABLE_Q4_DENSE_PAIR", "1", 1);
+        (void)unsetenv("DS4_ROCM_DISABLE_Q4_DENSE_PAIR");
+        const bool pair1_ok = run_pair_case(model, 1u, "pair n_tok=1");
+        const bool pair1_tail_ok = run_pair_case(
+            model, 1u, "pair K=1024 n_tok=1", false, kTailK);
+        const bool pair3_ok = run_pair_case(
+            model, 3u, "pair n_tok=3 reverse M=(33,65)", true);
+        const bool pair8_ok = run_pair_case(model, 8u, "pair n_tok=8");
+        const bool pair_guard_ok = run_pair_guards(model);
+        const bool pair_opt_in_ok = run_pair_opt_in_guards(model);
+        ok = pair1_ok && pair1_tail_ok && pair3_ok && pair8_ok && pair_guard_ok &&
+             pair_opt_in_ok && ok;
+    }
+    if (model_ready && run_grouped_decode) {
+        std::fprintf(stderr,
+                     "ROCm Q4 grouped attention-A decode parity "
+                     "(one grouped dispatch vs eight legacy calls):\n");
+        ok = run_grouped_attention_decode_case(model) && ok;
+    }
+    if (model_ready && run_prefill) {
+        std::fprintf(stderr,
+                     "ROCm Q4 tiled prefill parity "
+                     "(forced DISABLE vs default+REQUIRE):\n");
+        const bool q8_wave32_ok = run_prefill_q8_wave32_oracle(model);
+        const bool prefill9_ok = run_prefill_parity_case(
+            model, 9u, model.weight0_offset, kM0, true,
+            "prefill K=4096 M=65 n_tok=9");
+        const bool prefill30_ok = run_prefill_parity_case(
+            model, 30u, model.weight0_offset, kM0, true,
+            "prefill K=4096 M=65 n_tok=30 (token-tail nt=6)");
+        const bool prefill128_ok = run_prefill_parity_case(
+            model, 128u, model.weight1_offset, kM1, true,
+            "prefill K=4096 M=33 n_tok=128");
+        const bool prefill_tail9_ok = run_prefill_parity_case(
+            model, 9u, model.tail_k1024_offset, kM0, true,
+            "prefill K=1024 M=65 n_tok=9 (K-tail nb=4)", kTailK);
+        const bool prefill_tail128_ok = run_prefill_parity_case(
+            model, 128u, model.tail_k1024_offset, kM0, true,
+            "prefill K=1024 M=65 n_tok=128 (K-tail nb=4)", kTailK);
+        const bool prefill_q_b_tile4_ok = run_prefill_parity_case(
+            model, 9u, model.q_b_k1024_offset, kQbOutDim, false,
+            "prefill q_b K=1024 M=32768 n_tok=9 (tile4)", kTailK);
+        const bool q_b_f16_null_qhalf_ok =
+            run_q_b_f16_null_qhalf_case(model);
+        const bool prefill_single9_ok = run_prefill_parity_case(
+            model, 9u, model.attn_b_offset, kAttnOutDim, true,
+            "prefill K=256 M=65 n_tok=9 (K-tail nb=1)", kAttnLowDim);
+        const bool prefill_single128_ok = run_prefill_parity_case(
+            model, 128u, model.attn_b_offset, kAttnOutDim, true,
+            "prefill K=256 M=65 n_tok=128 (K-tail nb=1)", kAttnLowDim);
+        const bool prefill_pair9_ok =
+            run_prefill_pair_case(model, 9u, false, kTailK);
+        const bool prefill_pair30_reverse_ok =
+            run_prefill_pair_case(model, 30u, true);
+        const bool prefill_pair128_ok =
+            run_prefill_pair_case(model, 128u, false);
+        const bool attention9_ok = run_attention_prefill_case(
+            model, 9u,
+            "attention prefill groups=8 K=4096 rank=32 M=65 n_tok=9");
+        const bool attention30_ok = run_attention_prefill_case(
+            model, 30u,
+            "attention prefill groups=8 K=4096 rank=32 M=65 "
+            "n_tok=30 (token-tail nt=6)");
+        const bool attention128_ok = run_attention_prefill_case(
+            model, 128u,
+            "attention prefill groups=8 K=4096 rank=32 M=65 n_tok=128");
+        const bool attention_q8_9_ok = run_attention_prefill_case(
+            model, 9u,
+            "attention prefill Q4-A/Q8-B groups=8 K=4096 rank=32 M=65 "
+            "n_tok=9",
+            kQ8Type);
+        const bool attention_q8_30_ok = run_attention_prefill_case(
+            model, 30u,
+            "attention prefill Q4-A/Q8-B groups=8 K=4096 rank=32 M=65 "
+            "n_tok=30 (token-tail nt=6)",
+            kQ8Type);
+        const bool prefill_wmma_ok = run_prefill_wmma_smoke(model);
+        const bool output_wmma_ok =
+            run_attention_output_wmma_smoke(model);
+        const bool gate_ok = run_prefill_gate_guards(model);
+        ok = q8_wave32_ok && prefill9_ok && prefill30_ok && prefill128_ok &&
+             prefill_tail9_ok &&
+             prefill_tail128_ok && prefill_single9_ok &&
+             prefill_q_b_tile4_ok && q_b_f16_null_qhalf_ok &&
+             prefill_single128_ok && prefill_pair9_ok &&
+             prefill_pair30_reverse_ok && prefill_pair128_ok && attention9_ok &&
+             attention30_ok && attention128_ok && attention_q8_9_ok &&
+             attention_q8_30_ok && prefill_wmma_ok && output_wmma_ok &&
+             gate_ok && ok;
+        if (run_prefill_long) {
+            // A 64 MiB activation and a roughly 0.5 Gi-op projection stress
+            // arbitrary token-grid tails without the much slower CPU oracle.
+            const bool long_ok = run_prefill_parity_case(
+                model, 4096u, model.weight1_offset, kM1, false,
+                "prefill stress K=4096 M=33 n_tok=4096");
+            ok = long_ok && ok;
+        }
+        /* Run the file-backed SSD oracle last.  Switching model maps marks the
+         * process multi-model and intentionally disables optional caches; no
+         * later parity case should inherit that conservative policy. */
+        const bool prefill_q_b_tile4_ssd_ok =
+            run_prefill_k1024_tile4_ssd_case(model);
+        ok = prefill_q_b_tile4_ssd_ok && ok;
+    }
+
+    // Registered host ranges must be released before their aligned backing
+    // allocation is destroyed.
+    ds4_gpu_cleanup();
+    std::fprintf(stderr, "ROCm Q4 dense/pair/prefill oracle: %s\n",
+                 ok ? "PASS" : "FAIL");
+    return ok ? 0 : 1;
+}