From 4e6ba97933634889c81f28a85f440e6f4518edf9 Mon Sep 17 00:00:00 2001 From: Nathan Wilson Date: Thu, 27 Aug 2026 03:00:24 +0000 Subject: [PATCH 1/6] qwen4exp: load NextN/MTP draft tensors (sidecar and in-file) The qwen4exp converter drops the MTP head, but community sidecar drafters now exist (agentionai's Qwen3.8-Flash-Next-MTP-Q8_0: arch qwen4exp, block_count 49, nextn_predict_layers 1, upstream tensor naming, hc_norm gammas verified (1+w)-folded against the raw HF checkpoint by range read - exact +1.0 offset). Follows the deepseek4 pattern: optional NEXTN_PREDICT_LAYERS KV with a tensor-presence downgrade, mtp_only detection when the trunk is absent, trunk tensors TENSOR_NOT_REQUIRED in that case, NextN tensors TENSOR_SKIP unless the context requests them. qwen4exp-specific differences from deepseek4, both dictated by the file format: - the NextN block is always a full-attention QSA layer (gated attention, own nh=4 indexer, full 512-expert MoE, both HC pairs); is_recr() is derived from full_attention_interval and would misclassify blk.48 as linear attention, so the MTP layer forces the full-attention branch. - nextn.hnorm is hc-space ({hc_dim}, 10240), not {n_embd}: the draft head consumes the target's 4-stream hyper-connection state. A standalone load of a sidecar now aborts with a clear message instead of segfaulting in the first hc_mix (the trunk tensors are null by design; the mainline graph cannot be built from a drafter file). Draft graph and --spec-type draft-mtp wiring are the follow-up commit; this one is loader-only and mainline GGUFs are unaffected (KV absent -> n_layer_nextn 0, identical load path). Co-Authored-By: Claude Opus 5 (cherry picked from commit 23d8b2c45881585d16db045eb80ea5d81ec8f736) (cherry picked from commit 354390810c690ce84eade56633f868124f44c7ed) --- src/models/qwen4exp.cpp | 127 ++++++++++++++++++++++++++-------------- 1 file changed, 84 insertions(+), 43 deletions(-) diff --git a/src/models/qwen4exp.cpp b/src/models/qwen4exp.cpp index abf6a0502fb..921b7b21e0b 100644 --- a/src/models/qwen4exp.cpp +++ b/src/models/qwen4exp.cpp @@ -7,6 +7,19 @@ #include void llama_model_qwen4exp::load_arch_hparams(llama_model_loader & ml) { + // NextN/MTP draft head, the deepseek4 pattern: the KV is optional and a missing tensor + // downgrades it, so mainline GGUFs (whose converter drops the head) load unchanged and + // a sidecar drafter (mtp-only file, e.g. blk.48 for the 48-layer model) is recognised. + ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false); + if (hparams.n_layer_nextn > 0 && hparams.n_layer_nextn < hparams.n_layer_all) { + const uint32_t n_layer_main = hparams.n_layer_all - hparams.n_layer_nextn; + const std::string mtp_probe = "blk." + std::to_string(n_layer_main) + ".nextn.eh_proj.weight"; + if (ml.get_weight(mtp_probe.c_str()) == nullptr) { + hparams.n_layer_nextn = 0; + } + } + GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < block_count"); + ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp, false); ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, false); ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); @@ -112,6 +125,14 @@ void llama_model_qwen4exp::load_arch_tensors(llama_model_loader & ml) { const int64_t hc_dim = hc * n_embd; const int64_t hc_lr = hparams.hc_low_rank; + // A sidecar drafter file carries ONLY the NextN block plus the shared head/embedding + // (deepseek4's DSpark shape): trunk tensors become optional there, and the NextN tensors + // load only when the context asked for them. + const uint32_t n_layer_main = hparams.n_layer_all - hparams.n_layer_nextn; + const bool mtp_only = (hparams.n_layer_nextn > 0) && (ml.get_weight("blk.0.attn_norm.weight") == nullptr || ml.get_weight("blk.0.hc_attn_norm.weight") == nullptr); + const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0; + const int mtp_flags = ml.load_mtp ? 0 : TENSOR_SKIP; + tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab }, 0); // there is no output_norm: the final hyper-connection mixer carries it @@ -140,9 +161,12 @@ void llama_model_qwen4exp::load_arch_tensors(llama_model_loader & ml) { { hparams.ple_head_dim, ple_rows }, TENSOR_READ_LAZY); } - for (int il = 0; il < n_layer; ++il) { + for (int il = 0; il < (int) hparams.n_layer_all; ++il) { auto & layer = layers[il]; + const bool is_mtp_layer = il >= (int) n_layer_main; + const int flags = is_mtp_layer ? mtp_flags : trunk_flags; + const int64_t n_ff_exp = hparams.n_ff_exp ? hparams.n_ff_exp : n_ff / n_expert_used; const int64_t n_ff_shexp = hparams.n_ff_shexp ? hparams.n_ff_shexp : n_ff; @@ -155,57 +179,68 @@ void llama_model_qwen4exp::load_arch_tensors(llama_model_loader & ml) { const int64_t conv_dim = key_dim * 2 + value_dim; // two HC modules per layer: before the token mixer, before the MoE - layer.hc_attn_norm = create_tensor(tn(LLM_TENSOR_HC_ATTN_NORM, "weight", il), { hc_dim }, 0); - layer.hc_attn_down = create_tensor(tn(LLM_TENSOR_HC_ATTN_DOWN, "weight", il), { hc_dim, hc_lr }, 0); - layer.hc_attn_up = create_tensor(tn(LLM_TENSOR_HC_ATTN_UP, "weight", il), { hc_lr, hc_dim }, 0); - layer.hc_attn_inject = create_tensor(tn(LLM_TENSOR_HC_ATTN_INJECT, "weight", il), { hc_dim, hc }, 0); - layer.hc_ffn_norm = create_tensor(tn(LLM_TENSOR_HC_FFN_NORM, "weight", il), { hc_dim }, 0); - layer.hc_ffn_down = create_tensor(tn(LLM_TENSOR_HC_FFN_DOWN, "weight", il), { hc_dim, hc_lr }, 0); - layer.hc_ffn_up = create_tensor(tn(LLM_TENSOR_HC_FFN_UP, "weight", il), { hc_lr, hc_dim }, 0); - layer.hc_ffn_inject = create_tensor(tn(LLM_TENSOR_HC_FFN_INJECT, "weight", il), { hc_dim, hc }, 0); - - if (!hparams.is_recr(il)) { + layer.hc_attn_norm = create_tensor(tn(LLM_TENSOR_HC_ATTN_NORM, "weight", il), { hc_dim }, flags); + layer.hc_attn_down = create_tensor(tn(LLM_TENSOR_HC_ATTN_DOWN, "weight", il), { hc_dim, hc_lr }, flags); + layer.hc_attn_up = create_tensor(tn(LLM_TENSOR_HC_ATTN_UP, "weight", il), { hc_lr, hc_dim }, flags); + layer.hc_attn_inject = create_tensor(tn(LLM_TENSOR_HC_ATTN_INJECT, "weight", il), { hc_dim, hc }, flags); + layer.hc_ffn_norm = create_tensor(tn(LLM_TENSOR_HC_FFN_NORM, "weight", il), { hc_dim }, flags); + layer.hc_ffn_down = create_tensor(tn(LLM_TENSOR_HC_FFN_DOWN, "weight", il), { hc_dim, hc_lr }, flags); + layer.hc_ffn_up = create_tensor(tn(LLM_TENSOR_HC_FFN_UP, "weight", il), { hc_lr, hc_dim }, flags); + layer.hc_ffn_inject = create_tensor(tn(LLM_TENSOR_HC_FFN_INJECT, "weight", il), { hc_dim, hc }, flags); + + // the NextN block is always a full-attention QSA layer; is_recr() is derived from + // full_attention_interval and would misclassify it + if (is_mtp_layer || !hparams.is_recr(il)) { // full attention: wq holds [q|gate] interleaved per head - create_tensor_qkv(layer, il, n_embd, n_embd_head_k * n_head * 2, n_embd_k_gqa, n_embd_v_gqa, 0); - layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", il), { n_embd_head_k * n_head, n_embd }, 0); + create_tensor_qkv(layer, il, n_embd, n_embd_head_k * n_head * 2, n_embd_k_gqa, n_embd_v_gqa, flags); + layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", il), { n_embd_head_k * n_head, n_embd }, flags); - layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", il), { n_embd_head_k }, 0); - layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", il), { n_embd_head_k }, 0); + layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", il), { n_embd_head_k }, flags); + layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", il), { n_embd_head_k }, flags); const int64_t idx_dim = hparams.indexer_head_size; - layer.index_q_proj = create_tensor(tn(LLM_TENSOR_INDEXER_Q_PROJ, "weight", il), { n_embd, hparams.indexer_n_head * idx_dim }, 0); - layer.index_k_proj = create_tensor(tn(LLM_TENSOR_INDEXER_K_PROJ, "weight", il), { n_embd, idx_dim }, 0); - layer.index_q_norm = create_tensor(tn(LLM_TENSOR_INDEXER_Q_NORM, "weight", il), { idx_dim }, 0); - layer.index_k_norm = create_tensor(tn(LLM_TENSOR_INDEXER_K_NORM, "weight", il), { idx_dim }, 0); + layer.index_q_proj = create_tensor(tn(LLM_TENSOR_INDEXER_Q_PROJ, "weight", il), { n_embd, hparams.indexer_n_head * idx_dim }, flags); + layer.index_k_proj = create_tensor(tn(LLM_TENSOR_INDEXER_K_PROJ, "weight", il), { n_embd, idx_dim }, flags); + layer.index_q_norm = create_tensor(tn(LLM_TENSOR_INDEXER_Q_NORM, "weight", il), { idx_dim }, flags); + layer.index_k_norm = create_tensor(tn(LLM_TENSOR_INDEXER_K_NORM, "weight", il), { idx_dim }, flags); } else { - layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", il), { n_embd, key_dim * 2 + value_dim }, 0); - layer.wqkv_gate = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "weight", il), { n_embd, value_dim }, 0); - layer.ssm_conv1d = create_tensor(tn(LLM_TENSOR_SSM_CONV1D, "weight", il), { hparams.ssm_d_conv, conv_dim }, 0); - layer.ssm_dt = create_tensor(tn(LLM_TENSOR_SSM_DT, "bias", il), { hparams.ssm_dt_rank }, 0); - layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A_NOSCAN, il), { hparams.ssm_dt_rank }, 0); - layer.ssm_beta = create_tensor(tn(LLM_TENSOR_SSM_BETA, "weight", il), { n_embd, n_v_heads }, 0); - layer.ssm_alpha = create_tensor(tn(LLM_TENSOR_SSM_ALPHA, "weight", il), { n_embd, n_v_heads }, 0); - layer.ssm_norm = create_tensor(tn(LLM_TENSOR_SSM_NORM, "weight", il), { head_v_dim }, 0); - layer.ssm_out = create_tensor(tn(LLM_TENSOR_SSM_OUT, "weight", il), { value_dim, n_embd }, 0); + layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", il), { n_embd, key_dim * 2 + value_dim }, flags); + layer.wqkv_gate = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "weight", il), { n_embd, value_dim }, flags); + layer.ssm_conv1d = create_tensor(tn(LLM_TENSOR_SSM_CONV1D, "weight", il), { hparams.ssm_d_conv, conv_dim }, flags); + layer.ssm_dt = create_tensor(tn(LLM_TENSOR_SSM_DT, "bias", il), { hparams.ssm_dt_rank }, flags); + layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A_NOSCAN, il), { hparams.ssm_dt_rank }, flags); + layer.ssm_beta = create_tensor(tn(LLM_TENSOR_SSM_BETA, "weight", il), { n_embd, n_v_heads }, flags); + layer.ssm_alpha = create_tensor(tn(LLM_TENSOR_SSM_ALPHA, "weight", il), { n_embd, n_v_heads }, flags); + layer.ssm_norm = create_tensor(tn(LLM_TENSOR_SSM_NORM, "weight", il), { head_v_dim }, flags); + layer.ssm_out = create_tensor(tn(LLM_TENSOR_SSM_OUT, "weight", il), { value_dim, n_embd }, flags); } - if (hparams.is_ple(il)) { - layer.ple_key = create_tensor(tn(LLM_TENSOR_PLE_KEY, "weight", il), { n_embd, hc_dim }, 0); - layer.ple_value = create_tensor(tn(LLM_TENSOR_PLE_VALUE, "weight", il), { n_embd, n_embd }, 0); - layer.ple_norm_key = create_tensor(tn(LLM_TENSOR_PLE_NORM_KEY, "weight", il), { hc_dim }, 0); - layer.ple_norm_query = create_tensor(tn(LLM_TENSOR_PLE_NORM_QUERY, "weight", il), { hc_dim }, 0); - layer.ple_norm_conv = create_tensor(tn(LLM_TENSOR_PLE_NORM_CONV, "weight", il), { hc_dim }, 0); - layer.ple_conv1d = create_tensor(tn(LLM_TENSOR_PLE_CONV1D, "weight", il), { hparams.ple_conv_kernel, hc_dim }, 0); + if (!is_mtp_layer && hparams.is_ple(il)) { + layer.ple_key = create_tensor(tn(LLM_TENSOR_PLE_KEY, "weight", il), { n_embd, hc_dim }, flags); + layer.ple_value = create_tensor(tn(LLM_TENSOR_PLE_VALUE, "weight", il), { n_embd, n_embd }, flags); + layer.ple_norm_key = create_tensor(tn(LLM_TENSOR_PLE_NORM_KEY, "weight", il), { hc_dim }, flags); + layer.ple_norm_query = create_tensor(tn(LLM_TENSOR_PLE_NORM_QUERY, "weight", il), { hc_dim }, flags); + layer.ple_norm_conv = create_tensor(tn(LLM_TENSOR_PLE_NORM_CONV, "weight", il), { hc_dim }, flags); + layer.ple_conv1d = create_tensor(tn(LLM_TENSOR_PLE_CONV1D, "weight", il), { hparams.ple_conv_kernel, hc_dim }, flags); } - layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", il), { n_embd, n_expert }, 0); - layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", il), { n_ff_exp, n_embd, n_expert }, 0); - create_tensor_gate_up_exps(layer, il, n_embd, n_ff_exp, n_expert, 0); - - layer.ffn_gate_inp_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP_SHEXP, "weight", il), { n_embd }, 0); - layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", il), { n_embd, n_ff_shexp }, 0); - layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", il), { n_embd, n_ff_shexp }, 0); - layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", il), { n_ff_shexp, n_embd }, 0); + layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", il), { n_embd, n_expert }, flags); + layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", il), { n_ff_exp, n_embd, n_expert }, flags); + create_tensor_gate_up_exps(layer, il, n_embd, n_ff_exp, n_expert, flags); + + layer.ffn_gate_inp_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP_SHEXP, "weight", il), { n_embd }, flags); + layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", il), { n_embd, n_ff_shexp }, flags); + layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", il), { n_embd, n_ff_shexp }, flags); + layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", il), { n_ff_shexp, n_embd }, flags); + + if (is_mtp_layer) { + // eh_proj fuses [enorm(embd(next_tok)) ; collapsed hidden] -> n_embd. Note hnorm is + // hc-space (hc_dim), unlike deepseek4's: the draft head consumes the target's + // 4-stream hyper-connection state, collapsed by the (shared) head mixer. + layer.nextn.eh_proj = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ, "weight", il), { 2 * n_embd, n_embd }, flags); + layer.nextn.enorm = create_tensor(tn(LLM_TENSOR_NEXTN_ENORM, "weight", il), { n_embd }, flags); + layer.nextn.hnorm = create_tensor(tn(LLM_TENSOR_NEXTN_HNORM, "weight", il), { hc_dim }, flags); + } } } @@ -287,6 +322,12 @@ ggml_tensor * llama_model_qwen4exp::graph::build_hc_combine( llama_model_qwen4exp::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_build_delta_net_base(params), model(model) { + // An MTP sidecar file carries only the NextN block; the trunk tensors are absent by design + // and this graph cannot be built from it. Without this check the first hc_mix segfaults. + if (model.hparams.n_layer_nextn > 0 && model.layers[0].hc_attn_norm == nullptr) { + GGML_ABORT("this file is an MTP draft sidecar - load it as a draft model (-md) with --spec-type draft-mtp, not as a standalone model"); + } + const int64_t hc = hparams.dsv4_hc_mult; GGML_ASSERT(hparams.n_embd_head_v() == hparams.n_embd_head_k()); From af89cc23c8aecf17c34a1fb28ab87b4aa56230b2 Mon Sep 17 00:00:00 2001 From: Nathan Wilson Date: Thu, 27 Aug 2026 06:04:08 +0000 Subject: [PATCH 2/6] qwen4exp: NextN/MTP draft graph and context - spec decode works end to end Second half of the MTP wiring (loader was the previous commit). Verified end to end on the 64 GB box: Q4_K_M target + agentionai's community MTP sidecar, --spec-type draft-mtp, coherent temp-0 generation through the Vulkan lane. - graph_mtp: [enorm(embd(tok)) repeated across streams ; grouped-RMS hnorm(h)] -> eh_proj per stream -> one HC-wrapped full-attention block -> the shared head mixer (which doubles as the output norm). h arrives hc-space: the mainline graph exports the 4-stream residual under cparams.embeddings_nextn. - the MTP context holds a PLAIN attention cache over the nextn layer(s), dense, no recurrent state - the deepseek32 pattern. Discovered the hard way: a hybrid memory with an empty recurrent layer set fails its buffer allocation. - exports hand the scheduler REAL nodes: a naked reshape view gets no backend assignment and the h_nextn extraction asserts. - the draft runs dense attention (QSA gate is null-safe now); the reference precedent is deepseek32, whose MTP head also runs dense. Correctness only at this point: no acceptance-rate or throughput claims - the smoke ran under heavy memory pressure and the interactive-trap log spin. Measured cleanly it may or may not pay; that is the next session's A/B. Co-Authored-By: Claude Opus 5 (cherry picked from commit b5b7c8c6f4c436a526f5201f5863907fe96fc90c) (cherry picked from commit 39817c476489c37747854bebc29d375770618f1c) --- src/llama-model.cpp | 29 +++++++++ src/models/models.h | 13 +++- src/models/qwen4exp.cpp | 134 +++++++++++++++++++++++++++++++++++++++- 3 files changed, 174 insertions(+), 2 deletions(-) diff --git a/src/llama-model.cpp b/src/llama-model.cpp index e679b24e87f..00fc0196169 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -2482,6 +2482,35 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, return il < hparams.n_layer() && !hparams.is_recr(il); }; } + + if (arch == LLM_ARCH_QWEN4EXP && hparams.n_layer_nextn > 0 && + params.ctx_type == LLAMA_CONTEXT_TYPE_MTP) { + // A hybrid memory with an empty recurrent layer set fails its buffer + // allocation, so the MTP context gets a PLAIN attention cache over + // the nextn layer(s), dense - the deepseek32 MTP pattern. + llama_kv_cache::layer_filter_cb filter_mtp = + [&](uint32_t il) { return il >= hparams.n_layer(); }; + + res = new llama_kv_cache( + *this, + hparams, + params.type_k, + params.type_v, + !cparams.flash_attn, + cparams.offload_kqv, + cparams.kv_unified, + cparams.n_ctx_seq, + cparams.n_seq_max, + 1, + hparams.n_swa, + hparams.swa_type, + nullptr, + filter_mtp, + nullptr, + nullptr); + break; + } + } if (hparams.swa_type != LLAMA_SWA_TYPE_NONE) { diff --git a/src/models/models.h b/src/models/models.h index 9b87a40d5af..9733ecc6f81 100644 --- a/src/models/models.h +++ b/src/models/models.h @@ -2285,7 +2285,13 @@ struct llama_model_qwen4exp : public llama_model_base { struct graph : public llm_build_delta_net_base { graph(const llama_model & model, const llm_graph_params & params); - private: + protected: + // build-nothing constructor for graph_mtp: initialises the context without running + // the mainline body (whose trunk tensors a sidecar file does not carry) + struct mtp_tag {}; + graph(const llama_model & model, const llm_graph_params & params, mtp_tag) : + llm_build_delta_net_base(params), model(model) {} + // HC replaces every layer norm: residual is [n_embd, hc, n_tokens] ggml_tensor * build_hc_mix( ggml_tensor * x, @@ -2377,6 +2383,11 @@ struct llama_model_qwen4exp : public llama_model_base { const llama_model & model; }; + struct graph_mtp : public graph { + graph_mtp(const llama_model & model, const llm_graph_params & params); + }; + + std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; diff --git a/src/models/qwen4exp.cpp b/src/models/qwen4exp.cpp index 921b7b21e0b..2f7facef522 100644 --- a/src/models/qwen4exp.cpp +++ b/src/models/qwen4exp.cpp @@ -245,6 +245,9 @@ void llama_model_qwen4exp::load_arch_tensors(llama_model_loader & ml) { } std::unique_ptr llama_model_qwen4exp::build_arch_graph(const llm_graph_params & params) const { + if (params.gtype == LLM_GRAPH_TYPE_DECODER_MTP) { + return std::make_unique(*this, params); + } return std::make_unique(*this, params); } @@ -418,6 +421,21 @@ llama_model_qwen4exp::graph::graph(const llama_model & model, const llm_graph_pa cb(res_hc, "l_last", il); } + // hand the drafter the 4-stream residual, pre-collapse: nextn.hnorm is hc-space + if (cparams.embeddings_nextn) { + // export the REAL node, not a reshape view: the scheduler assigns no backend to a + // naked view and the extraction then hits GGML_ASSERT(backend_h). Contiguous + // [n_embd, hc, T] has the same memory layout as the flat rows the reader expects. + ggml_tensor * h_nextn = res_hc; + if (cparams.embeddings_nextn_masked && inp_out_ids) { + ggml_tensor * flat = ggml_reshape_2d(ctx0, res_hc, n_embd*hc, n_tokens); + h_nextn = ggml_get_rows(ctx0, flat, inp_out_ids); + } + cb(h_nextn, "h_nextn", -1); + res->t_h_nextn = h_nextn; + ggml_build_forward_expand(gf, h_nextn); + } + // the final mixer is the output norm: there is no separate one ggml_tensor * cur = build_hc_mix(res_hc, model.hc_head_norm, model.hc_head_down, model.hc_head_up, @@ -749,7 +767,7 @@ ggml_tensor * llama_model_qwen4exp::graph::build_layer_attn( GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); // indexer reads the same block input as q/k/v; no cache or no ratio means dense - const bool qsa = mctx_hyb->get_idx() != nullptr && hparams.dsv4_compress_ratios[il] > 0; + const bool qsa = mctx_hyb != nullptr && mctx_hyb->get_idx() != nullptr && hparams.dsv4_compress_ratios[il] > 0; ggml_tensor * top_k = qsa ? build_qsa_top_k(mctx_hyb, cur, inp_pos, inp->get_kq_mask(), sections, il) : nullptr; @@ -1252,3 +1270,117 @@ ggml_tensor * llama_model_qwen4exp::graph::build_ple( return ggml_add(ctx0, hidden, ggml_add(ctx0, gated, conv_out)); } + + +// NextN/MTP draft graph: one full-attention QSA block fed by [enorm(embd(tok)) ; hnorm(h)] +// through eh_proj, closing with the shared head mixer. Mirrors deepseek4::graph_mtp; the +// differences are dictated by the format: hnorm is hc-space (the drafter consumes the target's +// 4-stream residual, exported by the mainline graph under cparams.embeddings_nextn), the block +// is HC-wrapped, and the head mixer doubles as the output norm. +llama_model_qwen4exp::graph_mtp::graph_mtp(const llama_model & model, const llm_graph_params & params) : + graph(model, params, mtp_tag{}) { + GGML_ASSERT(hparams.n_layer_nextn == 1 && "QWEN4EXP MTP currently only supports a single MTP block"); + GGML_ASSERT(cparams.nextn_layer_offset >= 0 && + cparams.nextn_layer_offset < (int) hparams.n_layer_nextn && + "nextn_layer_offset out of range [0, n_layer_nextn)"); + GGML_ASSERT(ubatch.token && "QWEN4EXP MTP requires token input"); + + const int64_t hc = hparams.dsv4_hc_mult; + const int64_t hc_dim = hc * n_embd; + GGML_ASSERT(hparams.n_embd_out() == (uint32_t) hc_dim && "QWEN4EXP MTP hidden width mismatch"); + + const int il = hparams.n_layer() + cparams.nextn_layer_offset; + const auto & layer = model.layers[il]; + + GGML_ASSERT(layer.nextn.eh_proj && layer.nextn.enorm && layer.nextn.hnorm && + "MTP block tensors absent - was the draft context created with load_mtp?"); + + int sections[4]; + std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections); + + auto inp_h = std::make_unique(hparams.n_embd_out()); + + inp_h->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens); + ggml_set_input(inp_h->tokens); + + inp_h->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd_out(), n_tokens); + ggml_set_input(inp_h->embd); + + inp_h->h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd_out(), n_tokens); + ggml_set_input(inp_h->h); + ggml_set_name(inp_h->h, "mtp_h_input"); + + ggml_tensor * tok_embd = ggml_get_rows(ctx0, model.tok_embd, inp_h->tokens); + cb(tok_embd, "mtp_tok_embd", il); + + ggml_tensor * h_state = ggml_reshape_3d(ctx0, inp_h->h, n_embd, hc, n_tokens); + res->add_input(std::move(inp_h)); + + ggml_tensor * inp_pos = build_inp_pos(); + ggml_tensor * inp_out_ids = build_inp_out_ids(); + + // the MTP context holds a plain attention cache over the nextn layer(s) only, the + // deepseek32 pattern: the draft runs dense (no indexer cache, no recurrent state) + auto * inp_attn = build_attn_inp_kv(); + const llama_memory_hybrid_idx_context * mctx_hyb = nullptr; + + // hnorm is the same grouped RMSNorm as every HC norm: rms over one stream, flat gamma + ggml_tensor * h_norm = ggml_rms_norm(ctx0, h_state, hparams.f_norm_rms_eps); + h_norm = ggml_reshape_2d(ctx0, h_norm, hc_dim, n_tokens); + h_norm = ggml_mul(ctx0, h_norm, layer.nextn.hnorm); + h_norm = ggml_reshape_3d(ctx0, h_norm, n_embd, hc, n_tokens); + cb(h_norm, "mtp_hnorm", il); + + ggml_tensor * e_norm = ggml_rms_norm(ctx0, tok_embd, hparams.f_norm_rms_eps); + e_norm = ggml_mul(ctx0, e_norm, layer.nextn.enorm); + e_norm = ggml_reshape_3d(ctx0, e_norm, n_embd, 1, n_tokens); + e_norm = ggml_repeat_4d(ctx0, e_norm, n_embd, hc, n_tokens, 1); + cb(e_norm, "mtp_enorm", il); + + ggml_tensor * concat = ggml_concat(ctx0, e_norm, h_norm, 0); + ggml_tensor * res_hc = build_lora_mm(layer.nextn.eh_proj, concat); + cb(res_hc, "mtp_eh_proj", il); + + // one HC-wrapped full-attention QSA block, the mainline loop body minus PLE/GDN + ggml_tensor * inject = nullptr; + ggml_tensor * cur = build_hc_mix(res_hc, + layer.hc_attn_norm, layer.hc_attn_down, layer.hc_attn_up, layer.hc_attn_inject, + &inject, il); + ggml_build_forward_expand(gf, cur); + + cur = build_layer_attn(inp_attn, mctx_hyb, cur, inp_pos, sections, il); + res_hc = build_hc_combine(res_hc, cur, inject, il); + + cur = build_hc_mix(res_hc, + layer.hc_ffn_norm, layer.hc_ffn_down, layer.hc_ffn_up, layer.hc_ffn_inject, + &inject, il); + cur = build_layer_ffn(cur, il); + cb(cur, "mtp_ffn_out", il); + res_hc = build_hc_combine(res_hc, cur, inject, il); + + // chained-draft export: the drafter's own hc state, gathered to the output rows + ggml_tensor * h_nextn = res_hc; // real node; see the export note in the mainline graph + if (inp_out_ids) { + ggml_tensor * flat = ggml_reshape_2d(ctx0, res_hc, hc_dim, n_tokens); + h_nextn = ggml_get_rows(ctx0, flat, inp_out_ids); + } + cb(h_nextn, "h_nextn", -1); + res->t_h_nextn = h_nextn; + ggml_build_forward_expand(gf, h_nextn); + + // the head mixer is the output norm; the sidecar carries its own copy of it + cur = build_hc_mix(res_hc, + model.hc_head_norm, model.hc_head_down, model.hc_head_up, + nullptr, nullptr, -1); + if (inp_out_ids) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + } + cb(cur, "result_norm", -1); + res->t_embd = cur; + + cur = build_lora_mm(model.output, cur, model.output_s); + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); +} From a4770544b5669246843e121adaa548009189afee Mon Sep 17 00:00:00 2001 From: Nathan Wilson Date: Sat, 22 Aug 2026 08:07:08 +0000 Subject: [PATCH 3/6] Reapply "common: use full checkpoints for MTP rollback" This reverts commit a17e8432b (the v0.6.9 revert). The stall that motivated the revert was not in the state-save path: it was a replay re-verification livelock in the server's full-checkpoint rollback, fixed in the previous commit. With that in place, full checkpoints restore token-exact MTP rollback on recurrent targets again. Verified on gfx1151 Vulkan (Qwen3.6-35B-A3B Native-MTP Q6_K, greedy): the v0.6.8 repro completes 800 tokens in 12 s, a 3000-token request runs through the former stall horizon to EOS at 62 tok/s, and repeat runs are bit-identical. CPU spec-vs-nospec is token-exact through the canonical 192-token matrix. Assisted-by: Claude Fable 5 (cherry picked from commit f25eefeaf0386c18499f23f4fc4f400397638d51) --- common/common.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/common.h b/common/common.h index 4e9448bb106..831399047b7 100644 --- a/common/common.h +++ b/common/common.h @@ -393,7 +393,7 @@ struct common_params_speculative { uint32_t need_n_rs_seq() const { bool needs_rs_seq = std::any_of(types.begin(), types.end(), [&](auto t) { - return t == COMMON_SPECULATIVE_TYPE_DRAFT_MTP || t == COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3 || t == COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH || t == COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK; + return t == COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3 || t == COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH || t == COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK; }); return needs_rs_seq ? draft.n_max : 0u; From 4b1f6f811c380c088603720657ecf9e2b9cf46db Mon Sep 17 00:00:00 2001 From: Nathan Wilson Date: Sat, 22 Aug 2026 08:06:54 +0000 Subject: [PATCH 4/6] server : do not re-verify replayed draft tokens after a checkpoint restore With full-checkpoint rollback, a partial draft acceptance restores the pre-round state and re-decodes the accepted tokens to rebuild it. The replay went through the same verification as a fresh draft. On backends where logits change with batch shape or memory layout (Vulkan), that re-verification can reject a token the original verification accepted; the rejection restores the same checkpoint and replays again, and the slot loops on one position without emitting anything. qwen35moe with --spec-type draft-mtp stalled this way a few hundred tokens into long generations (the v0.6.8 MTP hang): the loop repeated "accepted 2/3, restore at pos 995" every 27 ms with the GPU at 90 percent. Accept the replayed tokens without re-verifying and sample only the continuation from the final position. The replayed prefix was accepted by the verification that triggered the restore; the replay exists to rebuild state. On backends with batch-shape invariant logits the re-verification always agreed, so behavior there is unchanged (verified bit-identical on CPU with and without this change, 800-token greedy pair, 118 restore rounds). Assisted-by: Claude Fable 5 (cherry picked from commit 9c5d899ff7966179f56e49edd5c7a57f7b6172e6) --- tools/server/server-context.cpp | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index f5477356d61..5ed439417f1 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -3884,12 +3884,25 @@ struct server_context_impl { common_sampler_ptr smpl_save(common_sampler_clone(slot.smpl.get())); GGML_ASSERT(slot.spec_i_batch.size() == n_draft + 1); - const auto & synth_probs = common_speculative_get_synth_probs(spec.get()); - auto accepted = synth_probs.empty() - ? common_sampler_sample_and_accept_n(slot.smpl.get(), slot.ctx_tgt, slot.spec_i_batch, slot.spec_draft) - : server_sample_and_accept_synth( - slot.smpl.get(), slot.ctx_tgt, slot.spec_i_batch, slot.spec_draft, - synth_probs, slot.spec_synth_rng, slot.spec_is_replay); + std::vector accepted; + if (slot.spec_is_replay) { + // replayed tokens were accepted before the restore; re-verifying them can + // disagree when logits depend on batch shape, and each disagreement restores + // the same checkpoint again - the slot stops making progress + accepted = slot.spec_draft; + for (const llama_token id : accepted) { + common_sampler_accept(slot.smpl.get(), id, true); + } + accepted.push_back(common_sampler_sample(slot.smpl.get(), slot.ctx_tgt, slot.spec_i_batch.back())); + common_sampler_accept(slot.smpl.get(), accepted.back(), true); + } else { + const auto & synth_probs = common_speculative_get_synth_probs(spec.get()); + accepted = synth_probs.empty() + ? common_sampler_sample_and_accept_n(slot.smpl.get(), slot.ctx_tgt, slot.spec_i_batch, slot.spec_draft) + : server_sample_and_accept_synth( + slot.smpl.get(), slot.ctx_tgt, slot.spec_i_batch, slot.spec_draft, + synth_probs, slot.spec_synth_rng, slot.spec_is_replay); + } slot.spec_i_batch.clear(); GGML_ASSERT(accepted.size() >= 1); From 77476ce87ece69f966b3f967d15f0fa512fd3402 Mon Sep 17 00:00:00 2001 From: Jay Beavers Date: Sun, 30 Aug 2026 18:58:39 -0700 Subject: [PATCH 5/6] qwen4exp : defer the output-row gather past the h_nextn export The MTP drafter consumes one hyper-connection row per prompt token: its prefill path memcpys n_tokens-1 rows straight out of the nextn buffer. The mainline graph dropped the non-output rows inside the last layer, before the export, so the exported residual carried only n_outputs rows and the reader walked off the end -- GGML_ASSERT(offset + size <= ggml_nbytes(tensor)) on the first decode. Skip the early gather while a drafter is attached and re-apply it immediately after the export, so the drafter sees every row and the output head still sees only the output rows. Costs one extra layer of compute on dropped rows in the last layer, which is why the gather sits where it does when no drafter is attached. Setting the target context masked instead does not work: the flag also gates a second gather in the export, and the drafter has no output-indexed path. --- src/models/qwen4exp.cpp | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/models/qwen4exp.cpp b/src/models/qwen4exp.cpp index 2f7facef522..25596ee898f 100644 --- a/src/models/qwen4exp.cpp +++ b/src/models/qwen4exp.cpp @@ -393,7 +393,9 @@ llama_model_qwen4exp::graph::graph(const llama_model & model, const llm_graph_pa cur = build_layer_attn(inp->get_attn(), mctx_hyb, cur, inp_pos, sections, il); } - if (il == n_layer - 1 && inp_out_ids) { + // the MTP drafter consumes one h-row per prompt token, so when it is attached + // the rows cannot be dropped here; the gather moves below the h_nextn export. + if (il == n_layer - 1 && inp_out_ids && !cparams.embeddings_nextn) { // everything below is per token, so drop the rows that produce no output cur = ggml_get_rows(ctx0, cur, inp_out_ids); inject = ggml_get_rows(ctx0, inject, inp_out_ids); @@ -434,6 +436,13 @@ llama_model_qwen4exp::graph::graph(const llama_model & model, const llm_graph_pa cb(h_nextn, "h_nextn", -1); res->t_h_nextn = h_nextn; ggml_build_forward_expand(gf, h_nextn); + + // deferred from the last layer: everything below is per output token + if (inp_out_ids) { + res_hc = ggml_reshape_2d(ctx0, res_hc, n_embd*hc, res_hc->ne[2]); + res_hc = ggml_get_rows(ctx0, res_hc, inp_out_ids); + res_hc = ggml_reshape_3d(ctx0, res_hc, n_embd, hc, res_hc->ne[1]); + } } // the final mixer is the output norm: there is no separate one From 175b66c51f08583caba7eb8ef68c6b6ed83f73ce Mon Sep 17 00:00:00 2001 From: Jay Beavers Date: Sun, 30 Aug 2026 19:41:17 -0700 Subject: [PATCH 6/6] server : keep speculative checkpoints on device qwen4exp is a recurrent hybrid, so the target context cannot partially seq_rm and the server classifies it SEQ_RM_TYPE_FULL. Every speculative round then takes a full recurrent-state checkpoint, and every rejected draft restores one. Through the host path that means serializing each GDN layer conv+state row, the 4-stream hyper-connection residual and the PLE history into a host vector with one synchronous backend read per tensor, then pushing it all back. The cost is flat in context and swamps everything else: with the MTP drafter attached, decode ran 201 ms/token against 29 ms/token on a tree that keeps the state on device, and a 120-token generation spent roughly 600 ms of each 825 ms round in checkpoint traffic. It also inflates the reported prompt eval time, which runs to first token and so absorbs the first save: 827 ms for an 11-token prompt. Request ON_DEVICE at the six speculative checkpoint sites so the state stays in device buffers. The library already implements this path; only the server never asked for it. Measured on gfx1151 Vulkan, Qwen3.8-Flash-Next UD-Q4_K_XL with a Q4_K_M MTP head: 4.77 -> 25.83 tok/s at short context, 4.33 -> 16.08 at 70k, draft acceptance unchanged at 70-80 percent. The prompt-cache checkpoints are deliberately left host-resident: they retain several historical states rather than one live round. Diagnosed by Claude Fable 5. The ON_DEVICE flag and the mechanism come from Gaetan Puleo (c8b681b6f), carried in Nathanw1014/llama.cpp as 08a3255. --- tools/server/server-context.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 5ed439417f1..40b45f3a859 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -2993,7 +2993,7 @@ struct server_context_impl { llama_memory_seq_pos_max(llama_get_memory(ctx_tgt), slot.id)); if (use_ckpt_dft) { - slot.spec_ckpt.update_dft(ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); + slot.spec_ckpt.update_dft(ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE); } slot.spec_prompt = slot.prompt.tokens.get_text_tokens(); @@ -3032,7 +3032,7 @@ struct server_context_impl { if (ctx_dft) { if (use_ckpt_dft) { - ckpt.load_dft(ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); + ckpt.load_dft(ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE); } if (!llama_memory_seq_rm(llama_get_memory(ctx_dft), slot.id, ckpt.pos_max + 1, -1)) { @@ -3051,7 +3051,7 @@ struct server_context_impl { if (use_ckpt_tgt) { //const int64_t t_start = ggml_time_us(); - ckpt.update_tgt(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); + ckpt.update_tgt(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE); //const int64_t t_total = ggml_time_us() - t_start; //printf("checkpoint total: %f ms\n", t_total / 1000.0); @@ -3063,7 +3063,7 @@ struct server_context_impl { } if (use_ckpt_dft) { - ckpt.update_dft(ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); + ckpt.update_dft(ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE); } } }); @@ -3928,10 +3928,10 @@ struct server_context_impl { SLT_DBG(slot, "restoring speculative checkpoint (pos_min = %d, pos_max = %d, size = %zu)\n", ckpt.pos_min, ckpt.pos_max, ckpt.size()); - ckpt.load_tgt(slot.ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); + ckpt.load_tgt(slot.ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE); if (slot.ctx_dft) { - ckpt.load_dft(slot.ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); + ckpt.load_dft(slot.ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE); } slot.mem.seq_rm(slot.id, ckpt.pos_max + 1, -1);