qwen4exp : port NextN/MTP speculative decoding to master (+50% decode at 70k)

common/common.h

@@ -393,7 +393,7 @@ struct common_params_speculative {
 
     uint32_t need_n_rs_seq() const {
         bool needs_rs_seq = std::any_of(types.begin(), types.end(), [&](auto t) {
-            return t == COMMON_SPECULATIVE_TYPE_DRAFT_MTP || t == COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3 || t == COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH || t == COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK;
+            return t == COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3 || t == COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH || t == COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK;
         });
 
         return needs_rs_seq ? draft.n_max : 0u;

src/llama-model.cpp

@@ -2482,6 +2482,35 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
                                 return il < hparams.n_layer() && !hparams.is_recr(il);
                             };
                         }
+
+                        if (arch == LLM_ARCH_QWEN4EXP && hparams.n_layer_nextn > 0 &&
+                                params.ctx_type == LLAMA_CONTEXT_TYPE_MTP) {
+                            // A hybrid memory with an empty recurrent layer set fails its buffer
+                            // allocation, so the MTP context gets a PLAIN attention cache over
+                            // the nextn layer(s), dense - the deepseek32 MTP pattern.
+                            llama_kv_cache::layer_filter_cb filter_mtp =
+                                [&](uint32_t il) { return il >= hparams.n_layer(); };
+
+                            res = new llama_kv_cache(
+                                    *this,
+                                    hparams,
+                                    params.type_k,
+                                    params.type_v,
+                                    !cparams.flash_attn,
+                                    cparams.offload_kqv,
+                                    cparams.kv_unified,
+                                    cparams.n_ctx_seq,
+                                    cparams.n_seq_max,
+                                    1,
+                                    hparams.n_swa,
+                                    hparams.swa_type,
+                                    nullptr,
+                                    filter_mtp,
+                                    nullptr,
+                                    nullptr);
+                            break;
+                        }
+
                     }
 
                     if (hparams.swa_type != LLAMA_SWA_TYPE_NONE) {

src/models/models.h

@@ -2285,7 +2285,13 @@ struct llama_model_qwen4exp : public llama_model_base {
 
     struct graph : public llm_build_delta_net_base {
         graph(const llama_model & model, const llm_graph_params & params);
-    private:
+    protected:
+        // build-nothing constructor for graph_mtp: initialises the context without running
+        // the mainline body (whose trunk tensors a sidecar file does not carry)
+        struct mtp_tag {};
+        graph(const llama_model & model, const llm_graph_params & params, mtp_tag) :
+            llm_build_delta_net_base(params), model(model) {}
+
         // HC replaces every layer norm: residual is [n_embd, hc, n_tokens]
         ggml_tensor * build_hc_mix(
                     ggml_tensor * x,
@@ -2377,6 +2383,11 @@ struct llama_model_qwen4exp : public llama_model_base {
         const llama_model & model;
     };
 
+    struct graph_mtp : public graph {
+        graph_mtp(const llama_model & model, const llm_graph_params & params);
+    };
+
+
     std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
 };