From 8e84cc5df3787703d31ea28f77f8aa981797256a Mon Sep 17 00:00:00 2001 From: Stew Forster Date: Sun, 16 Aug 2026 20:25:28 -0500 Subject: [PATCH 1/6] spec : add adaptive MTP draft depth (draft-mtp-adaptive) Hysteresis state machine with a climb counter and a weighted drop-pressure accumulator. The depth climbs one step after 5 consecutive verifies that accepted every drafted token; any miss adds (N - acceptance) to the drop pressure and the depth drops one step once it exceeds 30. High depths fall quickly (a total miss adds N), low depths hold, and at the floor no pressure accumulates at all. The floor is max(1, n_min) and the ceiling is n_max, so --spec-draft-n-min/--spec-draft-n-max bound the adaptive range and the cold-start depth is 3. Assisted-by: pi --- common/arg.cpp | 5 +- common/common.cpp | 3 +- common/common.h | 3 +- common/speculative.cpp | 118 ++++++++++++++++++++++++++++++-- tools/server/server-context.cpp | 5 +- 5 files changed, 124 insertions(+), 10 deletions(-) diff --git a/common/arg.cpp b/common/arg.cpp index 79405b59e07..b3fd6abce91 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -365,7 +365,10 @@ common_models_handler common_models_handler_init(const common_params & params, l const bool spec_type_draft_mtp = std::find(params.speculative.types.begin(), params.speculative.types.end(), - COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params.speculative.types.end(); + COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params.speculative.types.end() || + std::find(params.speculative.types.begin(), + params.speculative.types.end(), + COMMON_SPECULATIVE_TYPE_DRAFT_MTP_ADAPTIVE) != params.speculative.types.end(); const bool spec_type_draft_dflash = std::find(params.speculative.types.begin(), params.speculative.types.end(), diff --git a/common/common.cpp b/common/common.cpp index d162a38800e..92a3d1493dc 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -1711,7 +1711,8 @@ struct llama_model_params common_model_params_to_llama(common_params & params) { mparams.progress_callback = params.load_progress_callback; mparams.progress_callback_user_data = params.load_progress_callback_user_data; mparams.no_alloc = params.no_alloc; - mparams.load_mtp = std::find(params.speculative.types.begin(), params.speculative.types.end(), COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params.speculative.types.end(); + mparams.load_mtp = std::find(params.speculative.types.begin(), params.speculative.types.end(), COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params.speculative.types.end() || + std::find(params.speculative.types.begin(), params.speculative.types.end(), COMMON_SPECULATIVE_TYPE_DRAFT_MTP_ADAPTIVE) != params.speculative.types.end(); return mparams; } diff --git a/common/common.h b/common/common.h index 4e9448bb106..2d7fe9080bf 100644 --- a/common/common.h +++ b/common/common.h @@ -173,6 +173,7 @@ enum common_speculative_type { COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE, // standalone draft model speculative decoding COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3, // Eagle3 speculative decoding COMMON_SPECULATIVE_TYPE_DRAFT_MTP, // Multi-token prediction + COMMON_SPECULATIVE_TYPE_DRAFT_MTP_ADAPTIVE, // MTP with adaptive draft depth COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH, // DFlash speculative decoding COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK, // DSpark speculative decoding (DFlash + Markov head) COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE, // simple self-speculative decoding based on n-grams @@ -393,7 +394,7 @@ struct common_params_speculative { uint32_t need_n_rs_seq() const { bool needs_rs_seq = std::any_of(types.begin(), types.end(), [&](auto t) { - return t == COMMON_SPECULATIVE_TYPE_DRAFT_MTP || t == COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3 || t == COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH || t == COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK; + return t == COMMON_SPECULATIVE_TYPE_DRAFT_MTP || t == COMMON_SPECULATIVE_TYPE_DRAFT_MTP_ADAPTIVE || t == COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3 || t == COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH || t == COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK; }); return needs_rs_seq ? draft.n_max : 0u; diff --git a/common/speculative.cpp b/common/speculative.cpp index 851a47b9a58..c4d7874524f 100644 --- a/common/speculative.cpp +++ b/common/speculative.cpp @@ -35,6 +35,7 @@ const std::map common_speculative_type_fro {"draft-simple", COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE}, {"draft-eagle3", COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3}, {"draft-mtp", COMMON_SPECULATIVE_TYPE_DRAFT_MTP}, + {"draft-mtp-adaptive", COMMON_SPECULATIVE_TYPE_DRAFT_MTP_ADAPTIVE}, {"draft-dflash", COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH}, {"draft-dspark", COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK}, {"ngram-simple", COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE}, @@ -1357,8 +1358,73 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { std::vector i_last; std::vector> chain_h; - common_speculative_impl_draft_mtp(const common_params_speculative & params, uint32_t n_seq) - : common_speculative_impl(COMMON_SPECULATIVE_TYPE_DRAFT_MTP, n_seq, params.draft.n_max) + // Adaptive draft depth (draft-mtp-adaptive): hysteresis state machine with two + // counters per sequence. The depth N climbs after N_CLIMB(N) consecutive verifies + // that accepted every drafted token, where the requirement shrinks with depth: + // 5 full accepts below depth 5, 4 at depth 5-6, 3 at depth 7+. Leaving the floor + // must really be earned, but sustained acceptance that carried the depth this far + // earns the next step cheaply. Any miss adds (N - acceptance) to a drop-pressure + // accumulator; when it exceeds ADAPTIVE_DROP_PRESSURE the depth drops one step + // and the pressure resets. A near miss (N-1) adds 1, a total miss at depth N adds + // N, so high depths fall quickly while low depths hold. The floor is max(1, n_min) + // and the ceiling is n_max (--spec-draft-n-max). + bool adaptive = false; + std::vector n_cap; // [n_seq] effective draft cap for the current draft() call + std::vector n_cur; // [n_seq] current adaptive depth N + std::vector n_last; // [n_seq] drafts attempted in the most recent draft() call + std::vector n_climb; // [n_seq] consecutive fully-accepted verifies + std::vector n_drop; // [n_seq] accumulated drop pressure: sum of (N - acceptance) + + static constexpr int ADAPTIVE_DROP_PRESSURE = 30; // accumulated (N - acceptance) to drop one step + + // consecutive full accepts needed to climb one step from depth N; cheaper the + // deeper we are, so predictable content accelerates while prose must earn its + // way off the floor + static int adaptive_climb_threshold(int depth) { + return depth >= 7 ? 3 : (depth >= 5 ? 4 : 5); + } + + void adaptive_update(llama_seq_id seq_id, int n_draft, int n_accepted) { + if (n_draft <= 0) { + return; + } + + const int floor = std::max(1, params.n_min); + const int n_max = std::max(1, params.n_max); + + int depth = n_cur[seq_id]; + + if (n_accepted == depth) { + n_drop[seq_id] = 0; + + // full acceptance: reset the drop pressure, accumulate the climb streak + if ((depth < n_max) && (++n_climb[seq_id] >= adaptive_climb_threshold(depth))) { + depth++; + n_climb[seq_id] = 0; + } + } else { + n_climb[seq_id] = 0; + + if (depth > floor) { + // any miss adds (N - acceptance) to the drop pressure; drop when the + // accumulated pressure exceeds the threshold + n_drop[seq_id] += depth - n_accepted; + if (n_drop[seq_id] > ADAPTIVE_DROP_PRESSURE) { + depth--; + n_drop[seq_id] = 0; + } + } + } + + if (depth != n_cur[seq_id]) { + SPC_DBG("adaptive draft depth seq %d: %d -> %d (n_draft=%d, n_accepted=%d)\n", + (int) seq_id, n_cur[seq_id], depth, n_draft, n_accepted); + } + n_cur[seq_id] = depth; + } + + common_speculative_impl_draft_mtp(const common_params_speculative & params, uint32_t n_seq, bool adaptive = false) + : common_speculative_impl(adaptive ? COMMON_SPECULATIVE_TYPE_DRAFT_MTP_ADAPTIVE : COMMON_SPECULATIVE_TYPE_DRAFT_MTP, n_seq, params.draft.n_max) , params(params.draft) { auto * ctx_tgt = this->params.ctx_tgt; @@ -1370,6 +1436,23 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { "MTP input row width must match the target h_nextn width"); n_mtp_layers = std::max(1, (int) llama_model_n_layer_nextn(llama_get_model(ctx_dft))); + this->adaptive = adaptive; + // n_cap/n_last are written by the shared draft loop in both modes + n_cap.assign(n_seq, 0); + n_last.assign(n_seq, 0); + if (adaptive) { + const int n_max = std::max(1, this->params.n_max); + n_cur.assign(n_seq, 0); + n_climb.assign(n_seq, 0); + n_drop.assign(n_seq, 0); + for (uint32_t s = 0; s < n_seq; ++s) { + // start at a safe depth of 3 (bounded by n_min/n_max); the controller + // moves it in [max(1,n_min), n_max] once acceptance feedback arrives + n_cur[s] = std::min(std::max(3, std::max(1, this->params.n_min)), n_max); + } + SPC_TRC("%s", "adaptive draft depth enabled (draft-mtp-adaptive)\n"); + } + SPC_TRC("%s", "adding speculative implementation 'draft-mtp'\n"); SPC_TRC("- n_max=%d, n_min=%d, p_min=%.2f, n_embd=%d, backend_sampling=%d\n", this->params.n_max, this->params.n_min, this->params.p_min, n_embd, (int) this->params.backend_sampling); SPC_TRC("- gpu_layers=%d, cache_k=%s, cache_v=%s, ctx_tgt=%s, ctx_dft=%s, devices=[%s]\n", @@ -1615,6 +1698,13 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { drafting[seq_id] = true; common_sampler_reset(smpls[seq_id].get()); + // effective draft cap for this step: adaptive depth (or the user n_max), + // then clamped by the per-call context bound from the server + n_cap[seq_id] = adaptive ? n_cur[seq_id] : params.n_max; + if (dp.n_max > 0 && dp.n_max < n_cap[seq_id]) { + n_cap[seq_id] = dp.n_max; + } + common_batch_add(batch, dp.id_last, dp.n_past, { seq_id }, true); std::memcpy(batch.embd + (size_t) (batch.n_tokens - 1) * n_embd, pending_h[seq_id].data(), row_bytes); @@ -1691,7 +1781,7 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { result.push_back(id); - if (params.n_max <= (int) result.size()) { + if (n_cap[seq_id] <= (int) result.size()) { drafting[seq_id] = false; n_drafting--; continue; @@ -1738,6 +1828,8 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { continue; } + n_last[seq_id] = (int) dp.result->size(); + if (dp.result->size() < (size_t) params.n_min) { dp.result->clear(); } @@ -1749,6 +1841,10 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { return; } + if (adaptive) { + adaptive_update(seq_id, n_last[seq_id], n_accepted); + } + const int32_t n_rows = verify_h_rows[seq_id]; if (n_rows <= 0) { return; @@ -2243,6 +2339,7 @@ std::string common_speculative_type_to_str(common_speculative_type type) { case COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE: return "draft-simple"; case COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3: return "draft-eagle3"; case COMMON_SPECULATIVE_TYPE_DRAFT_MTP: return "draft-mtp"; + case COMMON_SPECULATIVE_TYPE_DRAFT_MTP_ADAPTIVE: return "draft-mtp-adaptive"; case COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH: return "draft-dflash"; case COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK: return "draft-dspark"; case COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE: return "ngram-simple"; @@ -2334,6 +2431,7 @@ int32_t common_speculative_n_max(const common_params_speculative * spec) { case COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE: case COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3: case COMMON_SPECULATIVE_TYPE_DRAFT_MTP: + case COMMON_SPECULATIVE_TYPE_DRAFT_MTP_ADAPTIVE: case COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH: case COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK: n_max = std::max(n_max, std::max(0, spec->draft.n_max)); @@ -2520,7 +2618,10 @@ common_speculative_init_result::common_speculative_init_result( const bool has_draft = params.speculative.has_dft(); const bool spec_mtp = std::find(params.speculative.types.begin(), params.speculative.types.end(), - COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params.speculative.types.end(); + COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params.speculative.types.end() || + std::find(params.speculative.types.begin(), + params.speculative.types.end(), + COMMON_SPECULATIVE_TYPE_DRAFT_MTP_ADAPTIVE) != params.speculative.types.end(); auto mparams = common_model_params_to_llama(params); auto cparams = common_context_params_to_llama(params); @@ -2612,7 +2713,7 @@ common_speculative * common_speculative_init(common_params_speculative & params, }; // when adding a new type - update here the logic above - static_assert(COMMON_SPECULATIVE_TYPE_COUNT == 11); + static_assert(COMMON_SPECULATIVE_TYPE_COUNT == 12); // this list here defines the priority of the speculators // the one with highest priority are listed first @@ -2625,6 +2726,7 @@ common_speculative * common_speculative_init(common_params_speculative & params, add_config_if_enabled(COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE); add_config_if_enabled(COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3, params.draft.ctx_dft != nullptr); add_config_if_enabled(COMMON_SPECULATIVE_TYPE_DRAFT_MTP, params.draft.ctx_dft != nullptr); + add_config_if_enabled(COMMON_SPECULATIVE_TYPE_DRAFT_MTP_ADAPTIVE, params.draft.ctx_dft != nullptr); add_config_if_enabled(COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH, params.draft.ctx_dft != nullptr); add_config_if_enabled(COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK, params.draft.ctx_dft != nullptr); } @@ -2644,7 +2746,11 @@ common_speculative * common_speculative_init(common_params_speculative & params, break; } case COMMON_SPECULATIVE_TYPE_DRAFT_MTP: { - impls.push_back(std::make_unique(config.params, n_seq)); + impls.push_back(std::make_unique(config.params, n_seq, /*adaptive=*/ false)); + break; + } + case COMMON_SPECULATIVE_TYPE_DRAFT_MTP_ADAPTIVE: { + impls.push_back(std::make_unique(config.params, n_seq, /*adaptive=*/ true)); break; } case COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH: { diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index f5477356d61..64477c231aa 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -1019,7 +1019,10 @@ struct server_context_impl { const bool has_draft = params.speculative.has_dft(); const bool spec_mtp = std::find(params_base.speculative.types.begin(), params_base.speculative.types.end(), - COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params_base.speculative.types.end(); + COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params_base.speculative.types.end() || + std::find(params_base.speculative.types.begin(), + params_base.speculative.types.end(), + COMMON_SPECULATIVE_TYPE_DRAFT_MTP_ADAPTIVE) != params_base.speculative.types.end(); const bool has_spec = has_draft || spec_mtp; if (callback_state) { From 5f388fee926efeaf145c168b3cb5b8e941978228 Mon Sep 17 00:00:00 2001 From: Stew Forster Date: Mon, 17 Aug 2026 06:00:14 -0500 Subject: [PATCH 2/6] spec : rework adaptive MTP draft depth Adaptive MTP starts at the floor of --spec-draft-n-min-adaptive (default 3) and adjusts its own depth: consecutive full accepts climb one step, with a cost table that rises fast to depth 3, blocks 3->4 (where marginal content collapses), and climbs fast at depth; a drop-pressure accumulator of n_draft - n_accepted, with a budget of max(depth * 5, 20), lowers it. Fully accepted but truncated drafts count as full accepts, and only drafts this implementation actually produced update the controller. The depth is independent of --spec-draft-n-min, which keeps its usual meaning of a minimum draft length to verify for the non-adaptive spec types. Extracted the controller into a standalone struct and added unit test cases for adaptive MTP. Assisted-by: pi --- common/arg.cpp | 17 ++- common/common.cpp | 2 +- common/common.h | 31 ++-- common/speculative-adaptive.h | 92 ++++++++++++ common/speculative.cpp | 116 +++++---------- tests/CMakeLists.txt | 1 + tests/test-arg-parser.cpp | 15 ++ tests/test-speculative-adaptive.cpp | 219 ++++++++++++++++++++++++++++ 8 files changed, 393 insertions(+), 100 deletions(-) create mode 100644 common/speculative-adaptive.h create mode 100644 tests/test-speculative-adaptive.cpp diff --git a/common/arg.cpp b/common/arg.cpp index b3fd6abce91..f141c59811c 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -363,12 +363,8 @@ common_models_handler common_models_handler_init(const common_params & params, l common_download_hf_plan plan_spec; common_download_opts opts; - const bool spec_type_draft_mtp = std::find(params.speculative.types.begin(), - params.speculative.types.end(), - COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params.speculative.types.end() || - std::find(params.speculative.types.begin(), - params.speculative.types.end(), - COMMON_SPECULATIVE_TYPE_DRAFT_MTP_ADAPTIVE) != params.speculative.types.end(); + const bool spec_type_draft_mtp = std::find(params.speculative.types.begin(), params.speculative.types.end(), COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params.speculative.types.end() || + std::find(params.speculative.types.begin(), params.speculative.types.end(), COMMON_SPECULATIVE_TYPE_DRAFT_MTP_ADAPTIVE) != params.speculative.types.end(); const bool spec_type_draft_dflash = std::find(params.speculative.types.begin(), params.speculative.types.end(), @@ -1306,6 +1302,7 @@ bool common_params_parse(int argc, char ** argv, common_params & params, llama_e common_params_print_completion(ctx_arg); exit(0); } + params.lr.init(); } catch (const std::invalid_argument & ex) { fprintf(stderr, "%s\n", ex.what()); @@ -4189,6 +4186,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex } ).set_spec().set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_SYNTH_RATES")); + add_opt(common_arg( + {"--spec-draft-n-min-adaptive"}, "N", + string_format("minimum adaptive MTP draft depth; the depth starts here and never drops below it (default: %d)", params.speculative.draft.n_min_adaptive), + [](common_params & params, int value) { + params.speculative.draft.n_min_adaptive = value; + } + ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_N_MIN_ADAPTIVE")); + add_opt(common_arg( {"--spec-draft-p-split", "--draft-p-split"}, "P", string_format("speculative decoding split probability (default: %.2f)", (double)params.speculative.draft.p_split), diff --git a/common/common.cpp b/common/common.cpp index 92a3d1493dc..577a43c7f98 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -1712,7 +1712,7 @@ struct llama_model_params common_model_params_to_llama(common_params & params) { mparams.progress_callback_user_data = params.load_progress_callback_user_data; mparams.no_alloc = params.no_alloc; mparams.load_mtp = std::find(params.speculative.types.begin(), params.speculative.types.end(), COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params.speculative.types.end() || - std::find(params.speculative.types.begin(), params.speculative.types.end(), COMMON_SPECULATIVE_TYPE_DRAFT_MTP_ADAPTIVE) != params.speculative.types.end(); + std::find(params.speculative.types.begin(), params.speculative.types.end(), COMMON_SPECULATIVE_TYPE_DRAFT_MTP_ADAPTIVE) != params.speculative.types.end(); return mparams; } diff --git a/common/common.h b/common/common.h index 2d7fe9080bf..f161b08c12d 100644 --- a/common/common.h +++ b/common/common.h @@ -169,19 +169,19 @@ enum common_params_sampling_config : uint64_t { }; enum common_speculative_type { - COMMON_SPECULATIVE_TYPE_NONE, // no speculative decoding - COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE, // standalone draft model speculative decoding - COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3, // Eagle3 speculative decoding - COMMON_SPECULATIVE_TYPE_DRAFT_MTP, // Multi-token prediction - COMMON_SPECULATIVE_TYPE_DRAFT_MTP_ADAPTIVE, // MTP with adaptive draft depth - COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH, // DFlash speculative decoding - COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK, // DSpark speculative decoding (DFlash + Markov head) - COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE, // simple self-speculative decoding based on n-grams - COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K, // self-speculative decoding with n-gram keys only - COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V, // self-speculative decoding with n-gram keys and 4 m-gram values + COMMON_SPECULATIVE_TYPE_NONE, // no speculative decoding + COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE, // standalone draft model speculative decoding + COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3, // Eagle3 speculative decoding + COMMON_SPECULATIVE_TYPE_DRAFT_MTP, // Multi-token prediction + COMMON_SPECULATIVE_TYPE_DRAFT_MTP_ADAPTIVE, // MTP with adaptive draft depth + COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH, // DFlash speculative decoding + COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK, // DSpark speculative decoding (DFlash + Markov head) + COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE, // simple self-speculative decoding based on n-grams + COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K, // self-speculative decoding with n-gram keys only + COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V, // self-speculative decoding with n-gram keys and 4 m-gram values COMMON_SPECULATIVE_TYPE_NGRAM_MOD, - COMMON_SPECULATIVE_TYPE_NGRAM_CACHE, // self-speculative decoding with 3-level n-gram cache - COMMON_SPECULATIVE_TYPE_COUNT // number of types, unknown type + COMMON_SPECULATIVE_TYPE_NGRAM_CACHE, // self-speculative decoding with 3-level n-gram cache + COMMON_SPECULATIVE_TYPE_COUNT // number of types, unknown type }; // Grammar type enumeration @@ -326,6 +326,7 @@ struct common_params_model { struct common_params_speculative_draft { int32_t n_max = 3; // maximum number of tokens to draft during speculative decoding int32_t n_min = 0; // minimum number of draft tokens to use for speculative decoding + int32_t n_min_adaptive = 3; // minimum adaptive MTP draft depth (also the starting depth) float p_split = 0.1f; // speculative decoding split probability float p_min = 0.0f; // minimum speculative decoding probability (greedy) @@ -394,7 +395,11 @@ struct common_params_speculative { uint32_t need_n_rs_seq() const { bool needs_rs_seq = std::any_of(types.begin(), types.end(), [&](auto t) { - return t == COMMON_SPECULATIVE_TYPE_DRAFT_MTP || t == COMMON_SPECULATIVE_TYPE_DRAFT_MTP_ADAPTIVE || t == COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3 || t == COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH || t == COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK; + return t == COMMON_SPECULATIVE_TYPE_DRAFT_MTP || + t == COMMON_SPECULATIVE_TYPE_DRAFT_MTP_ADAPTIVE || + t == COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3 || + t == COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH || + t == COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK; }); return needs_rs_seq ? draft.n_max : 0u; diff --git a/common/speculative-adaptive.h b/common/speculative-adaptive.h new file mode 100644 index 00000000000..644664c6a31 --- /dev/null +++ b/common/speculative-adaptive.h @@ -0,0 +1,92 @@ +#pragma once + +#include + +// Adaptive draft depth controller for MTP speculative decoding (draft-mtp-adaptive). +// +// Hysteresis state machine with a climb counter and a weighted drop-pressure +// accumulator. The depth N climbs one step after N_CLIMB(N) consecutive verifies +// that accepted every drafted token. The climb cost is low at the floor and at +// depth, high in the middle: 2 at depth 1, 4 at depth 2, 6 at depth 3, then +// 5/4/3/2 from depth 4 upward. Getting from the floor to depth 3 needs only 6 +// full accepts, but pushing past 3 (where prose acceptance collapses) costs 6 +// full accepts of 3-token drafts, which predictable content clears quickly and +// marginal content never does. Any miss adds (n_draft - n_accepted) to a +// drop-pressure accumulator; when it reaches depth * 5 the depth drops one step +// and the pressure resets. A near miss (n_draft-1) adds 1, a total miss adds +// n_draft, so high depths fall quickly while low depths hold. The drop budget +// scales with depth but never drops below 20, so shallow depths shed bad content +// quickly without collapsing to the floor on a few bad rounds; deep depths hold +// a little longer. At the floor no pressure accumulates at all. The depth starts +// at the floor max(1, --spec-draft-n-min-adaptive) and stays in +// [floor, n_max]; --spec-draft-n-max bounds the upper end of the adaptive +// range. +struct common_speculative_adaptive { + int n_cur = 0; // current adaptive draft depth N + int n_climb = 0; // consecutive verifies that accepted every drafted token + int n_drop = 0; // accumulated drop pressure: sum of (n_draft - n_accepted) + + // consecutive full accepts needed to climb one step from depth N; low at the + // floor and at depth, high in the middle where acceptance is marginal + static int climb_threshold(int depth) { + switch (depth) { + case 1: return 2; + case 2: return 4; + case 3: return 6; + case 4: return 5; + case 5: return 4; + case 6: return 3; + default: return 2; // depth >= 7 + } + } + + // accumulated (n_draft - n_accepted) needed to drop one step from depth N; + // scaled by depth, with a floor so shallow depths do not collapse too fast + static int drop_pressure(int depth) { + return std::max(depth * 5, 20); + } + + // reset to the floor max(1, n_min_adaptive), bounded by the ceiling n_max; + // the controller climbs from there once acceptance feedback arrives + void reset(int n_max, int n_min_adaptive) { + const int cap = std::max(1, n_max); + const int floor = std::max(1, n_min_adaptive); + + n_cur = std::min(floor, cap); + n_climb = 0; + n_drop = 0; + } + + // feed one verification result: n_draft is the number of tokens this + // implementation drafted, n_accepted the number the target accepted + void update(int n_draft, int n_accepted, int n_max, int n_min_adaptive) { + if (n_draft <= 0) { + return; + } + + const int cap = std::max(1, n_max); + const int floor = std::max(1, n_min_adaptive); + + if (n_accepted == n_draft) { + n_drop = 0; + + // full acceptance: reset the drop pressure, accumulate the climb streak + if (n_cur < cap && ++n_climb >= climb_threshold(n_cur)) { + n_cur++; + n_climb = 0; + } + } else { + n_climb = 0; + + // any miss adds (n_draft - n_accepted) to the drop pressure; drop one + // step when the accumulated pressure reaches the depth-scaled budget + if (n_cur > floor) { + n_drop += n_draft - n_accepted; + if (n_drop >= drop_pressure(n_cur)) { + n_cur--; + n_drop = 0; + } + } + } + } +}; diff --git a/common/speculative.cpp b/common/speculative.cpp index c4d7874524f..3db7c0e8423 100644 --- a/common/speculative.cpp +++ b/common/speculative.cpp @@ -9,6 +9,7 @@ #include "ngram-map.h" #include "ngram-mod.h" #include "sampling.h" +#include "speculative-adaptive.h" #include "../src/llama-ext.h" // staging API: llama_set_embeddings_nextn / llama_get_embeddings_nextn_ith (used by MTP) @@ -1358,70 +1359,11 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { std::vector i_last; std::vector> chain_h; - // Adaptive draft depth (draft-mtp-adaptive): hysteresis state machine with two - // counters per sequence. The depth N climbs after N_CLIMB(N) consecutive verifies - // that accepted every drafted token, where the requirement shrinks with depth: - // 5 full accepts below depth 5, 4 at depth 5-6, 3 at depth 7+. Leaving the floor - // must really be earned, but sustained acceptance that carried the depth this far - // earns the next step cheaply. Any miss adds (N - acceptance) to a drop-pressure - // accumulator; when it exceeds ADAPTIVE_DROP_PRESSURE the depth drops one step - // and the pressure resets. A near miss (N-1) adds 1, a total miss at depth N adds - // N, so high depths fall quickly while low depths hold. The floor is max(1, n_min) - // and the ceiling is n_max (--spec-draft-n-max). + // Adaptive draft depth (draft-mtp-adaptive), see common_speculative_adaptive bool adaptive = false; - std::vector n_cap; // [n_seq] effective draft cap for the current draft() call - std::vector n_cur; // [n_seq] current adaptive depth N - std::vector n_last; // [n_seq] drafts attempted in the most recent draft() call - std::vector n_climb; // [n_seq] consecutive fully-accepted verifies - std::vector n_drop; // [n_seq] accumulated drop pressure: sum of (N - acceptance) - - static constexpr int ADAPTIVE_DROP_PRESSURE = 30; // accumulated (N - acceptance) to drop one step - - // consecutive full accepts needed to climb one step from depth N; cheaper the - // deeper we are, so predictable content accelerates while prose must earn its - // way off the floor - static int adaptive_climb_threshold(int depth) { - return depth >= 7 ? 3 : (depth >= 5 ? 4 : 5); - } - - void adaptive_update(llama_seq_id seq_id, int n_draft, int n_accepted) { - if (n_draft <= 0) { - return; - } - - const int floor = std::max(1, params.n_min); - const int n_max = std::max(1, params.n_max); - - int depth = n_cur[seq_id]; - - if (n_accepted == depth) { - n_drop[seq_id] = 0; - - // full acceptance: reset the drop pressure, accumulate the climb streak - if ((depth < n_max) && (++n_climb[seq_id] >= adaptive_climb_threshold(depth))) { - depth++; - n_climb[seq_id] = 0; - } - } else { - n_climb[seq_id] = 0; - - if (depth > floor) { - // any miss adds (N - acceptance) to the drop pressure; drop when the - // accumulated pressure exceeds the threshold - n_drop[seq_id] += depth - n_accepted; - if (n_drop[seq_id] > ADAPTIVE_DROP_PRESSURE) { - depth--; - n_drop[seq_id] = 0; - } - } - } - - if (depth != n_cur[seq_id]) { - SPC_DBG("adaptive draft depth seq %d: %d -> %d (n_draft=%d, n_accepted=%d)\n", - (int) seq_id, n_cur[seq_id], depth, n_draft, n_accepted); - } - n_cur[seq_id] = depth; - } + std::vector n_cap; // [n_seq] effective draft cap for the current draft() call + std::vector n_last; // [n_seq] drafts attempted in the most recent draft() call + std::vector adaptive_ctrl; // [n_seq] per-seq adaptive depth controller common_speculative_impl_draft_mtp(const common_params_speculative & params, uint32_t n_seq, bool adaptive = false) : common_speculative_impl(adaptive ? COMMON_SPECULATIVE_TYPE_DRAFT_MTP_ADAPTIVE : COMMON_SPECULATIVE_TYPE_DRAFT_MTP, n_seq, params.draft.n_max) @@ -1440,18 +1382,6 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { // n_cap/n_last are written by the shared draft loop in both modes n_cap.assign(n_seq, 0); n_last.assign(n_seq, 0); - if (adaptive) { - const int n_max = std::max(1, this->params.n_max); - n_cur.assign(n_seq, 0); - n_climb.assign(n_seq, 0); - n_drop.assign(n_seq, 0); - for (uint32_t s = 0; s < n_seq; ++s) { - // start at a safe depth of 3 (bounded by n_min/n_max); the controller - // moves it in [max(1,n_min), n_max] once acceptance feedback arrives - n_cur[s] = std::min(std::max(3, std::max(1, this->params.n_min)), n_max); - } - SPC_TRC("%s", "adaptive draft depth enabled (draft-mtp-adaptive)\n"); - } SPC_TRC("%s", "adding speculative implementation 'draft-mtp'\n"); SPC_TRC("- n_max=%d, n_min=%d, p_min=%.2f, n_embd=%d, backend_sampling=%d\n", this->params.n_max, this->params.n_min, this->params.p_min, n_embd, (int) this->params.backend_sampling); @@ -1510,6 +1440,23 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { } this->n_max = this->params.n_max; + // a floor above the ceiling would pin the depth below the floor, so the + // configuration is invalid + if (this->params.n_min_adaptive < 1 || this->params.n_min_adaptive > this->params.n_max) { + GGML_ABORT("%s: invalid adaptive draft range: n_min_adaptive=%d, n_max=%d (n_min_adaptive must be in [1, n_max])", + __func__, this->params.n_min_adaptive, this->params.n_max); + } + + if (adaptive) { + adaptive_ctrl.assign(n_seq, common_speculative_adaptive()); + for (uint32_t s = 0; s < n_seq; ++s) { + // start at the floor max(1, n_min_adaptive), bounded by n_max; + // the controller climbs from there once acceptance feedback arrives + adaptive_ctrl[s].reset(this->params.n_max, this->params.n_min_adaptive); + } + SPC_TRC("%s", "adaptive draft depth enabled (draft-mtp-adaptive)\n"); + } + pending_h.assign(n_seq, std::vector(n_embd, 0.0f)); i_last.assign(n_seq, -1); @@ -1700,7 +1647,7 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { // effective draft cap for this step: adaptive depth (or the user n_max), // then clamped by the per-call context bound from the server - n_cap[seq_id] = adaptive ? n_cur[seq_id] : params.n_max; + n_cap[seq_id] = adaptive ? adaptive_ctrl[seq_id].n_cur : params.n_max; if (dp.n_max > 0 && dp.n_max < n_cap[seq_id]) { n_cap[seq_id] = dp.n_max; } @@ -1830,19 +1777,28 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { n_last[seq_id] = (int) dp.result->size(); - if (dp.result->size() < (size_t) params.n_min) { + // the adaptive controller decides its own depth, so the generic n_min + // draft cutoff does not apply to it + if (!adaptive && dp.result->size() < (size_t) params.n_min) { dp.result->clear(); } } } - void accept(llama_seq_id seq_id, uint16_t n_accepted, bool /*is_other*/) override { + void accept(llama_seq_id seq_id, uint16_t n_accepted, bool is_other) override { if (seq_id < 0 || seq_id >= (llama_seq_id) n_seq) { return; } - if (adaptive) { - adaptive_update(seq_id, n_last[seq_id], n_accepted); + // update the adaptive controller only when this implementation produced the + // accepted draft; on is_other the stats belong to a different speculator + if (adaptive && !is_other) { + const int depth_before = adaptive_ctrl[seq_id].n_cur; + adaptive_ctrl[seq_id].update(n_last[seq_id], n_accepted, params.n_max, params.n_min_adaptive); + if (adaptive_ctrl[seq_id].n_cur != depth_before) { + SPC_DBG("adaptive draft depth seq %d: %d -> %d (n_draft=%d, n_accepted=%d)\n", + (int) seq_id, depth_before, adaptive_ctrl[seq_id].n_cur, n_last[seq_id], n_accepted); + } } const int32_t n_rows = verify_h_rows[seq_id]; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index c46377c7623..43bdc89dfb5 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -289,6 +289,7 @@ llama_build_and_test(test-thread-safety.cpp ARGS -m "${MODEL_DEST}" -ngl 99 -p " set_tests_properties(test-thread-safety PROPERTIES FIXTURES_REQUIRED test-download-model) llama_build_and_test(test-arg-parser.cpp) +llama_build_and_test(test-speculative-adaptive.cpp) llama_build_and_test(test-model-resolution.cpp) # the test serves its repos from an httplib server, and the library links it privately target_link_libraries(test-model-resolution PRIVATE cpp-httplib) diff --git a/tests/test-arg-parser.cpp b/tests/test-arg-parser.cpp index e0907631abd..84f1ed98684 100644 --- a/tests/test-arg-parser.cpp +++ b/tests/test-arg-parser.cpp @@ -274,6 +274,21 @@ static void test(void) { assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), synth_params, LLAMA_EXAMPLE_SERVER)); } + // the adaptive floor defaults to 2 and parses explicitly + argv = {"binary_name", "-m", "model_file.gguf"}; + assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_SPECULATIVE)); + assert(params.speculative.draft.n_min_adaptive == 3); + argv = {"binary_name", "-m", "model_file.gguf", "--spec-draft-n-min-adaptive", "5"}; + assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_SPECULATIVE)); + assert(params.speculative.draft.n_min_adaptive == 5); + + // the adaptive MTP type parses to the dedicated enum value + argv = {"binary_name", "-m", "model_file.gguf", "--spec-type", "draft-mtp-adaptive"}; + common_params spec_params; + assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), spec_params, LLAMA_EXAMPLE_SPECULATIVE)); + assert(std::find(spec_params.speculative.types.begin(), spec_params.speculative.types.end(), + COMMON_SPECULATIVE_TYPE_DRAFT_MTP_ADAPTIVE) != spec_params.speculative.types.end()); + argv = {"binary_name", "-lm", "none"}; assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON)); assert(params.load_mode == LLAMA_LOAD_MODE_NONE); diff --git a/tests/test-speculative-adaptive.cpp b/tests/test-speculative-adaptive.cpp new file mode 100644 index 00000000000..9e3f3722d90 --- /dev/null +++ b/tests/test-speculative-adaptive.cpp @@ -0,0 +1,219 @@ +#include "speculative-adaptive.h" + +#include +#include + +#undef NDEBUG + +static void test_reset(void) { + common_speculative_adaptive ctrl; + + // cold start at the floor max(1, n_min_adaptive) + ctrl.reset(8, 1); + assert(ctrl.n_cur == 1); + assert(ctrl.n_climb == 0); + assert(ctrl.n_drop == 0); + + // the default adaptive floor of 3 starts the controller at depth 3 + ctrl.reset(8, 3); + assert(ctrl.n_cur == 3); + + // the ceiling clamps the cold start to n_max + ctrl.reset(1, 3); + assert(ctrl.n_cur == 1); +} + +static void test_climb(void) { + common_speculative_adaptive ctrl; + ctrl.reset(8, 1); // ceiling 8, cold start at the floor 1 + + // depth 1 climbs after 2 consecutive full accepts + ctrl.update(1, 1, 8, 1); + assert(ctrl.n_cur == 1); + ctrl.update(1, 1, 8, 1); + assert(ctrl.n_cur == 2); + + // a miss resets the climb streak + ctrl.update(2, 1, 8, 1); // near miss + assert(ctrl.n_cur == 2); + assert(ctrl.n_climb == 0); + + // depth 2 climbs after 4 consecutive full accepts + for (int i = 0; i < 3; ++i) { + ctrl.update(2, 2, 8, 1); + assert(ctrl.n_cur == 2); + } + ctrl.update(2, 2, 8, 1); + assert(ctrl.n_cur == 3); + + // depth 3 is the barrier: 6 consecutive full accepts to reach depth 4 + for (int i = 0; i < 5; ++i) { + ctrl.update(3, 3, 8, 1); + assert(ctrl.n_cur == 3); + } + ctrl.update(3, 3, 8, 1); + assert(ctrl.n_cur == 4); + + // a full accept of a draft truncated below the depth (e.g. clamped by the + // server context bound) counts as a full accept, not as a miss + ctrl.update(3, 3, 8, 1); // depth 4, only 3 tokens drafted, all accepted + assert(ctrl.n_climb == 1); + assert(ctrl.n_drop == 0); + + // depth 4-5 need 5 consecutive full accepts + for (int i = 0; i < 4; ++i) { + ctrl.update(4, 4, 8, 1); + } + assert(ctrl.n_cur == 5); + + // depth 5 needs 4 consecutive full accepts + for (int i = 0; i < 3; ++i) { + ctrl.update(5, 5, 8, 1); + assert(ctrl.n_cur == 5); + } + ctrl.update(5, 5, 8, 1); + assert(ctrl.n_cur == 6); + + // depth 6 needs 3 consecutive full accepts + for (int i = 0; i < 2; ++i) { + ctrl.update(6, 6, 8, 1); + assert(ctrl.n_cur == 6); + } + ctrl.update(6, 6, 8, 1); + assert(ctrl.n_cur == 7); + + // depth 7+ needs 2 consecutive full accepts + ctrl.update(7, 7, 8, 1); + assert(ctrl.n_cur == 7); + ctrl.update(7, 7, 8, 1); + assert(ctrl.n_cur == 8); + + // the ceiling blocks further climbs + for (int i = 0; i < 8; ++i) { + ctrl.update(8, 8, 8, 1); + } + assert(ctrl.n_cur == 8); + + // no feedback for a zero-length draft + ctrl.update(0, 0, 8, 1); + assert(ctrl.n_cur == 8); + assert(ctrl.n_climb == 0); +} + +static void test_drop(void) { + common_speculative_adaptive ctrl; + ctrl.reset(8, 1); // cold start at the floor + + // at the floor no pressure accumulates at all + for (int i = 0; i < 100; ++i) { + ctrl.update(1, 0, 8, 1); + } + assert(ctrl.n_cur == 1); + assert(ctrl.n_drop == 0); + + // climb to depth 3 (2 + 4 full accepts) + for (int i = 0; i < 2; ++i) { + ctrl.update(1, 1, 8, 1); + } + for (int i = 0; i < 4; ++i) { + ctrl.update(2, 2, 8, 1); + } + assert(ctrl.n_cur == 3); + + // at depth 3 the drop budget is floored at 20: a total miss adds 3, so + // 7 misses drop one step + for (int i = 0; i < 6; ++i) { + ctrl.update(3, 0, 8, 1); + assert(ctrl.n_cur == 3); + } + assert(ctrl.n_drop == 18); + ctrl.update(3, 0, 8, 1); + assert(ctrl.n_cur == 2); + assert(ctrl.n_drop == 0); + + // at depth 2 the budget is floored at 20: a near miss adds 1, so 20 near + // misses drop one step (the depth-1 collapse needs real sustained failure) + for (int i = 0; i < 19; ++i) { + ctrl.update(2, 1, 8, 1); + assert(ctrl.n_cur == 2); + } + assert(ctrl.n_drop == 19); + ctrl.update(2, 1, 8, 1); + assert(ctrl.n_cur == 1); + + // back at the floor, misses no longer accumulate pressure + for (int i = 0; i < 100; ++i) { + ctrl.update(1, 0, 8, 1); + } + assert(ctrl.n_cur == 1); + assert(ctrl.n_drop == 0); + + // deep depths hold a little longer: at depth 5 the budget is 25, a total + // miss adds 5, so 5 misses drop one step + ctrl.reset(8, 1); + ctrl.n_cur = 5; // simulate a controller that already climbed to 5 + for (int i = 0; i < 4; ++i) { + ctrl.update(5, 0, 8, 1); + assert(ctrl.n_cur == 5); + } + assert(ctrl.n_drop == 20); + ctrl.update(5, 0, 8, 1); // 20 + 5 = 25 -> drop + assert(ctrl.n_cur == 4); + assert(ctrl.n_drop == 0); +} + +static void test_full_accept_resets_pressure(void) { + common_speculative_adaptive ctrl; + ctrl.reset(8, 1); + ctrl.n_cur = 3; + + // accumulate pressure, then a full accept wipes it out + for (int i = 0; i < 5; ++i) { + ctrl.update(3, 1, 8, 1); // near miss: +2 pressure at depth 3 + } + assert(ctrl.n_drop == 10); + ctrl.update(3, 3, 8, 1); + assert(ctrl.n_drop == 0); + + // the miss pressure uses the drafted count, not the depth: a truncated + // draft (2 tokens at depth 3) with 1 accepted adds 1, not 2 + ctrl.update(2, 1, 8, 1); + assert(ctrl.n_drop == 1); +} + +static void test_floor(void) { + common_speculative_adaptive ctrl; + + // with the floor at 2 the depth never drops below 2, no matter how bad + // the content gets + ctrl.reset(8, 2); + for (int i = 0; i < 1000; ++i) { + ctrl.update(2, 0, 8, 2); + } + assert(ctrl.n_cur == 2); + assert(ctrl.n_drop == 0); + + // climbs still work from the floor + for (int i = 0; i < 4; ++i) { + ctrl.update(2, 2, 8, 2); + } + assert(ctrl.n_cur == 3); + + // and drops stop at the floor, not below it + for (int i = 0; i < 100; ++i) { + ctrl.update(3, 0, 8, 2); + } + assert(ctrl.n_cur == 2); +} + +int main(void) { + test_reset(); + test_climb(); + test_drop(); + test_full_accept_resets_pressure(); + test_floor(); + + printf("test-speculative-adaptive: all tests OK\n\n"); + + return 0; +} From 45a71c536206b19aff93b6ff0574cad0a9aaf4f5 Mon Sep 17 00:00:00 2001 From: Stew Forster Date: Tue, 18 Aug 2026 14:41:32 -0500 Subject: [PATCH 3/6] spec : fix adaptive draft range check for plain draft-mtp n_min_adaptive only applies to draft-mtp-adaptive, but the range check ran unconditionally in the shared MTP ctor, so plain draft-mtp aborted whenever the effective n_max was below the default floor of 3 (e.g. --spec-draft-n-max 2). Gate the check on adaptive mode, and when the chain_heads clamp capped n_max at the model MTP layer count, say so in the abort message. Assisted-by: pi --- common/speculative.cpp | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/common/speculative.cpp b/common/speculative.cpp index 3db7c0e8423..5099c7f1062 100644 --- a/common/speculative.cpp +++ b/common/speculative.cpp @@ -1430,6 +1430,9 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { is_mem_shared = llama_get_ctx_other(ctx_dft) == ctx_tgt; chain_heads = n_mtp_layers > 1 && !is_mem_shared; + // remember the user n_max: chain_heads caps it at the model MTP layer + // count, and the adaptive range abort below must explain the cap + const int32_t n_max_user = this->params.n_max; if (chain_heads) { this->params.n_max = std::min(this->params.n_max, n_mtp_layers); @@ -1440,14 +1443,19 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { } this->n_max = this->params.n_max; - // a floor above the ceiling would pin the depth below the floor, so the - // configuration is invalid - if (this->params.n_min_adaptive < 1 || this->params.n_min_adaptive > this->params.n_max) { - GGML_ABORT("%s: invalid adaptive draft range: n_min_adaptive=%d, n_max=%d (n_min_adaptive must be in [1, n_max])", - __func__, this->params.n_min_adaptive, this->params.n_max); - } - if (adaptive) { + // a floor above the ceiling would pin the depth below the floor, so the + // configuration is invalid + if (this->params.n_min_adaptive < 1 || this->params.n_min_adaptive > this->params.n_max) { + if (n_max_user > this->params.n_max) { + // n_max was capped by the MTP layer count, not by the user + GGML_ABORT("%s: invalid adaptive draft range: n_min_adaptive=%d, n_max=%d (n_max is capped by the model MTP layer count %d; set --spec-draft-n-min-adaptive to at most %d)", + __func__, this->params.n_min_adaptive, this->params.n_max, n_mtp_layers, n_mtp_layers); + } + GGML_ABORT("%s: invalid adaptive draft range: n_min_adaptive=%d, n_max=%d (n_min_adaptive must be in [1, n_max])", + __func__, this->params.n_min_adaptive, this->params.n_max); + } + adaptive_ctrl.assign(n_seq, common_speculative_adaptive()); for (uint32_t s = 0; s < n_seq; ++s) { // start at the floor max(1, n_min_adaptive), bounded by n_max; From 8b6bd9b1a4b846212503a26847d93362ce261aea Mon Sep 17 00:00:00 2001 From: Stew Forster Date: Wed, 19 Aug 2026 08:42:32 -0500 Subject: [PATCH 4/6] spec : reset adaptive MTP draft depth on new generation Assisted-by: pi --- common/speculative.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/common/speculative.cpp b/common/speculative.cpp index 5099c7f1062..864b91b96c9 100644 --- a/common/speculative.cpp +++ b/common/speculative.cpp @@ -1501,6 +1501,12 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { return; } + // new generation: the depth learned for the previous content is stale, + // so the controller starts from the floor again + if (adaptive) { + adaptive_ctrl[seq_id].reset(this->params.n_max, this->params.n_min_adaptive); + } + auto * ctx_dft = this->params.ctx_dft; const llama_pos pos_max = llama_memory_seq_pos_max(llama_get_memory(ctx_dft), seq_id); From 1cb60e778272e4c18800cf03b6750287306649fd Mon Sep 17 00:00:00 2001 From: Stew Forster Date: Sat, 22 Aug 2026 19:58:15 -0500 Subject: [PATCH 5/6] spec : tune adaptive MTP climb table and fix rs snapshot copies climb_threshold: the 3->4 barrier is hardened to 10 consecutive full accepts so prose/reasoning stay pinned at the floor; 4->5 raised to 6; 5->6 and 6->7 lowered to 3 and 2 so code accelerates to the deep hold without over-drafting into the marginal depths. recurrent snapshot fix: the conv-state loop wrote n_rs_seq + 1 copies per layer per round, but a rollback can only reach n_seq_tokens - 1 slots back, so copies beyond the batch were dead work (~2.6% per-round overhead at n_rs_seq=10 on shallow verifies). Start the loop at max(1, K - n_seq_tokens + 1). tests: move the #undef NDEBUG before so the asserts actually run in Release builds (they were silent no-ops), and re-derive the climb expectations for the new table. Assisted-by: Pi --- common/speculative-adaptive.h | 8 ++++---- src/models/delta-net-base.cpp | 7 ++++++- tests/test-speculative-adaptive.cpp | 25 ++++++++++++------------- 3 files changed, 22 insertions(+), 18 deletions(-) diff --git a/common/speculative-adaptive.h b/common/speculative-adaptive.h index 644664c6a31..ab9f98818fb 100644 --- a/common/speculative-adaptive.h +++ b/common/speculative-adaptive.h @@ -32,10 +32,10 @@ struct common_speculative_adaptive { switch (depth) { case 1: return 2; case 2: return 4; - case 3: return 6; - case 4: return 5; - case 5: return 4; - case 6: return 3; + case 3: return 10; // hardened 3->4 barrier: keeps prose/reasoning pinned + case 4: return 6; + case 5: return 3; + case 6: return 2; default: return 2; // depth >= 7 } } diff --git a/src/models/delta-net-base.cpp b/src/models/delta-net-base.cpp index ad661264773..30b5f50f1f6 100644 --- a/src/models/delta-net-base.cpp +++ b/src/models/delta-net-base.cpp @@ -501,7 +501,12 @@ ggml_tensor * llm_build_delta_net_base::build_conv_state( const int64_t K = (int64_t) cparams.n_rs_seq + 1; - for (int64_t t = 1; t <= K; ++t) { + // only the snapshot slots reachable by a rollback inside this batch are + // useful: rollback <= n_seq_tokens - 1, so slots beyond the batch repeat + // the pre-batch state and would only waste kernel launches per round + const int64_t t_min = std::max(1, K - ubatch.n_seq_tokens + 1); + + for (int64_t t = t_min; t <= K; ++t) { const int64_t s_idx = std::max(0, conv_input->ne[0] - conv_states->ne[0] - K + t); const int64_t s_slot = K - t; diff --git a/tests/test-speculative-adaptive.cpp b/tests/test-speculative-adaptive.cpp index 9e3f3722d90..ea959f22329 100644 --- a/tests/test-speculative-adaptive.cpp +++ b/tests/test-speculative-adaptive.cpp @@ -1,10 +1,10 @@ #include "speculative-adaptive.h" +#undef NDEBUG + #include #include -#undef NDEBUG - static void test_reset(void) { common_speculative_adaptive ctrl; @@ -46,8 +46,9 @@ static void test_climb(void) { ctrl.update(2, 2, 8, 1); assert(ctrl.n_cur == 3); - // depth 3 is the barrier: 6 consecutive full accepts to reach depth 4 - for (int i = 0; i < 5; ++i) { + // depth 3 is the hardened barrier: 10 consecutive full accepts to reach + // depth 4, so prose/reasoning stay pinned at the floor + for (int i = 0; i < 9; ++i) { ctrl.update(3, 3, 8, 1); assert(ctrl.n_cur == 3); } @@ -60,25 +61,23 @@ static void test_climb(void) { assert(ctrl.n_climb == 1); assert(ctrl.n_drop == 0); - // depth 4-5 need 5 consecutive full accepts - for (int i = 0; i < 4; ++i) { + // depth 4 needs 6 consecutive full accepts + for (int i = 0; i < 5; ++i) { ctrl.update(4, 4, 8, 1); } assert(ctrl.n_cur == 5); - // depth 5 needs 4 consecutive full accepts - for (int i = 0; i < 3; ++i) { + // depth 5 needs 3 consecutive full accepts + for (int i = 0; i < 2; ++i) { ctrl.update(5, 5, 8, 1); assert(ctrl.n_cur == 5); } ctrl.update(5, 5, 8, 1); assert(ctrl.n_cur == 6); - // depth 6 needs 3 consecutive full accepts - for (int i = 0; i < 2; ++i) { - ctrl.update(6, 6, 8, 1); - assert(ctrl.n_cur == 6); - } + // depth 6 needs 2 consecutive full accepts + ctrl.update(6, 6, 8, 1); + assert(ctrl.n_cur == 6); ctrl.update(6, 6, 8, 1); assert(ctrl.n_cur == 7); From 45e3d2661e68168b4e5821a2cc58dac29d9b1897 Mon Sep 17 00:00:00 2001 From: Stew Forster Date: Tue, 1 Sep 2026 20:42:39 -0500 Subject: [PATCH 6/6] Rebase to master tip and fix comments I rebased the code against the latest master tip and corrected a handful of outdated comments --- common/speculative-adaptive.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/common/speculative-adaptive.h b/common/speculative-adaptive.h index ab9f98818fb..2da0e68d0eb 100644 --- a/common/speculative-adaptive.h +++ b/common/speculative-adaptive.h @@ -7,9 +7,9 @@ // Hysteresis state machine with a climb counter and a weighted drop-pressure // accumulator. The depth N climbs one step after N_CLIMB(N) consecutive verifies // that accepted every drafted token. The climb cost is low at the floor and at -// depth, high in the middle: 2 at depth 1, 4 at depth 2, 6 at depth 3, then -// 5/4/3/2 from depth 4 upward. Getting from the floor to depth 3 needs only 6 -// full accepts, but pushing past 3 (where prose acceptance collapses) costs 6 +// depth, high in the middle: 2 at depth 1, 4 at depth 2, 10 at depth 3, then +// 6/3/2/2 from depth 4 upward. Getting from the floor to depth 3 needs only 6 +// full accepts, but pushing past 3 (where prose acceptance collapses) costs 10 // full accepts of 3-token drafts, which predictable content clears quickly and // marginal content never does. Any miss adds (n_draft - n_accepted) to a // drop-pressure accumulator; when it reaches depth * 5 the depth drops one step