diff --git a/common/arg.cpp b/common/arg.cpp index 79405b59e07..f141c59811c 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -363,9 +363,8 @@ common_models_handler common_models_handler_init(const common_params & params, l common_download_hf_plan plan_spec; common_download_opts opts; - const bool spec_type_draft_mtp = std::find(params.speculative.types.begin(), - params.speculative.types.end(), - COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params.speculative.types.end(); + const bool spec_type_draft_mtp = std::find(params.speculative.types.begin(), params.speculative.types.end(), COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params.speculative.types.end() || + std::find(params.speculative.types.begin(), params.speculative.types.end(), COMMON_SPECULATIVE_TYPE_DRAFT_MTP_ADAPTIVE) != params.speculative.types.end(); const bool spec_type_draft_dflash = std::find(params.speculative.types.begin(), params.speculative.types.end(), @@ -1303,6 +1302,7 @@ bool common_params_parse(int argc, char ** argv, common_params & params, llama_e common_params_print_completion(ctx_arg); exit(0); } + params.lr.init(); } catch (const std::invalid_argument & ex) { fprintf(stderr, "%s\n", ex.what()); @@ -4186,6 +4186,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex } ).set_spec().set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_SYNTH_RATES")); + add_opt(common_arg( + {"--spec-draft-n-min-adaptive"}, "N", + string_format("minimum adaptive MTP draft depth; the depth starts here and never drops below it (default: %d)", params.speculative.draft.n_min_adaptive), + [](common_params & params, int value) { + params.speculative.draft.n_min_adaptive = value; + } + ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_N_MIN_ADAPTIVE")); + add_opt(common_arg( {"--spec-draft-p-split", "--draft-p-split"}, "P", string_format("speculative decoding split probability (default: %.2f)", (double)params.speculative.draft.p_split), diff --git a/common/common.cpp b/common/common.cpp index d162a38800e..577a43c7f98 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -1711,7 +1711,8 @@ struct llama_model_params common_model_params_to_llama(common_params & params) { mparams.progress_callback = params.load_progress_callback; mparams.progress_callback_user_data = params.load_progress_callback_user_data; mparams.no_alloc = params.no_alloc; - mparams.load_mtp = std::find(params.speculative.types.begin(), params.speculative.types.end(), COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params.speculative.types.end(); + mparams.load_mtp = std::find(params.speculative.types.begin(), params.speculative.types.end(), COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params.speculative.types.end() || + std::find(params.speculative.types.begin(), params.speculative.types.end(), COMMON_SPECULATIVE_TYPE_DRAFT_MTP_ADAPTIVE) != params.speculative.types.end(); return mparams; } diff --git a/common/common.h b/common/common.h index 4e9448bb106..f161b08c12d 100644 --- a/common/common.h +++ b/common/common.h @@ -169,18 +169,19 @@ enum common_params_sampling_config : uint64_t { }; enum common_speculative_type { - COMMON_SPECULATIVE_TYPE_NONE, // no speculative decoding - COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE, // standalone draft model speculative decoding - COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3, // Eagle3 speculative decoding - COMMON_SPECULATIVE_TYPE_DRAFT_MTP, // Multi-token prediction - COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH, // DFlash speculative decoding - COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK, // DSpark speculative decoding (DFlash + Markov head) - COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE, // simple self-speculative decoding based on n-grams - COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K, // self-speculative decoding with n-gram keys only - COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V, // self-speculative decoding with n-gram keys and 4 m-gram values + COMMON_SPECULATIVE_TYPE_NONE, // no speculative decoding + COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE, // standalone draft model speculative decoding + COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3, // Eagle3 speculative decoding + COMMON_SPECULATIVE_TYPE_DRAFT_MTP, // Multi-token prediction + COMMON_SPECULATIVE_TYPE_DRAFT_MTP_ADAPTIVE, // MTP with adaptive draft depth + COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH, // DFlash speculative decoding + COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK, // DSpark speculative decoding (DFlash + Markov head) + COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE, // simple self-speculative decoding based on n-grams + COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K, // self-speculative decoding with n-gram keys only + COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V, // self-speculative decoding with n-gram keys and 4 m-gram values COMMON_SPECULATIVE_TYPE_NGRAM_MOD, - COMMON_SPECULATIVE_TYPE_NGRAM_CACHE, // self-speculative decoding with 3-level n-gram cache - COMMON_SPECULATIVE_TYPE_COUNT // number of types, unknown type + COMMON_SPECULATIVE_TYPE_NGRAM_CACHE, // self-speculative decoding with 3-level n-gram cache + COMMON_SPECULATIVE_TYPE_COUNT // number of types, unknown type }; // Grammar type enumeration @@ -325,6 +326,7 @@ struct common_params_model { struct common_params_speculative_draft { int32_t n_max = 3; // maximum number of tokens to draft during speculative decoding int32_t n_min = 0; // minimum number of draft tokens to use for speculative decoding + int32_t n_min_adaptive = 3; // minimum adaptive MTP draft depth (also the starting depth) float p_split = 0.1f; // speculative decoding split probability float p_min = 0.0f; // minimum speculative decoding probability (greedy) @@ -393,7 +395,11 @@ struct common_params_speculative { uint32_t need_n_rs_seq() const { bool needs_rs_seq = std::any_of(types.begin(), types.end(), [&](auto t) { - return t == COMMON_SPECULATIVE_TYPE_DRAFT_MTP || t == COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3 || t == COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH || t == COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK; + return t == COMMON_SPECULATIVE_TYPE_DRAFT_MTP || + t == COMMON_SPECULATIVE_TYPE_DRAFT_MTP_ADAPTIVE || + t == COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3 || + t == COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH || + t == COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK; }); return needs_rs_seq ? draft.n_max : 0u; diff --git a/common/speculative-adaptive.h b/common/speculative-adaptive.h new file mode 100644 index 00000000000..2da0e68d0eb --- /dev/null +++ b/common/speculative-adaptive.h @@ -0,0 +1,92 @@ +#pragma once + +#include + +// Adaptive draft depth controller for MTP speculative decoding (draft-mtp-adaptive). +// +// Hysteresis state machine with a climb counter and a weighted drop-pressure +// accumulator. The depth N climbs one step after N_CLIMB(N) consecutive verifies +// that accepted every drafted token. The climb cost is low at the floor and at +// depth, high in the middle: 2 at depth 1, 4 at depth 2, 10 at depth 3, then +// 6/3/2/2 from depth 4 upward. Getting from the floor to depth 3 needs only 6 +// full accepts, but pushing past 3 (where prose acceptance collapses) costs 10 +// full accepts of 3-token drafts, which predictable content clears quickly and +// marginal content never does. Any miss adds (n_draft - n_accepted) to a +// drop-pressure accumulator; when it reaches depth * 5 the depth drops one step +// and the pressure resets. A near miss (n_draft-1) adds 1, a total miss adds +// n_draft, so high depths fall quickly while low depths hold. The drop budget +// scales with depth but never drops below 20, so shallow depths shed bad content +// quickly without collapsing to the floor on a few bad rounds; deep depths hold +// a little longer. At the floor no pressure accumulates at all. The depth starts +// at the floor max(1, --spec-draft-n-min-adaptive) and stays in +// [floor, n_max]; --spec-draft-n-max bounds the upper end of the adaptive +// range. +struct common_speculative_adaptive { + int n_cur = 0; // current adaptive draft depth N + int n_climb = 0; // consecutive verifies that accepted every drafted token + int n_drop = 0; // accumulated drop pressure: sum of (n_draft - n_accepted) + + // consecutive full accepts needed to climb one step from depth N; low at the + // floor and at depth, high in the middle where acceptance is marginal + static int climb_threshold(int depth) { + switch (depth) { + case 1: return 2; + case 2: return 4; + case 3: return 10; // hardened 3->4 barrier: keeps prose/reasoning pinned + case 4: return 6; + case 5: return 3; + case 6: return 2; + default: return 2; // depth >= 7 + } + } + + // accumulated (n_draft - n_accepted) needed to drop one step from depth N; + // scaled by depth, with a floor so shallow depths do not collapse too fast + static int drop_pressure(int depth) { + return std::max(depth * 5, 20); + } + + // reset to the floor max(1, n_min_adaptive), bounded by the ceiling n_max; + // the controller climbs from there once acceptance feedback arrives + void reset(int n_max, int n_min_adaptive) { + const int cap = std::max(1, n_max); + const int floor = std::max(1, n_min_adaptive); + + n_cur = std::min(floor, cap); + n_climb = 0; + n_drop = 0; + } + + // feed one verification result: n_draft is the number of tokens this + // implementation drafted, n_accepted the number the target accepted + void update(int n_draft, int n_accepted, int n_max, int n_min_adaptive) { + if (n_draft <= 0) { + return; + } + + const int cap = std::max(1, n_max); + const int floor = std::max(1, n_min_adaptive); + + if (n_accepted == n_draft) { + n_drop = 0; + + // full acceptance: reset the drop pressure, accumulate the climb streak + if (n_cur < cap && ++n_climb >= climb_threshold(n_cur)) { + n_cur++; + n_climb = 0; + } + } else { + n_climb = 0; + + // any miss adds (n_draft - n_accepted) to the drop pressure; drop one + // step when the accumulated pressure reaches the depth-scaled budget + if (n_cur > floor) { + n_drop += n_draft - n_accepted; + if (n_drop >= drop_pressure(n_cur)) { + n_cur--; + n_drop = 0; + } + } + } + } +}; diff --git a/common/speculative.cpp b/common/speculative.cpp index 851a47b9a58..864b91b96c9 100644 --- a/common/speculative.cpp +++ b/common/speculative.cpp @@ -9,6 +9,7 @@ #include "ngram-map.h" #include "ngram-mod.h" #include "sampling.h" +#include "speculative-adaptive.h" #include "../src/llama-ext.h" // staging API: llama_set_embeddings_nextn / llama_get_embeddings_nextn_ith (used by MTP) @@ -35,6 +36,7 @@ const std::map common_speculative_type_fro {"draft-simple", COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE}, {"draft-eagle3", COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3}, {"draft-mtp", COMMON_SPECULATIVE_TYPE_DRAFT_MTP}, + {"draft-mtp-adaptive", COMMON_SPECULATIVE_TYPE_DRAFT_MTP_ADAPTIVE}, {"draft-dflash", COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH}, {"draft-dspark", COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK}, {"ngram-simple", COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE}, @@ -1357,8 +1359,14 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { std::vector i_last; std::vector> chain_h; - common_speculative_impl_draft_mtp(const common_params_speculative & params, uint32_t n_seq) - : common_speculative_impl(COMMON_SPECULATIVE_TYPE_DRAFT_MTP, n_seq, params.draft.n_max) + // Adaptive draft depth (draft-mtp-adaptive), see common_speculative_adaptive + bool adaptive = false; + std::vector n_cap; // [n_seq] effective draft cap for the current draft() call + std::vector n_last; // [n_seq] drafts attempted in the most recent draft() call + std::vector adaptive_ctrl; // [n_seq] per-seq adaptive depth controller + + common_speculative_impl_draft_mtp(const common_params_speculative & params, uint32_t n_seq, bool adaptive = false) + : common_speculative_impl(adaptive ? COMMON_SPECULATIVE_TYPE_DRAFT_MTP_ADAPTIVE : COMMON_SPECULATIVE_TYPE_DRAFT_MTP, n_seq, params.draft.n_max) , params(params.draft) { auto * ctx_tgt = this->params.ctx_tgt; @@ -1370,6 +1378,11 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { "MTP input row width must match the target h_nextn width"); n_mtp_layers = std::max(1, (int) llama_model_n_layer_nextn(llama_get_model(ctx_dft))); + this->adaptive = adaptive; + // n_cap/n_last are written by the shared draft loop in both modes + n_cap.assign(n_seq, 0); + n_last.assign(n_seq, 0); + SPC_TRC("%s", "adding speculative implementation 'draft-mtp'\n"); SPC_TRC("- n_max=%d, n_min=%d, p_min=%.2f, n_embd=%d, backend_sampling=%d\n", this->params.n_max, this->params.n_min, this->params.p_min, n_embd, (int) this->params.backend_sampling); SPC_TRC("- gpu_layers=%d, cache_k=%s, cache_v=%s, ctx_tgt=%s, ctx_dft=%s, devices=[%s]\n", @@ -1417,6 +1430,9 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { is_mem_shared = llama_get_ctx_other(ctx_dft) == ctx_tgt; chain_heads = n_mtp_layers > 1 && !is_mem_shared; + // remember the user n_max: chain_heads caps it at the model MTP layer + // count, and the adaptive range abort below must explain the cap + const int32_t n_max_user = this->params.n_max; if (chain_heads) { this->params.n_max = std::min(this->params.n_max, n_mtp_layers); @@ -1427,6 +1443,28 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { } this->n_max = this->params.n_max; + if (adaptive) { + // a floor above the ceiling would pin the depth below the floor, so the + // configuration is invalid + if (this->params.n_min_adaptive < 1 || this->params.n_min_adaptive > this->params.n_max) { + if (n_max_user > this->params.n_max) { + // n_max was capped by the MTP layer count, not by the user + GGML_ABORT("%s: invalid adaptive draft range: n_min_adaptive=%d, n_max=%d (n_max is capped by the model MTP layer count %d; set --spec-draft-n-min-adaptive to at most %d)", + __func__, this->params.n_min_adaptive, this->params.n_max, n_mtp_layers, n_mtp_layers); + } + GGML_ABORT("%s: invalid adaptive draft range: n_min_adaptive=%d, n_max=%d (n_min_adaptive must be in [1, n_max])", + __func__, this->params.n_min_adaptive, this->params.n_max); + } + + adaptive_ctrl.assign(n_seq, common_speculative_adaptive()); + for (uint32_t s = 0; s < n_seq; ++s) { + // start at the floor max(1, n_min_adaptive), bounded by n_max; + // the controller climbs from there once acceptance feedback arrives + adaptive_ctrl[s].reset(this->params.n_max, this->params.n_min_adaptive); + } + SPC_TRC("%s", "adaptive draft depth enabled (draft-mtp-adaptive)\n"); + } + pending_h.assign(n_seq, std::vector(n_embd, 0.0f)); i_last.assign(n_seq, -1); @@ -1463,6 +1501,12 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { return; } + // new generation: the depth learned for the previous content is stale, + // so the controller starts from the floor again + if (adaptive) { + adaptive_ctrl[seq_id].reset(this->params.n_max, this->params.n_min_adaptive); + } + auto * ctx_dft = this->params.ctx_dft; const llama_pos pos_max = llama_memory_seq_pos_max(llama_get_memory(ctx_dft), seq_id); @@ -1615,6 +1659,13 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { drafting[seq_id] = true; common_sampler_reset(smpls[seq_id].get()); + // effective draft cap for this step: adaptive depth (or the user n_max), + // then clamped by the per-call context bound from the server + n_cap[seq_id] = adaptive ? adaptive_ctrl[seq_id].n_cur : params.n_max; + if (dp.n_max > 0 && dp.n_max < n_cap[seq_id]) { + n_cap[seq_id] = dp.n_max; + } + common_batch_add(batch, dp.id_last, dp.n_past, { seq_id }, true); std::memcpy(batch.embd + (size_t) (batch.n_tokens - 1) * n_embd, pending_h[seq_id].data(), row_bytes); @@ -1691,7 +1742,7 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { result.push_back(id); - if (params.n_max <= (int) result.size()) { + if (n_cap[seq_id] <= (int) result.size()) { drafting[seq_id] = false; n_drafting--; continue; @@ -1738,17 +1789,32 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { continue; } - if (dp.result->size() < (size_t) params.n_min) { + n_last[seq_id] = (int) dp.result->size(); + + // the adaptive controller decides its own depth, so the generic n_min + // draft cutoff does not apply to it + if (!adaptive && dp.result->size() < (size_t) params.n_min) { dp.result->clear(); } } } - void accept(llama_seq_id seq_id, uint16_t n_accepted, bool /*is_other*/) override { + void accept(llama_seq_id seq_id, uint16_t n_accepted, bool is_other) override { if (seq_id < 0 || seq_id >= (llama_seq_id) n_seq) { return; } + // update the adaptive controller only when this implementation produced the + // accepted draft; on is_other the stats belong to a different speculator + if (adaptive && !is_other) { + const int depth_before = adaptive_ctrl[seq_id].n_cur; + adaptive_ctrl[seq_id].update(n_last[seq_id], n_accepted, params.n_max, params.n_min_adaptive); + if (adaptive_ctrl[seq_id].n_cur != depth_before) { + SPC_DBG("adaptive draft depth seq %d: %d -> %d (n_draft=%d, n_accepted=%d)\n", + (int) seq_id, depth_before, adaptive_ctrl[seq_id].n_cur, n_last[seq_id], n_accepted); + } + } + const int32_t n_rows = verify_h_rows[seq_id]; if (n_rows <= 0) { return; @@ -2243,6 +2309,7 @@ std::string common_speculative_type_to_str(common_speculative_type type) { case COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE: return "draft-simple"; case COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3: return "draft-eagle3"; case COMMON_SPECULATIVE_TYPE_DRAFT_MTP: return "draft-mtp"; + case COMMON_SPECULATIVE_TYPE_DRAFT_MTP_ADAPTIVE: return "draft-mtp-adaptive"; case COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH: return "draft-dflash"; case COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK: return "draft-dspark"; case COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE: return "ngram-simple"; @@ -2334,6 +2401,7 @@ int32_t common_speculative_n_max(const common_params_speculative * spec) { case COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE: case COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3: case COMMON_SPECULATIVE_TYPE_DRAFT_MTP: + case COMMON_SPECULATIVE_TYPE_DRAFT_MTP_ADAPTIVE: case COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH: case COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK: n_max = std::max(n_max, std::max(0, spec->draft.n_max)); @@ -2520,7 +2588,10 @@ common_speculative_init_result::common_speculative_init_result( const bool has_draft = params.speculative.has_dft(); const bool spec_mtp = std::find(params.speculative.types.begin(), params.speculative.types.end(), - COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params.speculative.types.end(); + COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params.speculative.types.end() || + std::find(params.speculative.types.begin(), + params.speculative.types.end(), + COMMON_SPECULATIVE_TYPE_DRAFT_MTP_ADAPTIVE) != params.speculative.types.end(); auto mparams = common_model_params_to_llama(params); auto cparams = common_context_params_to_llama(params); @@ -2612,7 +2683,7 @@ common_speculative * common_speculative_init(common_params_speculative & params, }; // when adding a new type - update here the logic above - static_assert(COMMON_SPECULATIVE_TYPE_COUNT == 11); + static_assert(COMMON_SPECULATIVE_TYPE_COUNT == 12); // this list here defines the priority of the speculators // the one with highest priority are listed first @@ -2625,6 +2696,7 @@ common_speculative * common_speculative_init(common_params_speculative & params, add_config_if_enabled(COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE); add_config_if_enabled(COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3, params.draft.ctx_dft != nullptr); add_config_if_enabled(COMMON_SPECULATIVE_TYPE_DRAFT_MTP, params.draft.ctx_dft != nullptr); + add_config_if_enabled(COMMON_SPECULATIVE_TYPE_DRAFT_MTP_ADAPTIVE, params.draft.ctx_dft != nullptr); add_config_if_enabled(COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH, params.draft.ctx_dft != nullptr); add_config_if_enabled(COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK, params.draft.ctx_dft != nullptr); } @@ -2644,7 +2716,11 @@ common_speculative * common_speculative_init(common_params_speculative & params, break; } case COMMON_SPECULATIVE_TYPE_DRAFT_MTP: { - impls.push_back(std::make_unique(config.params, n_seq)); + impls.push_back(std::make_unique(config.params, n_seq, /*adaptive=*/ false)); + break; + } + case COMMON_SPECULATIVE_TYPE_DRAFT_MTP_ADAPTIVE: { + impls.push_back(std::make_unique(config.params, n_seq, /*adaptive=*/ true)); break; } case COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH: { diff --git a/src/models/delta-net-base.cpp b/src/models/delta-net-base.cpp index ad661264773..30b5f50f1f6 100644 --- a/src/models/delta-net-base.cpp +++ b/src/models/delta-net-base.cpp @@ -501,7 +501,12 @@ ggml_tensor * llm_build_delta_net_base::build_conv_state( const int64_t K = (int64_t) cparams.n_rs_seq + 1; - for (int64_t t = 1; t <= K; ++t) { + // only the snapshot slots reachable by a rollback inside this batch are + // useful: rollback <= n_seq_tokens - 1, so slots beyond the batch repeat + // the pre-batch state and would only waste kernel launches per round + const int64_t t_min = std::max(1, K - ubatch.n_seq_tokens + 1); + + for (int64_t t = t_min; t <= K; ++t) { const int64_t s_idx = std::max(0, conv_input->ne[0] - conv_states->ne[0] - K + t); const int64_t s_slot = K - t; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index c46377c7623..43bdc89dfb5 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -289,6 +289,7 @@ llama_build_and_test(test-thread-safety.cpp ARGS -m "${MODEL_DEST}" -ngl 99 -p " set_tests_properties(test-thread-safety PROPERTIES FIXTURES_REQUIRED test-download-model) llama_build_and_test(test-arg-parser.cpp) +llama_build_and_test(test-speculative-adaptive.cpp) llama_build_and_test(test-model-resolution.cpp) # the test serves its repos from an httplib server, and the library links it privately target_link_libraries(test-model-resolution PRIVATE cpp-httplib) diff --git a/tests/test-arg-parser.cpp b/tests/test-arg-parser.cpp index e0907631abd..84f1ed98684 100644 --- a/tests/test-arg-parser.cpp +++ b/tests/test-arg-parser.cpp @@ -274,6 +274,21 @@ static void test(void) { assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), synth_params, LLAMA_EXAMPLE_SERVER)); } + // the adaptive floor defaults to 2 and parses explicitly + argv = {"binary_name", "-m", "model_file.gguf"}; + assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_SPECULATIVE)); + assert(params.speculative.draft.n_min_adaptive == 3); + argv = {"binary_name", "-m", "model_file.gguf", "--spec-draft-n-min-adaptive", "5"}; + assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_SPECULATIVE)); + assert(params.speculative.draft.n_min_adaptive == 5); + + // the adaptive MTP type parses to the dedicated enum value + argv = {"binary_name", "-m", "model_file.gguf", "--spec-type", "draft-mtp-adaptive"}; + common_params spec_params; + assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), spec_params, LLAMA_EXAMPLE_SPECULATIVE)); + assert(std::find(spec_params.speculative.types.begin(), spec_params.speculative.types.end(), + COMMON_SPECULATIVE_TYPE_DRAFT_MTP_ADAPTIVE) != spec_params.speculative.types.end()); + argv = {"binary_name", "-lm", "none"}; assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON)); assert(params.load_mode == LLAMA_LOAD_MODE_NONE); diff --git a/tests/test-speculative-adaptive.cpp b/tests/test-speculative-adaptive.cpp new file mode 100644 index 00000000000..ea959f22329 --- /dev/null +++ b/tests/test-speculative-adaptive.cpp @@ -0,0 +1,218 @@ +#include "speculative-adaptive.h" + +#undef NDEBUG + +#include +#include + +static void test_reset(void) { + common_speculative_adaptive ctrl; + + // cold start at the floor max(1, n_min_adaptive) + ctrl.reset(8, 1); + assert(ctrl.n_cur == 1); + assert(ctrl.n_climb == 0); + assert(ctrl.n_drop == 0); + + // the default adaptive floor of 3 starts the controller at depth 3 + ctrl.reset(8, 3); + assert(ctrl.n_cur == 3); + + // the ceiling clamps the cold start to n_max + ctrl.reset(1, 3); + assert(ctrl.n_cur == 1); +} + +static void test_climb(void) { + common_speculative_adaptive ctrl; + ctrl.reset(8, 1); // ceiling 8, cold start at the floor 1 + + // depth 1 climbs after 2 consecutive full accepts + ctrl.update(1, 1, 8, 1); + assert(ctrl.n_cur == 1); + ctrl.update(1, 1, 8, 1); + assert(ctrl.n_cur == 2); + + // a miss resets the climb streak + ctrl.update(2, 1, 8, 1); // near miss + assert(ctrl.n_cur == 2); + assert(ctrl.n_climb == 0); + + // depth 2 climbs after 4 consecutive full accepts + for (int i = 0; i < 3; ++i) { + ctrl.update(2, 2, 8, 1); + assert(ctrl.n_cur == 2); + } + ctrl.update(2, 2, 8, 1); + assert(ctrl.n_cur == 3); + + // depth 3 is the hardened barrier: 10 consecutive full accepts to reach + // depth 4, so prose/reasoning stay pinned at the floor + for (int i = 0; i < 9; ++i) { + ctrl.update(3, 3, 8, 1); + assert(ctrl.n_cur == 3); + } + ctrl.update(3, 3, 8, 1); + assert(ctrl.n_cur == 4); + + // a full accept of a draft truncated below the depth (e.g. clamped by the + // server context bound) counts as a full accept, not as a miss + ctrl.update(3, 3, 8, 1); // depth 4, only 3 tokens drafted, all accepted + assert(ctrl.n_climb == 1); + assert(ctrl.n_drop == 0); + + // depth 4 needs 6 consecutive full accepts + for (int i = 0; i < 5; ++i) { + ctrl.update(4, 4, 8, 1); + } + assert(ctrl.n_cur == 5); + + // depth 5 needs 3 consecutive full accepts + for (int i = 0; i < 2; ++i) { + ctrl.update(5, 5, 8, 1); + assert(ctrl.n_cur == 5); + } + ctrl.update(5, 5, 8, 1); + assert(ctrl.n_cur == 6); + + // depth 6 needs 2 consecutive full accepts + ctrl.update(6, 6, 8, 1); + assert(ctrl.n_cur == 6); + ctrl.update(6, 6, 8, 1); + assert(ctrl.n_cur == 7); + + // depth 7+ needs 2 consecutive full accepts + ctrl.update(7, 7, 8, 1); + assert(ctrl.n_cur == 7); + ctrl.update(7, 7, 8, 1); + assert(ctrl.n_cur == 8); + + // the ceiling blocks further climbs + for (int i = 0; i < 8; ++i) { + ctrl.update(8, 8, 8, 1); + } + assert(ctrl.n_cur == 8); + + // no feedback for a zero-length draft + ctrl.update(0, 0, 8, 1); + assert(ctrl.n_cur == 8); + assert(ctrl.n_climb == 0); +} + +static void test_drop(void) { + common_speculative_adaptive ctrl; + ctrl.reset(8, 1); // cold start at the floor + + // at the floor no pressure accumulates at all + for (int i = 0; i < 100; ++i) { + ctrl.update(1, 0, 8, 1); + } + assert(ctrl.n_cur == 1); + assert(ctrl.n_drop == 0); + + // climb to depth 3 (2 + 4 full accepts) + for (int i = 0; i < 2; ++i) { + ctrl.update(1, 1, 8, 1); + } + for (int i = 0; i < 4; ++i) { + ctrl.update(2, 2, 8, 1); + } + assert(ctrl.n_cur == 3); + + // at depth 3 the drop budget is floored at 20: a total miss adds 3, so + // 7 misses drop one step + for (int i = 0; i < 6; ++i) { + ctrl.update(3, 0, 8, 1); + assert(ctrl.n_cur == 3); + } + assert(ctrl.n_drop == 18); + ctrl.update(3, 0, 8, 1); + assert(ctrl.n_cur == 2); + assert(ctrl.n_drop == 0); + + // at depth 2 the budget is floored at 20: a near miss adds 1, so 20 near + // misses drop one step (the depth-1 collapse needs real sustained failure) + for (int i = 0; i < 19; ++i) { + ctrl.update(2, 1, 8, 1); + assert(ctrl.n_cur == 2); + } + assert(ctrl.n_drop == 19); + ctrl.update(2, 1, 8, 1); + assert(ctrl.n_cur == 1); + + // back at the floor, misses no longer accumulate pressure + for (int i = 0; i < 100; ++i) { + ctrl.update(1, 0, 8, 1); + } + assert(ctrl.n_cur == 1); + assert(ctrl.n_drop == 0); + + // deep depths hold a little longer: at depth 5 the budget is 25, a total + // miss adds 5, so 5 misses drop one step + ctrl.reset(8, 1); + ctrl.n_cur = 5; // simulate a controller that already climbed to 5 + for (int i = 0; i < 4; ++i) { + ctrl.update(5, 0, 8, 1); + assert(ctrl.n_cur == 5); + } + assert(ctrl.n_drop == 20); + ctrl.update(5, 0, 8, 1); // 20 + 5 = 25 -> drop + assert(ctrl.n_cur == 4); + assert(ctrl.n_drop == 0); +} + +static void test_full_accept_resets_pressure(void) { + common_speculative_adaptive ctrl; + ctrl.reset(8, 1); + ctrl.n_cur = 3; + + // accumulate pressure, then a full accept wipes it out + for (int i = 0; i < 5; ++i) { + ctrl.update(3, 1, 8, 1); // near miss: +2 pressure at depth 3 + } + assert(ctrl.n_drop == 10); + ctrl.update(3, 3, 8, 1); + assert(ctrl.n_drop == 0); + + // the miss pressure uses the drafted count, not the depth: a truncated + // draft (2 tokens at depth 3) with 1 accepted adds 1, not 2 + ctrl.update(2, 1, 8, 1); + assert(ctrl.n_drop == 1); +} + +static void test_floor(void) { + common_speculative_adaptive ctrl; + + // with the floor at 2 the depth never drops below 2, no matter how bad + // the content gets + ctrl.reset(8, 2); + for (int i = 0; i < 1000; ++i) { + ctrl.update(2, 0, 8, 2); + } + assert(ctrl.n_cur == 2); + assert(ctrl.n_drop == 0); + + // climbs still work from the floor + for (int i = 0; i < 4; ++i) { + ctrl.update(2, 2, 8, 2); + } + assert(ctrl.n_cur == 3); + + // and drops stop at the floor, not below it + for (int i = 0; i < 100; ++i) { + ctrl.update(3, 0, 8, 2); + } + assert(ctrl.n_cur == 2); +} + +int main(void) { + test_reset(); + test_climb(); + test_drop(); + test_full_accept_resets_pressure(); + test_floor(); + + printf("test-speculative-adaptive: all tests OK\n\n"); + + return 0; +} diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index f5477356d61..64477c231aa 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -1019,7 +1019,10 @@ struct server_context_impl { const bool has_draft = params.speculative.has_dft(); const bool spec_mtp = std::find(params_base.speculative.types.begin(), params_base.speculative.types.end(), - COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params_base.speculative.types.end(); + COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params_base.speculative.types.end() || + std::find(params_base.speculative.types.begin(), + params_base.speculative.types.end(), + COMMON_SPECULATIVE_TYPE_DRAFT_MTP_ADAPTIVE) != params_base.speculative.types.end(); const bool has_spec = has_draft || spec_mtp; if (callback_state) {