From 48f7f63fafd8a4aedbdaf691fe88e4607546bf69 Mon Sep 17 00:00:00 2001 From: rinaldofesta Date: Thu, 3 Sep 2026 20:06:09 +0200 Subject: [PATCH] metal: bypass DSpark speculation once a request proves unprofitable On Metal every verified DSpark row re-streams its routed experts, so speculation on text the proposer cannot predict runs slower than plain decode. The README measures 36-44 t/s against 46.5 t/s plain on prose. The scheduler already pauses and re-probes, but it never gives up, so the loss repeats for the whole request. Track accepted drafts and scheduler cycles for the lifetime of the request. Once two probe windows have run and the lifetime average falls below 0.5 accepted drafts per cycle, disable speculation for the rest of the request, the same bypass the gfx1151 fast path already takes. The rule uses counters, not timings, so a greedy request follows the same schedule on every run. Measured on a MacBook Pro M5 Max 128 GB, Metal, DeepSeek V4 Flash 0731 IQ2_XXS/Q2_K, greedy, ratios read at the first bypass decision: 8192 tokens of Italian prose 0.25 (bypass fires, decode 35.9 -> 39.2 t/s against 40.7 plain), reasoning with thinking 1.25 (no bypass, 47.9 t/s kept), short English prose 1.75 at the first decision (no bypass, stays a mild loss at 41.6 vs 45.0 plain, out of scope here), Python code never reaches the decision (54.7 t/s kept). Output stays byte-identical across scheduler variants on the reasoning prompt; the existing batched floating-point divergence of --dspark is unchanged. Co-Authored-By: Claude Fable 5.1 --- ds4.c | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/ds4.c b/ds4.c index 91ab214ab..534f186c7 100644 --- a/ds4.c +++ b/ds4.c @@ -54201,6 +54201,7 @@ struct ds4_session { uint32_t dspark_sched_accepted; uint32_t dspark_sched_no_draft; uint32_t dspark_sched_skip; + uint32_t dspark_sched_lifetime_cycles; uint32_t dspark_sched_lifetime_accepted; double dspark_sched_life_extra_ms; double dspark_sched_life_saved_ms; @@ -54353,6 +54354,7 @@ static void ds4_session_dspark_scheduler_begin_request(ds4_session *s) { if (!s) return; ds4_session_dspark_scheduler_reset(s); s->dspark_sched_skip = 0; + s->dspark_sched_lifetime_cycles = 0; s->dspark_sched_lifetime_accepted = 0; s->dspark_sched_life_extra_ms = 0.0; s->dspark_sched_life_saved_ms = 0.0; @@ -54393,6 +54395,7 @@ static void ds4_session_dspark_scheduler_note( } s->dspark_sched_cycles++; + s->dspark_sched_lifetime_cycles++; s->dspark_sched_accepted += accepted_drafts; if (accepted_drafts != 0) { if (s->dspark_sched_lifetime_accepted <= @@ -54504,16 +54507,29 @@ static void ds4_session_dspark_scheduler_note( s->dspark_sched_accepted != 0 && extra_per_accept_ms * 1000.0 > (double)max_ms_per_accept_milli; if (low_accept || many_no_draft || slow_accept || measured_unprofitable) { - if (ds4_session_dspark_rocm_gfx1151_fast_path(s)) { + /* Two windows avoid judging the warmup. Below 0.5 accepted drafts + * per scheduler cycle each verified row costs more than it saves on + * Metal: 8k tokens of Italian prose measure 0.25 at this decision, + * reasoning 1.25, and code never reaches this branch. Counters, + * not timings, keep greedy scheduling reproducible across runs. */ + const bool low_lifetime_metal_bypass = + s->engine && s->engine->backend == DS4_BACKEND_METAL && + s->dspark_sched_lifetime_cycles >= 2u * window && + 2u * s->dspark_sched_lifetime_accepted < + s->dspark_sched_lifetime_cycles; + if (ds4_session_dspark_rocm_gfx1151_fast_path(s) || + low_lifetime_metal_bypass) { s->dspark_sched_bypass = true; s->dspark_sched_skip = 0; if (getenv("DS4_DSPARK_SPEC_LOG") != NULL) { fprintf(stderr, "ds4: DSpark scheduler bypass accepted=%u avg=%.3f " - "no_draft=%u\n", + "no_draft=%u lifetime=%u/%u\n", s->dspark_sched_accepted, (double)avg_milli / 1000.0, - s->dspark_sched_no_draft); + s->dspark_sched_no_draft, + s->dspark_sched_lifetime_accepted, + s->dspark_sched_lifetime_cycles); } ds4_session_dspark_scheduler_reset(s); return;