diff --git a/README.md b/README.md index 842800967..64f999206 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,7 @@ Model links open the exact weights used by the measured setup. Drafter links ope | [Gemma 4 26B-A4B Q4_K_M](https://huggingface.co/bartowski/google_gemma-4-26B-A4B-it-GGUF/blob/main/google_gemma-4-26B-A4B-it-Q4_K_M.gguf) + [DFlash Q8_0 drafter](https://huggingface.co/Lucebox/gemma-4-26B-A4B-it-DFlash-GGUF/blob/main/gemma-4-26B-A4B-it-DFlash-q8_0.gguf) | Decode | **1.31×** | | [Gemma 4 31B IT Q4_K_M](https://huggingface.co/bartowski/google_gemma-4-31B-it-GGUF/blob/main/google_gemma-4-31B-it-Q4_K_M.gguf) + [DFlash Q8_0 drafter](https://huggingface.co/Lucebox/gemma-4-31B-it-DFlash-GGUF/blob/main/gemma-4-31B-it-DFlash-q8_0.gguf) | Decode | **3.2×** | | [DeepSeek V4 Flash ROCmFPX MIX Strix](https://huggingface.co/Lucebox/DeepSeek-V4-Flash-0731-ROCmFP3/blob/main/DeepSeek-V4-Flash-0731-ROCMFPX-MIX-STRIX.gguf) + [DSpark Q4RMFP4 drafter](https://huggingface.co/Lucebox/DeepSeek-V4-Flash-0731-DSpark-GGUF/blob/main/DeepSeek-V4-Flash-0731-DSpark-draft-Q4RMFP4-denseF16.gguf) | Decode | Up to **1.81×** vs target-only, **32.7 vs 18.1 tok/s** | +| [Ling 3.0 Flash 124B-A5.1B Q4_K_M](https://huggingface.co/bloomer010/Ling-3.0-flash-GGUF) | Decode | **34.6 tok/s** median AR on DGX Spark | ## Tested Machines (GPU/APU) diff --git a/server/CMakeLists.txt b/server/CMakeLists.txt index a20f168ab..cab194c56 100644 --- a/server/CMakeLists.txt +++ b/server/CMakeLists.txt @@ -400,6 +400,7 @@ set(DFLASH27B_SRC_INCLUDE_DIRS ${CMAKE_CURRENT_SOURCE_DIR}/src/draft ${CMAKE_CURRENT_SOURCE_DIR}/src/qwen35 ${CMAKE_CURRENT_SOURCE_DIR}/src/qwen35moe + ${CMAKE_CURRENT_SOURCE_DIR}/src/bailingmoe3 ${CMAKE_CURRENT_SOURCE_DIR}/src/laguna ${CMAKE_CURRENT_SOURCE_DIR}/src/qwen3 ${CMAKE_CURRENT_SOURCE_DIR}/src/gemma4 @@ -412,6 +413,9 @@ add_library(dflash_common STATIC src/qwen35/gguf_target_loader.cpp src/qwen35/qwen35_target_graph.cpp src/qwen35/qwen35_roctx.cpp + src/bailingmoe3/bailingmoe3_loader.cpp + src/bailingmoe3/bailingmoe3_graph.cpp + src/bailingmoe3/bailingmoe3_backend.cpp src/draft/draft_gguf_loader.cpp src/draft/draft_safetensors_loader.cpp src/draft/draft_graph.cpp diff --git a/server/src/bailingmoe3/bailingmoe3_backend.cpp b/server/src/bailingmoe3/bailingmoe3_backend.cpp new file mode 100644 index 000000000..feb7f0367 --- /dev/null +++ b/server/src/bailingmoe3/bailingmoe3_backend.cpp @@ -0,0 +1,47 @@ +#include "bailingmoe3_backend.h" + +#include + +namespace dflash::common { +namespace { + +Qwen35Config make_qwen_runtime_config(const BailingMoe3Config & cfg) { + Qwen35Config runtime; + runtime.target_path = cfg.model_path; + runtime.device = cfg.device; + runtime.stream_fd = cfg.stream_fd; + // The Ling baseline uses the ordinary contiguous F16/Q4 KV cache and the + // proven single-sequence AR loop. No DFlash draft or paged serving yet. + // Its compressed MLA head is 576-wide, whose CUDA kernel contract uses a + // 256-row K/V span and an explicit visibility mask even for decode. + runtime.kq_stride_pad = 256; + runtime.paged_attention = false; + runtime.max_concurrency = 1; + return runtime; +} + +} // namespace + +BailingMoe3Backend::BailingMoe3Backend(const BailingMoe3Config & cfg) + : Qwen35Backend(make_qwen_runtime_config(cfg)) {} + +bool BailingMoe3Backend::load_target_model(ggml_backend_t backend, + TargetWeights & out) { + return load_bailingmoe3_gguf(cfg_.target_path, backend, out); +} + +void BailingMoe3Backend::print_ready_banner() const { + const TargetWeights & weights = target_weights(); + std::printf( + "[bailingmoe3-daemon] ready layers=%d kda=%d mla=%d " + "experts=%d/%d groups=%d/%d ctx=%d\n", + weights.n_layer, + weights.n_layer - weights.n_layer / weights.full_attention_interval, + weights.n_layer / weights.full_attention_interval, + weights.n_expert_used, weights.n_expert, + weights.n_expert_groups_used, weights.n_expert_groups, + cfg_.device.max_ctx); + std::fflush(stdout); +} + +} // namespace dflash::common diff --git a/server/src/bailingmoe3/bailingmoe3_backend.h b/server/src/bailingmoe3/bailingmoe3_backend.h new file mode 100644 index 000000000..61e775753 --- /dev/null +++ b/server/src/bailingmoe3/bailingmoe3_backend.h @@ -0,0 +1,28 @@ +#pragma once + +#include "qwen35_backend.h" + +namespace dflash::common { + +// Configuration intentionally exposes only the features the first native +// Ling backend implements. Speculative decode and expert offload can be added +// after the autoregressive path has a logits-equivalent baseline. +struct BailingMoe3Config { + const char * model_path = nullptr; + DevicePlacement device; + int stream_fd = -1; +}; + +class BailingMoe3Backend final : public Qwen35Backend { +public: + explicit BailingMoe3Backend(const BailingMoe3Config & cfg); + + void print_ready_banner() const override; + bool supports_dflash_spec_decode() const override { return false; } + bool supports_remote_draft() const override { return false; } + +protected: + bool load_target_model(ggml_backend_t backend, TargetWeights & out) override; +}; + +} // namespace dflash::common diff --git a/server/src/bailingmoe3/bailingmoe3_graph.cpp b/server/src/bailingmoe3/bailingmoe3_graph.cpp new file mode 100644 index 000000000..2c3dd2f6f --- /dev/null +++ b/server/src/bailingmoe3/bailingmoe3_graph.cpp @@ -0,0 +1,249 @@ +#include "bailingmoe3_graph.h" + +#include "internal.h" +#include "qwen35_ops.h" + +#include +#include +#include + +namespace dflash::common { +namespace { + +ggml_tensor * build_causal_conv1d( + ggml_context * ctx, + ggml_cgraph * gf, + ggml_tensor * all_conv_state, + int qkv_index, + ggml_tensor * cur, + ggml_tensor * projection, + ggml_tensor * conv_weight, + int d_conv, + int d_inner, + int head_dim, + int n_head, + int n_tokens) { + const size_t state_element = ggml_element_size(all_conv_state); + const size_t channel_stride = all_conv_state->nb[1]; + ggml_tensor * conv_state = ggml_view_3d( + ctx, all_conv_state, d_conv - 1, d_inner, 1, + channel_stride, all_conv_state->nb[2], + static_cast(qkv_index) * d_inner * channel_stride); + + ggml_tensor * projected = ggml_mul_mat(ctx, projection, cur); + projected = ggml_reshape_3d(ctx, projected, d_inner, n_tokens, 1); + ggml_tensor * conv_input = + ggml_concat(ctx, conv_state, ggml_transpose(ctx, projected), 0); + + ggml_tensor * last = ggml_view_3d( + ctx, conv_input, d_conv - 1, d_inner, 1, + conv_input->nb[1], conv_input->nb[2], + static_cast(n_tokens) * state_element); + ggml_build_forward_expand(gf, ggml_cpy(ctx, last, conv_state)); + + ggml_tensor * conv_2d = ggml_reshape_2d(ctx, conv_weight, d_conv, d_inner); + ggml_tensor * result = ggml_ssm_conv(ctx, conv_input, conv_2d); + result = ggml_silu(ctx, ggml_reshape_2d(ctx, result, d_inner, n_tokens)); + return ggml_reshape_4d(ctx, result, head_dim, n_head, n_tokens, 1); +} + +} // namespace + +// Ling 3 stores MLA K/V in compressed latent space. K contains the normalized +// 512-wide latent plus the 64 RoPE dimensions; V is the latent alone. Query +// absorption through attn_k_b turns the 128 non-RoPE Q dimensions into the +// same latent space, and attn_v_b expands the attention result back to 128 +// value dimensions per head. +ggml_tensor * build_bailingmoe3_mla_block( + ggml_context * ctx, + ggml_cgraph * gf, + const TargetWeights & w, + const TargetLayer & L, + ggml_tensor * cur, + ggml_tensor * positions, + ggml_tensor * cache_k, + ggml_tensor * cache_v, + ggml_tensor * attn_mask, + int kv_start, + int n_tokens) { + const int n_head = w.n_head; + const int qk_dim = w.mla_qk_head_dim; + const int rope_dim = w.rope_dimension_count; + const int nope_dim = qk_dim - rope_dim; + const int kv_rank = w.kv_lora_rank; + const int v_dim = w.mla_v_head_dim; + GGML_ASSERT(attn_mask != nullptr); + + ggml_tensor * q_all = nullptr; + if (L.attn_q_a) { + q_all = ggml_mul_mat(ctx, L.attn_q_a, cur); + q_all = rms_norm_mul(ctx, q_all, L.attn_q_a_norm, w.rms_eps); + q_all = ggml_mul_mat(ctx, L.attn_q_b, q_all); + } else { + q_all = ggml_mul_mat(ctx, L.wq, cur); + } + q_all = ggml_reshape_3d(ctx, q_all, qk_dim, n_head, n_tokens); + ggml_tensor * q_nope = ggml_view_3d( + ctx, q_all, nope_dim, n_head, n_tokens, + q_all->nb[1], q_all->nb[2], 0); + ggml_tensor * q_pe = ggml_view_3d( + ctx, q_all, rope_dim, n_head, n_tokens, + q_all->nb[1], q_all->nb[2], + static_cast(nope_dim) * ggml_element_size(q_all)); + + ggml_tensor * kv_all = ggml_mul_mat(ctx, L.attn_kv_a_mqa, cur); + ggml_tensor * kv = ggml_view_2d( + ctx, kv_all, kv_rank, n_tokens, kv_all->nb[1], 0); + ggml_tensor * k_pe = ggml_view_3d( + ctx, kv_all, rope_dim, 1, n_tokens, + kv_all->nb[1], kv_all->nb[1], + static_cast(kv_rank) * ggml_element_size(kv_all)); + + // The shared hybrid graph input reserves four position lanes for M-RoPE. + // Ling uses ordinary RoPE, so expose just the first lane to ggml_rope. + ggml_tensor * rope_positions = positions; + if (positions->ne[0] != n_tokens) { + GGML_ASSERT(positions->ne[0] >= n_tokens); + rope_positions = ggml_view_1d(ctx, positions, n_tokens, 0); + } + + // Bailing V3 uses interleaved rotary pairs, i.e. GGML's NORMAL layout. + q_pe = ggml_cont(ctx, q_pe); + k_pe = ggml_cont(ctx, k_pe); + q_pe = ggml_rope_ext(ctx, q_pe, rope_positions, nullptr, + rope_dim, GGML_ROPE_TYPE_NORMAL, 0, + w.rope_theta, 1.0f, + 0.0f, 1.0f, 0.0f, 0.0f); + k_pe = ggml_rope_ext(ctx, k_pe, rope_positions, nullptr, + rope_dim, GGML_ROPE_TYPE_NORMAL, 0, + w.rope_theta, 1.0f, + 0.0f, 1.0f, 0.0f, 0.0f); + kv = rms_norm_mul(ctx, kv, L.attn_kv_a_norm, w.rms_eps); + + // Absorb the K projection into Q: [nope,T,H] -> [latent,T,H]. + q_nope = ggml_permute(ctx, q_nope, 0, 2, 1, 3); + q_nope = ggml_mul_mat(ctx, L.attn_k_b, q_nope); + q_nope = ggml_permute(ctx, q_nope, 0, 2, 1, 3); + ggml_tensor * q = ggml_concat(ctx, q_nope, q_pe, 0); + + ggml_tensor * kv_3d = ggml_reshape_3d(ctx, kv, kv_rank, 1, n_tokens); + ggml_tensor * k_cur = ggml_concat(ctx, kv_3d, k_pe, 0); + ggml_tensor * v_cur = kv_3d; + + // Persistent latent cache: [D,T,1]. + ggml_tensor * k_write = ggml_permute(ctx, k_cur, 0, 2, 1, 3); + ggml_tensor * v_write = ggml_permute(ctx, v_cur, 0, 2, 1, 3); + ggml_tensor * k_slot = ggml_view_3d( + ctx, cache_k, kv_rank + rope_dim, n_tokens, 1, + cache_k->nb[1], cache_k->nb[2], + static_cast(kv_start) * cache_k->nb[1]); + ggml_tensor * v_slot = ggml_view_3d( + ctx, cache_v, kv_rank, n_tokens, 1, + cache_v->nb[1], cache_v->nb[2], + static_cast(kv_start) * cache_v->nb[1]); + ggml_build_forward_expand(gf, ggml_cpy(ctx, k_write, k_slot)); + ggml_build_forward_expand(gf, ggml_cpy(ctx, v_write, v_slot)); + + const int kv_len = kv_start + n_tokens; + // The 576x512 CUDA flash-attention specialization requires the compressed + // K/V span to be padded to its 256-token launch stride. The caller always + // supplies a causal mask for Ling, so zero-initialized future cache rows + // remain invisible. + const int kv_len_padded = std::min( + ((kv_len + 255) / 256) * 256, static_cast(cache_k->ne[1])); + ggml_tensor * k_read = ggml_view_3d( + ctx, cache_k, kv_rank + rope_dim, kv_len_padded, 1, + cache_k->nb[1], cache_k->nb[2], 0); + ggml_tensor * v_read = ggml_view_3d( + ctx, cache_v, kv_rank, kv_len_padded, 1, + cache_v->nb[1], cache_v->nb[2], 0); + ggml_tensor * q_fa = ggml_cont(ctx, ggml_permute(ctx, q, 0, 2, 1, 3)); + ggml_tensor * attn = ggml_flash_attn_ext( + ctx, q_fa, k_read, v_read, attn_mask, + 1.0f / std::sqrt(static_cast(qk_dim)), 0.0f, 0.0f); + ggml_flash_attn_ext_set_prec(attn, GGML_PREC_F32); + + // Flash attention returns [latent,H,T]. Apply the per-head V expansion + // as a batched matmul in [latent,T,H] layout. + attn = ggml_permute(ctx, attn, 0, 2, 1, 3); + attn = ggml_mul_mat(ctx, L.attn_v_b, attn); + attn = ggml_cont(ctx, ggml_permute(ctx, attn, 0, 2, 1, 3)); + + ggml_tensor * gate = ggml_mul_mat(ctx, L.wqkv_gate, cur); + gate = ggml_sigmoid(ctx, + ggml_reshape_3d(ctx, gate, 1, n_head, n_tokens)); + attn = ggml_mul(ctx, attn, gate); + attn = ggml_cont_2d(ctx, attn, v_dim * n_head, n_tokens); + return ggml_mul_mat(ctx, L.wo, attn); +} + +// Ling 3 KDA block. The shared ggml CUDA primitive detects KDA from the +// vector gate's first dimension (128 rather than scalar 1). +ggml_tensor * build_bailingmoe3_kda_block( + ggml_context * ctx, + ggml_cgraph * gf, + const TargetWeights & w, + const TargetLayer & L, + ggml_tensor * cur, + ggml_tensor * conv_state, + ggml_tensor * ssm_state, + int n_tokens) { + const int head_dim = w.kda_head_dim; + const int n_head = w.n_head; + const int d_inner = head_dim * n_head; + + ggml_tensor * q = build_causal_conv1d( + ctx, gf, conv_state, 0, cur, L.wq, L.ssm_conv1d_q, + w.ssm_d_conv, d_inner, head_dim, n_head, n_tokens); + ggml_tensor * k = build_causal_conv1d( + ctx, gf, conv_state, 1, cur, L.wk, L.ssm_conv1d_k, + w.ssm_d_conv, d_inner, head_dim, n_head, n_tokens); + ggml_tensor * v = build_causal_conv1d( + ctx, gf, conv_state, 2, cur, L.wv, L.ssm_conv1d_v, + w.ssm_d_conv, d_inner, head_dim, n_head, n_tokens); + + ggml_tensor * gate = ggml_mul_mat(ctx, L.ssm_f_a, cur); + gate = ggml_add(ctx, gate, L.ssm_dt_bias); + gate = ggml_reshape_4d(ctx, gate, head_dim, n_head, n_tokens, 1); + ggml_tensor * a = ggml_reshape_4d(ctx, L.ssm_a, 1, n_head, 1, 1); + gate = ggml_scale(ctx, ggml_sigmoid(ctx, ggml_mul(ctx, gate, a)), + w.kda_gate_lower_bound); + + ggml_tensor * beta = ggml_mul_mat(ctx, L.ssm_beta, cur); + beta = ggml_sigmoid(ctx, + ggml_reshape_4d(ctx, beta, 1, n_head, n_tokens, 1)); + q = ggml_l2_norm(ctx, q, w.rms_eps); + k = ggml_l2_norm(ctx, k, w.rms_eps); + + ggml_tensor * state = ggml_reshape_4d( + ctx, ssm_state, head_dim, head_dim, n_head, 1); + ggml_tensor * packed = + ggml_gated_delta_net(ctx, q, k, v, gate, beta, state); + ggml_gated_delta_net_set_skip_intermediate(packed, true); + + const size_t element = ggml_element_size(packed); + ggml_tensor * output = ggml_view_4d( + ctx, packed, head_dim, n_head, n_tokens, 1, + static_cast(head_dim) * element, + static_cast(head_dim) * n_head * element, + static_cast(head_dim) * n_head * n_tokens * element, + 0); + ggml_tensor * new_state = ggml_view_4d( + ctx, packed, head_dim, head_dim, n_head, 1, + static_cast(head_dim) * element, + static_cast(head_dim) * head_dim * element, + static_cast(head_dim) * head_dim * n_head * element, + static_cast(head_dim) * n_head * n_tokens * element); + ggml_build_forward_expand(gf, ggml_cpy(ctx, new_state, state)); + + ggml_tensor * output_gate = ggml_mul_mat(ctx, L.ssm_g_a, cur); + output_gate = ggml_reshape_3d( + ctx, output_gate, head_dim, n_head, n_tokens); + output = ggml_reshape_3d(ctx, output, head_dim, n_head, n_tokens); + output = rms_norm_mul(ctx, output, L.ssm_norm, w.rms_eps); + output = ggml_mul(ctx, output, ggml_sigmoid(ctx, output_gate)); + output = ggml_cont_2d(ctx, output, d_inner, n_tokens); + return ggml_mul_mat(ctx, L.wo, output); +} + +} // namespace dflash::common diff --git a/server/src/bailingmoe3/bailingmoe3_graph.h b/server/src/bailingmoe3/bailingmoe3_graph.h new file mode 100644 index 000000000..e7a4cc3a9 --- /dev/null +++ b/server/src/bailingmoe3/bailingmoe3_graph.h @@ -0,0 +1,35 @@ +#pragma once + +struct ggml_cgraph; +struct ggml_context; +struct ggml_tensor; + +namespace dflash::common { + +struct TargetLayer; +struct TargetWeights; + +ggml_tensor * build_bailingmoe3_mla_block( + ggml_context * ctx, + ggml_cgraph * gf, + const TargetWeights & weights, + const TargetLayer & layer, + ggml_tensor * cur, + ggml_tensor * positions, + ggml_tensor * cache_k, + ggml_tensor * cache_v, + ggml_tensor * attn_mask, + int kv_start, + int n_tokens); + +ggml_tensor * build_bailingmoe3_kda_block( + ggml_context * ctx, + ggml_cgraph * gf, + const TargetWeights & weights, + const TargetLayer & layer, + ggml_tensor * cur, + ggml_tensor * conv_state, + ggml_tensor * ssm_state, + int n_tokens); + +} // namespace dflash::common diff --git a/server/src/bailingmoe3/bailingmoe3_loader.cpp b/server/src/bailingmoe3/bailingmoe3_loader.cpp new file mode 100644 index 000000000..bd5af37a4 --- /dev/null +++ b/server/src/bailingmoe3/bailingmoe3_loader.cpp @@ -0,0 +1,497 @@ +// Native GGUF loader for BailingMoE3 (Ling 3.x). +// +// Ling GGUFs contain one embedded NextN/MTP block after the 42-layer +// autoregressive trunk. This loader deliberately loads only the trunk; MTP is +// a decode optimization and is not required for a correctness-first backend. + +#include "internal.h" + +#include "common/gguf_bounds.h" +#include "common/gguf_mmap.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace dflash::common { +namespace { + +constexpr const char * kArch = "bailingmoe3"; + +uint32_t get_u32_or(const gguf_context * g, const std::string & key, + uint32_t fallback) { + const int64_t id = gguf_find_key(g, key.c_str()); + if (id < 0) return fallback; + if (gguf_get_kv_type(g, id) == GGUF_TYPE_ARRAY) { + if (gguf_get_arr_n(g, id) == 0) return fallback; + const gguf_type type = gguf_get_arr_type(g, id); + const void * data = gguf_get_arr_data(g, id); + if (type == GGUF_TYPE_UINT32) return static_cast(data)[0]; + if (type == GGUF_TYPE_INT32) { + const int32_t value = static_cast(data)[0]; + return value < 0 ? fallback : static_cast(value); + } + return fallback; + } + return gguf_get_val_u32(g, id); +} + +float get_f32_or(const gguf_context * g, const std::string & key, + float fallback) { + const int64_t id = gguf_find_key(g, key.c_str()); + if (id < 0) return fallback; + if (gguf_get_kv_type(g, id) == GGUF_TYPE_ARRAY) { + if (gguf_get_arr_n(g, id) == 0 || + gguf_get_arr_type(g, id) != GGUF_TYPE_FLOAT32) { + return fallback; + } + return static_cast(gguf_get_arr_data(g, id))[0]; + } + return gguf_get_val_f32(g, id); +} + +bool get_bool_or(const gguf_context * g, const std::string & key, + bool fallback) { + const int64_t id = gguf_find_key(g, key.c_str()); + if (id < 0 || gguf_get_kv_type(g, id) != GGUF_TYPE_BOOL) return fallback; + return gguf_get_val_bool(g, id); +} + +std::vector get_u32_array(const gguf_context * g, + const std::string & key) { + const int64_t id = gguf_find_key(g, key.c_str()); + if (id < 0 || gguf_get_kv_type(g, id) != GGUF_TYPE_ARRAY) return {}; + const gguf_type type = gguf_get_arr_type(g, id); + if (type != GGUF_TYPE_UINT32 && type != GGUF_TYPE_INT32) return {}; + const size_t n = gguf_get_arr_n(g, id); + const void * raw = gguf_get_arr_data(g, id); + std::vector result(n); + for (size_t i = 0; i < n; ++i) { + if (type == GGUF_TYPE_UINT32) { + result[i] = static_cast(raw)[i]; + } else { + const int32_t value = static_cast(raw)[i]; + if (value < 0) return {}; + result[i] = static_cast(value); + } + } + return result; +} + +std::vector get_f32_array(const gguf_context * g, + const std::string & key, + size_t count) { + std::vector result(count, 0.0f); + const int64_t id = gguf_find_key(g, key.c_str()); + if (id < 0) return result; + if (gguf_get_kv_type(g, id) == GGUF_TYPE_FLOAT32) { + std::fill(result.begin(), result.end(), gguf_get_val_f32(g, id)); + return result; + } + if (gguf_get_kv_type(g, id) != GGUF_TYPE_ARRAY || + gguf_get_arr_type(g, id) != GGUF_TYPE_FLOAT32) { + return result; + } + const size_t n = std::min(count, gguf_get_arr_n(g, id)); + const float * values = static_cast(gguf_get_arr_data(g, id)); + std::copy(values, values + n, result.begin()); + return result; +} + +size_t align_up(size_t value, size_t alignment) { + if (alignment == 0) return value; + const size_t remainder = value % alignment; + return remainder == 0 ? value : value + alignment - remainder; +} + +struct TensorAllocation { + ggml_tensor * tensor = nullptr; + size_t file_offset = 0; + size_t file_size = 0; + size_t buffer_offset = 0; +}; + +} // namespace + +bool load_bailingmoe3_gguf(const std::string & path, + ggml_backend_t backend, + TargetWeights & out) { + ggml_context * meta_ctx = nullptr; + gguf_init_params params{}; + params.no_alloc = true; + params.ctx = &meta_ctx; + gguf_context * gctx = gguf_init_from_file(path.c_str(), params); + if (!gctx) { + set_last_error("bailingmoe3: gguf_init_from_file failed: " + path); + return false; + } + + auto fail = [&](const std::string & message) { + set_last_error("bailingmoe3: " + message); + gguf_free(gctx); + if (meta_ctx) { + ggml_free(meta_ctx); + if (out.ctx == meta_ctx) out.ctx = nullptr; + meta_ctx = nullptr; + } + return false; + }; + + const int64_t arch_id = gguf_find_key(gctx, "general.architecture"); + const char * arch = arch_id >= 0 ? gguf_get_val_str(gctx, arch_id) : nullptr; + if (!arch || std::strcmp(arch, kArch) != 0) { + return fail(std::string("unexpected architecture '") + + (arch ? arch : "") + "'"); + } + + const std::string prefix = std::string(kArch) + "."; + const uint32_t block_count = get_u32_or(gctx, prefix + "block_count", 0); + const uint32_t nextn = get_u32_or(gctx, prefix + "nextn_predict_layers", 0); + if (block_count == 0 || nextn >= block_count) { + return fail("invalid block_count/nextn_predict_layers"); + } + const uint32_t n_layer = block_count - nextn; + const uint32_t n_embd = get_u32_or(gctx, prefix + "embedding_length", 0); + const uint32_t n_head = get_u32_or(gctx, prefix + "attention.head_count", 0); + const uint32_t kda_head_dim = get_u32_or(gctx, prefix + "kda.head_dim", 0); + const uint32_t ssm_conv = get_u32_or(gctx, prefix + "ssm.conv_kernel", 0); + const uint32_t rope_dim = get_u32_or(gctx, prefix + "rope.dimension_count", 0); + const uint32_t qk_mla = get_u32_or(gctx, prefix + "attention.key_length_mla", 0); + const uint32_t v_mla = get_u32_or(gctx, prefix + "attention.value_length_mla", 0); + const uint32_t kv_lora = get_u32_or(gctx, prefix + "attention.kv_lora_rank", 0); + const uint32_t q_lora = get_u32_or(gctx, prefix + "attention.q_lora_rank", 0); + const uint32_t n_ff = get_u32_or(gctx, prefix + "feed_forward_length", 0); + const uint32_t n_ff_exp = get_u32_or(gctx, prefix + "expert_feed_forward_length", 0); + const uint32_t n_ff_shexp = get_u32_or( + gctx, prefix + "expert_shared_feed_forward_length", 0); + const uint32_t n_expert = get_u32_or(gctx, prefix + "expert_count", 0); + const uint32_t n_expert_used = get_u32_or(gctx, prefix + "expert_used_count", 0); + const uint32_t n_expert_groups = get_u32_or( + gctx, prefix + "expert_group_count", 1); + const uint32_t n_expert_groups_used = get_u32_or( + gctx, prefix + "expert_group_used_count", 1); + const uint32_t dense_lead = get_u32_or(gctx, prefix + "leading_dense_block_count", 0); + const float kda_lower = get_f32_or(gctx, prefix + "kda.gate_lower_bound", 0.0f); + + if (n_layer == 0 || n_embd == 0 || n_head == 0 || kda_head_dim == 0 || + ssm_conv < 2 || rope_dim == 0 || qk_mla <= rope_dim || v_mla == 0 || + kv_lora == 0 || q_lora != 0 || n_ff == 0 || n_ff_exp == 0 || + n_ff_shexp == 0 || n_expert == 0 || n_expert_used == 0 || + n_expert_used > n_expert || n_expert_groups == 0 || + n_expert_groups_used == 0 || + n_expert_groups_used > n_expert_groups || + n_expert % n_expert_groups != 0 || dense_lead > n_layer || + kda_lower >= 0.0f) { + char message[640]; + std::snprintf(message, sizeof(message), + "invalid hparams: layers=%u(+%u MTP) embd=%u heads=%u " + "kda{head=%u conv=%u lower=%g} mla{qk=%u v=%u rope=%u kv_lora=%u q_lora=%u} " + "ff{dense=%u exp=%u shared=%u dense_lead=%u} " + "experts=%u used=%u groups=%u/%u", + n_layer, nextn, n_embd, n_head, kda_head_dim, ssm_conv, + static_cast(kda_lower), qk_mla, v_mla, rope_dim, + kv_lora, q_lora, n_ff, n_ff_exp, n_ff_shexp, dense_lead, + n_expert, n_expert_used, n_expert_groups_used, n_expert_groups); + return fail(message); + } + + // The per-layer KV-head array is the GGUF's authoritative recurrent/MLA + // pattern: zero means KDA, non-zero means MLA. LuceBox's shared hybrid + // cache uses a fixed interval, so verify that Ling's pattern is regular. + const std::vector kv_heads = + get_u32_array(gctx, prefix + "attention.head_count_kv"); + if (kv_heads.size() < n_layer) { + return fail("missing or short attention.head_count_kv array"); + } + int full_attention_interval = 0; + for (uint32_t il = 0; il < n_layer; ++il) { + if (kv_heads[il] != 0) { + const int interval = static_cast(il) + 1; + if (full_attention_interval == 0) full_attention_interval = interval; + if (interval % full_attention_interval != 0) { + return fail("irregular KDA/MLA layer pattern is not supported"); + } + } + } + if (full_attention_interval == 0 || n_layer % full_attention_interval != 0) { + return fail("no regular MLA layers found"); + } + for (uint32_t il = 0; il < n_layer; ++il) { + const bool expected_mla = ((il + 1) % full_attention_interval) == 0; + if ((kv_heads[il] != 0) != expected_mla) { + return fail("attention.head_count_kv does not match the inferred interval"); + } + } + + out.ctx = meta_ctx; + out.backend = backend; + out.n_layer = static_cast(n_layer); + out.n_embd = static_cast(n_embd); + out.n_head = static_cast(n_head); + out.n_head_kv = 1; // latent MLA is MQA + out.n_ff = static_cast(n_ff); + out.n_ff_exp = static_cast(n_ff_exp); + out.n_ff_shexp = static_cast(n_ff_shexp); + out.n_expert = static_cast(n_expert); + out.n_expert_used = static_cast(n_expert_used); + out.n_expert_groups = static_cast(n_expert_groups); + out.n_expert_groups_used = static_cast(n_expert_groups_used); + out.n_layer_dense_lead = static_cast(dense_lead); + out.full_attention_interval = full_attention_interval; + out.rope_dimension_count = static_cast(rope_dim); + out.rope_theta = get_f32_or(gctx, prefix + "rope.freq_base", 1000000.0f); + out.rms_eps = get_f32_or( + gctx, prefix + "attention.layer_norm_rms_epsilon", 1.0e-6f); + out.kda_head_dim = static_cast(kda_head_dim); + out.mla_qk_head_dim = static_cast(qk_mla); + out.mla_v_head_dim = static_cast(v_mla); + out.kv_lora_rank = static_cast(kv_lora); + out.q_lora_rank = static_cast(q_lora); + out.kda_gate_lower_bound = kda_lower; + out.n_embd_head_k = static_cast(kv_lora + rope_dim); + out.n_embd_head_v = static_cast(kv_lora); + out.ssm_d_conv = static_cast(ssm_conv); + out.ssm_d_inner = static_cast(n_head * kda_head_dim); + out.ssm_d_state = static_cast(kda_head_dim); + out.ssm_dt_rank = static_cast(n_head); + out.ssm_n_group = static_cast(n_head); + out.expert_gating_func = static_cast( + get_u32_or(gctx, prefix + "expert_gating_func", 2)); + out.expert_weights_scale = get_f32_or( + gctx, prefix + "expert_weights_scale", 1.0f); + out.expert_weights_norm = get_bool_or( + gctx, prefix + "expert_weights_norm", true); + out.is_moe = true; + out.is_bailingmoe3 = true; + + const uint32_t missing_token = 0xFFFFFFFFu; + const uint32_t eos = get_u32_or(gctx, "tokenizer.ggml.eos_token_id", missing_token); + const uint32_t eot = get_u32_or(gctx, "tokenizer.ggml.eot_token_id", missing_token); + out.eos_id = eos == missing_token ? -1 : static_cast(eos); + out.eos_chat_id = eot == missing_token ? -1 : static_cast(eot); + + out.layers.assign(n_layer, TargetLayer{}); + const std::vector clamp_exp = + get_f32_array(gctx, prefix + "swiglu_clamp_exp", n_layer); + const std::vector clamp_shexp = + get_f32_array(gctx, prefix + "swiglu_clamp_shexp", n_layer); + + auto tensor = [&](const char * name) { return ggml_get_tensor(meta_ctx, name); }; + auto layer_tensor = [&](uint32_t il, const char * suffix) { + char name[160]; + std::snprintf(name, sizeof(name), "blk.%u.%s", il, suffix); + return ggml_get_tensor(meta_ctx, name); + }; + + out.tok_embd = tensor("token_embd.weight"); + out.out_norm = tensor("output_norm.weight"); + out.output = tensor("output.weight"); + if (!out.tok_embd || !out.out_norm || !out.output) { + return fail("missing token_embd/output_norm/output tensor"); + } + out.n_vocab = static_cast(out.tok_embd->ne[1]); + + for (uint32_t il = 0; il < n_layer; ++il) { + TargetLayer & layer = out.layers[il]; + layer.attn_norm = layer_tensor(il, "attn_norm.weight"); + layer.ffn_norm = layer_tensor(il, "ffn_norm.weight"); + layer.attn_post_norm = layer.ffn_norm; + layer.ffn_swiglu_clamp_exp = clamp_exp[il]; + layer.ffn_swiglu_clamp_shexp = clamp_shexp[il]; + if (!layer.attn_norm || !layer.ffn_norm) { + return fail("layer " + std::to_string(il) + " missing norm tensor"); + } + + const bool mla = ((il + 1) % full_attention_interval) == 0; + layer.wo = layer_tensor(il, "attn_output.weight"); + if (mla) { + layer.wq = layer_tensor(il, "attn_q.weight"); + layer.attn_q_a = layer_tensor(il, "attn_q_a.weight"); + layer.attn_q_a_norm = layer_tensor(il, "attn_q_a_norm.weight"); + layer.attn_q_b = layer_tensor(il, "attn_q_b.weight"); + layer.attn_kv_a_mqa = layer_tensor(il, "attn_kv_a_mqa.weight"); + layer.attn_kv_a_norm = layer_tensor(il, "attn_kv_a_norm.weight"); + layer.attn_k_b = layer_tensor(il, "attn_k_b.weight"); + layer.attn_v_b = layer_tensor(il, "attn_v_b.weight"); + layer.wqkv_gate = layer_tensor(il, "attn_gate.weight"); + const bool has_q = layer.wq || + (layer.attn_q_a && layer.attn_q_a_norm && layer.attn_q_b); + if (!has_q || !layer.attn_kv_a_mqa || !layer.attn_kv_a_norm || + !layer.attn_k_b || !layer.attn_v_b || !layer.wqkv_gate || !layer.wo) { + return fail("layer " + std::to_string(il) + " missing MLA tensor"); + } + } else { + layer.wq = layer_tensor(il, "attn_q.weight"); + layer.wk = layer_tensor(il, "attn_k.weight"); + layer.wv = layer_tensor(il, "attn_v.weight"); + layer.ssm_conv1d_q = layer_tensor(il, "ssm_conv1d_q.weight"); + layer.ssm_conv1d_k = layer_tensor(il, "ssm_conv1d_k.weight"); + layer.ssm_conv1d_v = layer_tensor(il, "ssm_conv1d_v.weight"); + layer.ssm_f_a = layer_tensor(il, "ssm_f_a.weight"); + layer.ssm_beta = layer_tensor(il, "ssm_beta.weight"); + layer.ssm_a = layer_tensor(il, "ssm_a"); + layer.ssm_dt_bias = layer_tensor(il, "ssm_dt.bias"); + layer.ssm_g_a = layer_tensor(il, "ssm_g_a.weight"); + layer.ssm_norm = layer_tensor(il, "ssm_norm.weight"); + if (!layer.wq || !layer.wk || !layer.wv || !layer.wo || + !layer.ssm_conv1d_q || !layer.ssm_conv1d_k || + !layer.ssm_conv1d_v || !layer.ssm_f_a || !layer.ssm_beta || + !layer.ssm_a || !layer.ssm_dt_bias || !layer.ssm_g_a || + !layer.ssm_norm) { + return fail("layer " + std::to_string(il) + " missing KDA tensor"); + } + } + + if (il < dense_lead) { + layer.w_gate = layer_tensor(il, "ffn_gate.weight"); + layer.w_up = layer_tensor(il, "ffn_up.weight"); + layer.w_down = layer_tensor(il, "ffn_down.weight"); + if (!layer.w_gate || !layer.w_up || !layer.w_down) { + return fail("layer " + std::to_string(il) + " missing dense FFN tensor"); + } + } else { + layer.ffn_gate_inp = layer_tensor(il, "ffn_gate_inp.weight"); + layer.ffn_exp_probs_b = layer_tensor(il, "exp_probs_b.bias"); + layer.ffn_gate_exps = layer_tensor(il, "ffn_gate_exps.weight"); + layer.ffn_up_exps = layer_tensor(il, "ffn_up_exps.weight"); + layer.ffn_down_exps = layer_tensor(il, "ffn_down_exps.weight"); + layer.ffn_gate_shexp = layer_tensor(il, "ffn_gate_shexp.weight"); + layer.ffn_up_shexp = layer_tensor(il, "ffn_up_shexp.weight"); + layer.ffn_down_shexp = layer_tensor(il, "ffn_down_shexp.weight"); + if (!layer.ffn_gate_inp || !layer.ffn_exp_probs_b || + !layer.ffn_gate_exps || !layer.ffn_up_exps || + !layer.ffn_down_exps || !layer.ffn_gate_shexp || + !layer.ffn_up_shexp || !layer.ffn_down_shexp) { + return fail("layer " + std::to_string(il) + " missing MoE tensor"); + } + } + } + + // Allocate exactly the tensors referenced by the trunk. Prefix-based + // selection would also upload the 124B model's embedded MTP block. + std::unordered_set wanted; + auto add = [&](ggml_tensor * value) { if (value) wanted.insert(value); }; + add(out.out_norm); + add(out.output); + for (TargetLayer & layer : out.layers) { + add(layer.attn_norm); add(layer.ffn_norm); + add(layer.w_gate); add(layer.w_up); add(layer.w_down); + add(layer.wq); add(layer.wk); add(layer.wv); add(layer.wo); + add(layer.ssm_conv1d_q); add(layer.ssm_conv1d_k); add(layer.ssm_conv1d_v); + add(layer.ssm_f_a); add(layer.ssm_beta); add(layer.ssm_a); + add(layer.ssm_dt_bias); add(layer.ssm_g_a); add(layer.ssm_norm); + add(layer.attn_q_a); add(layer.attn_q_a_norm); add(layer.attn_q_b); + add(layer.attn_kv_a_mqa); add(layer.attn_kv_a_norm); + add(layer.attn_k_b); add(layer.attn_v_b); add(layer.wqkv_gate); + add(layer.ffn_gate_inp); add(layer.ffn_exp_probs_b); + add(layer.ffn_gate_exps); add(layer.ffn_up_exps); add(layer.ffn_down_exps); + add(layer.ffn_gate_shexp); add(layer.ffn_up_shexp); add(layer.ffn_down_shexp); + } + + const int64_t n_tensors = gguf_get_n_tensors(gctx); + ggml_backend_buffer_type_t buffer_type = + ggml_backend_get_default_buffer_type(backend); + const size_t alignment = ggml_backend_buft_get_alignment(buffer_type); + std::vector allocations; + allocations.reserve(wanted.size()); + size_t allocation_size = 0; + for (int64_t tid = 0; tid < n_tensors; ++tid) { + const char * name = gguf_get_tensor_name(gctx, tid); + ggml_tensor * value = ggml_get_tensor(meta_ctx, name); + if (!value || wanted.find(value) == wanted.end()) continue; + allocation_size = align_up(allocation_size, alignment); + TensorAllocation allocation; + allocation.tensor = value; + allocation.file_offset = + gguf_get_data_offset(gctx) + gguf_get_tensor_offset(gctx, tid); + allocation.file_size = gguf_get_tensor_size(gctx, tid); + allocation.buffer_offset = allocation_size; + allocation_size += ggml_backend_buft_get_alloc_size(buffer_type, value); + allocations.push_back(allocation); + } + if (allocations.size() != wanted.size()) { + return fail("failed to resolve every trunk tensor in the GGUF table"); + } + + out.buf = ggml_backend_alloc_buffer(backend, allocation_size); + if (!out.buf) return fail("weight buffer allocation failed"); + ggml_backend_buffer_set_usage(out.buf, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); + char * base = static_cast(ggml_backend_buffer_get_base(out.buf)); + for (const TensorAllocation & allocation : allocations) { + if (ggml_backend_tensor_alloc(out.buf, allocation.tensor, + base + allocation.buffer_offset) != GGML_STATUS_SUCCESS) { + ggml_backend_buffer_free(out.buf); + out.buf = nullptr; + return fail("weight tensor allocation failed"); + } + } + + GgufMmap mmap; + std::string mmap_error; + if (!mmap.open(path, mmap_error)) { + ggml_backend_buffer_free(out.buf); + out.buf = nullptr; + return fail(mmap_error); + } + const uint8_t * bytes = static_cast(mmap.data()); + const size_t file_size = mmap.size(); + for (const TensorAllocation & allocation : allocations) { + if (allocation.file_offset + allocation.file_size < allocation.file_offset || + allocation.file_offset + allocation.file_size > file_size) { + ggml_backend_buffer_free(out.buf); + out.buf = nullptr; + return fail("truncated tensor data for " + + std::string(allocation.tensor->name)); + } + ggml_backend_tensor_set(allocation.tensor, + bytes + allocation.file_offset, 0, allocation.file_size); + } + + const int64_t token_tid = gguf_find_tensor(gctx, "token_embd.weight"); + if (token_tid < 0) { + ggml_backend_buffer_free(out.buf); + out.buf = nullptr; + return fail("token_embd.weight missing from tensor table"); + } + const size_t token_relative_offset = gguf_get_tensor_offset(gctx, token_tid); + const size_t token_size = gguf_get_tensor_size(gctx, token_tid); + const size_t data_offset = gguf_get_data_offset(gctx); + if (!gguf_tensor_in_file(data_offset, token_relative_offset, token_size, file_size)) { + ggml_backend_buffer_free(out.buf); + out.buf = nullptr; + return fail("truncated token_embd.weight"); + } + out.embedder.tok_embd_owned.resize(token_size); + std::memcpy(out.embedder.tok_embd_owned.data(), + bytes + data_offset + token_relative_offset, token_size); + out.embedder.tok_embd_bytes = out.embedder.tok_embd_owned.data(); + out.embedder.tok_embd_type = gguf_get_tensor_type(gctx, token_tid); + out.embedder.n_embd = out.n_embd; + out.embedder.n_vocab = out.n_vocab; + out.embedder.row_bytes = token_size / static_cast(out.n_vocab); + + gguf_free(gctx); + gctx = nullptr; + meta_ctx = nullptr; // owned by out.ctx from here on + + char summary[384]; + std::snprintf(summary, sizeof(summary), + "bailingmoe3 trunk loaded: %d layers (%d KDA + %d MLA), " + "%zu tensors %.2f GiB, experts=%d/%d groups=%d/%d, " + "MTP blocks ignored=%u, eos=%d", + out.n_layer, out.n_layer - out.n_layer / out.full_attention_interval, + out.n_layer / out.full_attention_interval, allocations.size(), + allocation_size / (1024.0 * 1024.0 * 1024.0), + out.n_expert_used, out.n_expert, + out.n_expert_groups_used, out.n_expert_groups, + nextn, out.eos_id); + set_last_error(summary); + std::fprintf(stderr, "[bailingmoe3] %s\n", summary); + return true; +} + +} // namespace dflash::common diff --git a/server/src/common/backend_factory.cpp b/server/src/common/backend_factory.cpp index ee9b198bf..d462be899 100644 --- a/server/src/common/backend_factory.cpp +++ b/server/src/common/backend_factory.cpp @@ -6,6 +6,7 @@ #include "qwen35_backend.h" #include "qwen35moe_backend.h" +#include "bailingmoe3_backend.h" #include "laguna_backend.h" #include "laguna_layer_split_adapter.h" #include "qwen3_backend.h" @@ -92,6 +93,7 @@ constexpr bool layer_split_carries(FeatureSupport support) { DFLASH_CHECK_ARCH("qwen35", Qwen35Config, Qwen35LayerSplitAdapterConfig); DFLASH_CHECK_ARCH("qwen35moe", Qwen35Config, NoLayerSplitConfig); +DFLASH_CHECK_ARCH("bailingmoe3", BailingMoe3Config, NoLayerSplitConfig); DFLASH_CHECK_ARCH("laguna", LagunaBackendArgs, LagunaLayerSplitAdapterConfig); DFLASH_CHECK_ARCH("qwen3", Qwen3BackendConfig, NoLayerSplitConfig); DFLASH_CHECK_ARCH("gemma4", Gemma4BackendConfig, Gemma4LayerSplitAdapterConfig); @@ -321,6 +323,19 @@ std::unique_ptr create_backend( } return backend; + } else if (arch == "bailingmoe3") { + BailingMoe3Config cfg; + cfg.model_path = args.model_path; + cfg.device = args.device; + cfg.stream_fd = args.stream_fd; + + auto backend = std::make_unique(cfg); + if (!backend->init()) { + std::fprintf(stderr, "[backend_factory] BailingMoe3Backend init failed\n"); + return nullptr; + } + return backend; + } else if (arch == "laguna") { if (args.device.is_layer_split()) { LagunaLayerSplitAdapterConfig cfg; diff --git a/server/src/common/gguf_inspect.cpp b/server/src/common/gguf_inspect.cpp index 9c3b80143..58444be95 100644 --- a/server/src/common/gguf_inspect.cpp +++ b/server/src/common/gguf_inspect.cpp @@ -26,9 +26,10 @@ bool derive_effective_target_layer_count(const std::string & arch, return false; } - // Embedded NextN blocks are currently defined by the Qwen3.5/3.6 GGUF - // layout. Do not reinterpret similarly named metadata on other arches. - if (arch != "qwen35" && arch != "qwen35moe") { + // Embedded NextN blocks are defined by the Qwen3.5/3.6 and BailingMoE3 + // GGUF layouts. Do not reinterpret similarly named metadata elsewhere. + if (arch != "qwen35" && arch != "qwen35moe" && + arch != "bailingmoe3") { return true; } if (nextn_predict_layers == 0) { diff --git a/server/src/common/model_capabilities.h b/server/src/common/model_capabilities.h index f62087141..ebc7524c0 100644 --- a/server/src/common/model_capabilities.h +++ b/server/src/common/model_capabilities.h @@ -73,10 +73,11 @@ inline constexpr ArchCapabilities kArchCapabilities[] = { // arch split rdraft pflash offload draft ddtree vwidth dblock fa_win dswa paged {"qwen35", true, true, true, false, kBoth, kBoth, kNever, kMono, kBoth, kBoth, kMono}, {"qwen35moe", false, false, false, true, kMono, kMono, kNever, kNever, kMono, kMono, kNever}, - {"laguna", true, false, false, true, kMono, kMono, kMono, kNever, kNever, kNever, kNever}, - {"qwen3", false, false, true, false, kNever, kNever, kNever, kNever, kNever, kNever, kNever}, - {"gemma4", true, false, false, false, kMono, kNever, kNever, kNever, kBoth, kNever, kNever}, - {"deepseek4", true, false, false, false, kNever, kNever, kNever, kNever, kNever, kNever, kNever}, + {"bailingmoe3",false, false, false, false, kNever,kNever,kNever, kNever, kNever,kNever, kNever}, + {"laguna", true, false, false, true, kMono, kMono, kMono, kNever, kNever,kNever, kNever}, + {"qwen3", false, false, true, false, kNever, kNever, kNever, kNever, kNever,kNever, kNever}, + {"gemma4", true, false, false, false, kMono, kNever, kNever, kNever, kBoth,kNever, kNever}, + {"deepseek4", true, false, false, false, kNever, kNever, kNever, kNever, kNever,kNever, kNever}, }; inline constexpr std::size_t kArchCount = diff --git a/server/src/internal.h b/server/src/internal.h index 792be9fcc..07d6b89a0 100644 --- a/server/src/internal.h +++ b/server/src/internal.h @@ -87,11 +87,31 @@ struct TargetLayer { // one small GPU tensor per DeltaNet layer (src[9] of the GDN op). ggml_tensor * ssm_gate_ba = nullptr; + // BailingMoE3 / Ling 3 KDA. Unlike Qwen3.5's fused projection and + // convolution, Ling projects and convolves Q, K, and V independently and + // uses a vector-valued decay gate (KDA) per head dimension. + ggml_tensor * ssm_conv1d_q = nullptr; // [conv, 1, d_inner, 1] + ggml_tensor * ssm_conv1d_k = nullptr; // [conv, 1, d_inner, 1] + ggml_tensor * ssm_conv1d_v = nullptr; // [conv, 1, d_inner, 1] + ggml_tensor * ssm_f_a = nullptr; // [hidden, d_inner] + ggml_tensor * ssm_g_a = nullptr; // [hidden, d_inner] + + // BailingMoE3 / Ling 3 MLA. The latent K/V cache stores + // [kv_lora_rank + rope_dim] for K and [kv_lora_rank] for V. + ggml_tensor * attn_q_a = nullptr; + ggml_tensor * attn_q_a_norm = nullptr; + ggml_tensor * attn_q_b = nullptr; + ggml_tensor * attn_kv_a_mqa = nullptr; + ggml_tensor * attn_kv_a_norm = nullptr; + ggml_tensor * attn_k_b = nullptr; + ggml_tensor * attn_v_b = nullptr; + // MoE FFN (qwen35moe only; nullptr on dense qwen35) ggml_tensor * ffn_gate_inp = nullptr; // [hidden, n_expert] router ggml_tensor * ffn_gate_exps = nullptr; // [hidden, n_ff_exp, n_expert] ggml_tensor * ffn_up_exps = nullptr; // [hidden, n_ff_exp, n_expert] ggml_tensor * ffn_down_exps = nullptr; // [n_ff_exp, hidden, n_expert] + ggml_tensor * ffn_exp_probs_b = nullptr; // [n_expert] router correction bias ggml_tensor * ffn_gate_up_exps = nullptr; // [hidden, 2*n_ff_exp, n_expert] optional fused gate/up ggml_tensor * ffn_gate_inp_shexp = nullptr; // [hidden] shared-expert scalar gate ggml_tensor * ffn_gate_shexp = nullptr; // [hidden, n_ff_shexp] @@ -126,6 +146,11 @@ struct TargetLayer { float ffn_gate_shexp_s = 1.0f; float ffn_up_shexp_s = 1.0f; float ffn_down_shexp_s = 1.0f; + + // Optional per-layer activation limits used by late Ling 3 blocks. + // Zero means ordinary SwiGLU. + float ffn_swiglu_clamp_exp = 0.0f; + float ffn_swiglu_clamp_shexp = 0.0f; }; // CPU-side embedder: keeps a mmap of the GGUF alive and knows how to @@ -186,6 +211,8 @@ struct TargetWeights { int n_ff_shexp = 0; int n_expert = 0; int n_expert_used = 0; + int n_expert_groups = 1; + int n_expert_groups_used = 1; int n_vocab = DFLASH27B_TARGET_VOCAB; int rope_dimension_count = 64; float rope_theta = 10000000.0f; @@ -193,6 +220,17 @@ struct TargetWeights { float expert_weights_scale = 1.0f; int expert_gating_func = 1; // 1=softmax, 2=sigmoid (llama.cpp enum values) bool is_moe = false; + bool is_bailingmoe3 = false; + bool expert_weights_norm = true; + int n_layer_dense_lead = 0; + + // BailingMoE3 / Ling 3 architecture parameters. + int kda_head_dim = 0; + int mla_qk_head_dim = 0; + int mla_v_head_dim = 0; + int kv_lora_rank = 0; + int q_lora_rank = 0; + float kda_gate_lower_bound = 0.0f; int ssm_d_conv = 4; int ssm_d_inner = 6144; int ssm_d_state = 128; @@ -243,6 +281,12 @@ bool load_target_gguf_partial(const std::string & path, const TargetLoadPlan & plan, TargetWeights & out); +// Load the autoregressive trunk of a BailingMoE3 GGUF (Ling 3.x). Embedded +// NextN/MTP blocks are intentionally ignored by this baseline backend. +bool load_bailingmoe3_gguf(const std::string & path, + ggml_backend_t backend, + TargetWeights & out); + void free_target_weights(TargetWeights & w); // ─── Draft weights (z-lab DFlash, bf16) ─────────────────────────── diff --git a/server/src/qwen35/qwen35_backend.cpp b/server/src/qwen35/qwen35_backend.cpp index 20a4b4f1a..57d0f23d5 100644 --- a/server/src/qwen35/qwen35_backend.cpp +++ b/server/src/qwen35/qwen35_backend.cpp @@ -1616,7 +1616,8 @@ GenerateResult Qwen35Backend::restore_and_generate_impl(int slot, const bool pool = kvflash_active(); if (!build_target_step(sg_, w_, cache_, target_backend_, /*kv_start=*/cache_.cur_pos, /*n_tokens=*/1, - /*with_mask=*/pool, /*capture=*/false, + /*with_mask=*/pool || w_.is_bailingmoe3, + /*capture=*/false, /*capture_delta_intermediate=*/false, /*fa_window=*/0, /*logits_tail_rows=*/0, @@ -2255,7 +2256,8 @@ bool Qwen35Backend::do_ar_decode(int committed, int n_gen, const bool paged = cfg_.paged_attention; if (!build_target_step(sg_, w_, cache_, target_backend_, /*kv_start=*/committed, /*n_tokens=*/1, - /*with_mask=*/pool, /*capture=*/false, + /*with_mask=*/pool || w_.is_bailingmoe3, + /*capture=*/false, /*capture_delta_intermediate=*/false, /*fa_window=*/0, /*logits_tail_rows=*/0, @@ -2291,7 +2293,12 @@ bool Qwen35Backend::do_ar_decode(int committed, int n_gen, ggml_backend_tensor_set(sg_.kv_write_rows, row_vals.data(), 0, sizeof(int64_t) * n_head_kv); } - if (pool) kvflash_upload_mask(); + if (pool) { + kvflash_upload_mask(); + } else if (w_.is_bailingmoe3) { + upload_qwen35_causal_mask( + sg_.attn_mask, committed, 1, cfg_.kq_stride_pad); + } auto st = ggml_backend_graph_compute(target_backend_, sg_.gf); if (st != GGML_STATUS_SUCCESS) return false; diff --git a/server/src/qwen35/qwen35_target_graph.cpp b/server/src/qwen35/qwen35_target_graph.cpp index 0e87c1a36..7e7c3b613 100644 --- a/server/src/qwen35/qwen35_target_graph.cpp +++ b/server/src/qwen35/qwen35_target_graph.cpp @@ -31,6 +31,7 @@ // conv_kernel = 4 #include "internal.h" +#include "bailingmoe3_graph.h" #include "delta_net_chunked.h" #include "delta_net_specla.h" #include "kv_quant.h" @@ -184,10 +185,13 @@ bool create_target_cache_partial(const TargetWeights & w, // The rotation costs two extra launches per attention layer. Decision // logic lives in common/kv_rotation.h and is shared with the disk // prefix cache identity salt: a cache written under one rotation basis - // must never be adopted by a session using the other. - out.kv_k_rotated = dflash_kv_k_rotation_enabled(ggml_type_name(kv_k_type)); + // must never be adopted by a session using the other. Ling's latent MLA + // cache uses unequal K/V widths, so it cannot use the qwen FWHT path. + out.kv_k_rotated = + !w.is_bailingmoe3 && + dflash_kv_k_rotation_enabled(ggml_type_name(kv_k_type)); - const bool needs_256_stride = + const bool needs_256_stride = w.is_bailingmoe3 || kv_k_type == GGML_TYPE_TQ3_0 || kv_v_type == GGML_TYPE_TQ3_0; // kvflash mode: attention tensors are allocated at the (smaller) // physical pool capacity; logical positions are mapped to pool slots @@ -230,8 +234,11 @@ bool create_target_cache_partial(const TargetWeights & w, // [head_dim, max_ctx_alloc, n_head_kv] ggml_tensor * K = ggml_new_tensor_3d(out.base_ctx, kv_k_type, head_dim, max_ctx_alloc, w.n_head_kv); - ggml_tensor * V = ggml_new_tensor_3d(out.base_ctx, kv_v_type, - head_dim, max_ctx_alloc, w.n_head_kv); + const int v_head_dim = w.is_bailingmoe3 + ? w.n_embd_head_v : head_dim; + ggml_tensor * V = ggml_new_tensor_3d( + out.base_ctx, kv_v_type, v_head_dim, + max_ctx_alloc, w.n_head_kv); char name[64]; std::snprintf(name, sizeof(name), "cache_k_%d", il); ggml_set_name(K, name); @@ -2215,14 +2222,18 @@ static ggml_tensor * build_single_layer( for (int il = 0; il < layer_idx; il++) { if (((il + 1) % w.full_attention_interval) == 0) fa_idx++; } - cur = build_full_attn_block(ctx, gf, w, L, cur, positions, w.rope_sections, - cache.attn_k[fa_idx], cache.attn_v[fa_idx], - attn_mask, kv_start, n_tokens, - cache.kv_k_type, cache.kv_v_type, - cache.kv_k_rotated, - fa_window, - q_tail_capture, q_tail_start, - kv_write_rows); + cur = w.is_bailingmoe3 + ? build_bailingmoe3_mla_block( + ctx, gf, w, L, cur, positions, + cache.attn_k[fa_idx], cache.attn_v[fa_idx], + attn_mask, kv_start, n_tokens) + : build_full_attn_block( + ctx, gf, w, L, cur, positions, w.rope_sections, + cache.attn_k[fa_idx], cache.attn_v[fa_idx], + attn_mask, kv_start, n_tokens, + cache.kv_k_type, cache.kv_v_type, + cache.kv_k_rotated, fa_window, + q_tail_capture, q_tail_start, kv_write_rows); } else { int dn_idx = 0; for (int il = 0; il < layer_idx; il++) { @@ -2235,11 +2246,16 @@ static ggml_tensor * build_single_layer( cap_ptr->ssm_intermediate_states = cache.ssm_intermediate[dn_idx]; cap_ptr->conv_input = cache.conv_input_cache[dn_idx]; } - cur = build_delta_net_block(ctx, gf, w, L, cur, - cache.conv_state[dn_idx], cache.ssm_state[dn_idx], - n_tokens, cap_ptr, parent_ids, - /*skip_gdn_intermediate=*/true, - supports_qwen35_fused_kernels(cache.backend)); + cur = w.is_bailingmoe3 + ? build_bailingmoe3_kda_block( + ctx, gf, w, L, cur, cache.conv_state[dn_idx], + cache.ssm_state[dn_idx], n_tokens) + : build_delta_net_block( + ctx, gf, w, L, cur, + cache.conv_state[dn_idx], cache.ssm_state[dn_idx], + n_tokens, cap_ptr, parent_ids, + /*skip_gdn_intermediate=*/true, + supports_qwen35_fused_kernels(cache.backend)); } cur = ggml_add(ctx, cur, inpSA); @@ -2247,8 +2263,9 @@ static ggml_tensor * build_single_layer( ggml_tensor * ffn_residual = cur; ggml_tensor * post = rms_norm_mul(ctx, cur, L.attn_post_norm, eps); ggml_tensor * moe_selected = nullptr; - ggml_tensor * ffn = w.is_moe ? build_qwen35moe_ffn(ctx, post, w, L, &moe_selected) - : build_swiglu_ffn(ctx, post, L); + ggml_tensor * ffn = L.ffn_gate_inp + ? build_qwen35moe_ffn(ctx, post, w, L, &moe_selected) + : build_swiglu_ffn(ctx, post, L); if (moe_selected_out) { *moe_selected_out = moe_selected; } @@ -2350,28 +2367,38 @@ QwenGraphOutputs build_qwen35_graph( if (is_attn) { const bool want_q_cap = in.q_capture && cache.q_cap; ggml_tensor * q_fa = nullptr; - cur = build_full_attn_block(ctx, gf, w, L, cur, in.positions, w.rope_sections, - cache.attn_k[fa_idx], cache.attn_v[fa_idx], - in.attn_mask, in.kv_start, n_tokens, - cache.kv_k_type, cache.kv_v_type, - cache.kv_k_rotated, - in.fa_window, - /*q_tail_capture=*/nullptr, - /*q_tail_start=*/0, - in.kv_write_rows, - want_q_cap ? &q_fa : nullptr, - in.paged_block_table, - in.paged_kv_seq_lens, - in.paged_query_seq_ids, - in.paged_query_positions, - in.paged_max_kv_len, - in.active_slot_ids, - in.tree_sizes ? in.parent_ids : nullptr, - in.tree_sizes, - in.tree_width, - in.tree_scratch_base, - in.tree_scratch_stride, - cache.max_ctx); + if (w.is_bailingmoe3) { + GGML_ASSERT(!in.kv_write_rows && !in.paged_block_table && + !in.paged_query_seq_ids && !in.tree_sizes && + in.n_seqs == 1); + cur = build_bailingmoe3_mla_block( + ctx, gf, w, L, cur, in.positions, + cache.attn_k[fa_idx], cache.attn_v[fa_idx], + in.attn_mask, in.kv_start, n_tokens); + } else { + cur = build_full_attn_block(ctx, gf, w, L, cur, in.positions, w.rope_sections, + cache.attn_k[fa_idx], cache.attn_v[fa_idx], + in.attn_mask, in.kv_start, n_tokens, + cache.kv_k_type, cache.kv_v_type, + cache.kv_k_rotated, + in.fa_window, + /*q_tail_capture=*/nullptr, + /*q_tail_start=*/0, + in.kv_write_rows, + want_q_cap ? &q_fa : nullptr, + in.paged_block_table, + in.paged_kv_seq_lens, + in.paged_query_seq_ids, + in.paged_query_positions, + in.paged_max_kv_len, + in.active_slot_ids, + in.tree_sizes ? in.parent_ids : nullptr, + in.tree_sizes, + in.tree_width, + in.tree_scratch_base, + in.tree_scratch_stride, + cache.max_ctx); + } if (want_q_cap && q_fa) { // Last token's Q, all heads: src [head_dim, 1, n_head] view of // [head_dim, n_tokens, n_head]; dst = q_cap plane fa_idx @@ -2445,25 +2472,32 @@ QwenGraphOutputs build_qwen35_graph( ssm_st->nb[1], ssm_st->nb[2], ssm_st->nb[3], (size_t)in.seq_slot * ssm_st->nb[3]); } - cur = build_delta_net_block(ctx, gf, w, L, cur, - conv_st, ssm_st, - n_tokens, cap_ptr, in.parent_ids, - /*skip_gdn_intermediate=*/true, - supports_qwen35_fused_kernels(cache.backend), - in.n_seqs, - in.prefill_segments, - in.n_prefill_segments, - in.active_slot_ids, - in.state_slot_ids, - in.mapped_ar_seqs, - /*allow_inplace_state=*/ - in.n_prefill_tokens == 0, - in.specla_m_strict, in.specla_m_incl, - in.specla_m_eye, in.specla_hld, - in.specla_n_boundaries, - in.specla_n_chains, - in.specla_n_waves, - in.specla_max_parallel_chains); + if (w.is_bailingmoe3) { + GGML_ASSERT(!cap_ptr && !in.parent_ids && in.n_seqs == 1 && + in.n_prefill_segments == 0 && !in.active_slot_ids); + cur = build_bailingmoe3_kda_block( + ctx, gf, w, L, cur, conv_st, ssm_st, n_tokens); + } else { + cur = build_delta_net_block(ctx, gf, w, L, cur, + conv_st, ssm_st, + n_tokens, cap_ptr, in.parent_ids, + /*skip_gdn_intermediate=*/true, + supports_qwen35_fused_kernels(cache.backend), + in.n_seqs, + in.prefill_segments, + in.n_prefill_segments, + in.active_slot_ids, + in.state_slot_ids, + in.mapped_ar_seqs, + /*allow_inplace_state=*/ + in.n_prefill_tokens == 0, + in.specla_m_strict, in.specla_m_incl, + in.specla_m_eye, in.specla_hld, + in.specla_n_boundaries, + in.specla_n_chains, + in.specla_n_waves, + in.specla_max_parallel_chains); + } dn_idx++; } @@ -2476,9 +2510,10 @@ QwenGraphOutputs build_qwen35_graph( // FFN (dense SwiGLU for qwen35, MoE for qwen35moe) ggml_tensor * moe_selected = nullptr; - ggml_tensor * ffn = w.is_moe ? build_qwen35moe_ffn(ctx, post, w, L, - in.capture_moe_router ? &moe_selected : nullptr) - : build_swiglu_ffn(ctx, post, L); + ggml_tensor * ffn = L.ffn_gate_inp + ? build_qwen35moe_ffn(ctx, post, w, L, + in.capture_moe_router ? &moe_selected : nullptr) + : build_swiglu_ffn(ctx, post, L); if (in.capture_moe_router && moe_selected) { ggml_set_output(moe_selected); og_early.moe_selected[(size_t)il] = moe_selected; @@ -2669,30 +2704,39 @@ QwenLayerPrefnOutputs build_qwen35_layer_prefn( for (int il = 0; il < layer_idx; il++) { if (((il + 1) % w.full_attention_interval) == 0) fa_idx++; } - cur = build_full_attn_block(ctx, gf, w, L, cur, positions, w.rope_sections, - cache.attn_k[fa_idx], cache.attn_v[fa_idx], - attn_mask, kv_start, n_tokens, - cache.kv_k_type, cache.kv_v_type, - cache.kv_k_rotated, - fa_window, - /*q_tail_capture=*/nullptr, /*q_tail_start=*/0, - kv_write_rows); + cur = w.is_bailingmoe3 + ? build_bailingmoe3_mla_block( + ctx, gf, w, L, cur, positions, + cache.attn_k[fa_idx], cache.attn_v[fa_idx], + attn_mask, kv_start, n_tokens) + : build_full_attn_block( + ctx, gf, w, L, cur, positions, w.rope_sections, + cache.attn_k[fa_idx], cache.attn_v[fa_idx], + attn_mask, kv_start, n_tokens, + cache.kv_k_type, cache.kv_v_type, + cache.kv_k_rotated, fa_window, + /*q_tail_capture=*/nullptr, /*q_tail_start=*/0, + kv_write_rows); } else { int dn_idx = 0; for (int il = 0; il < layer_idx; il++) { if (((il + 1) % w.full_attention_interval) != 0) dn_idx++; } - cur = build_delta_net_block(ctx, gf, w, L, cur, - cache.conv_state[dn_idx], cache.ssm_state[dn_idx], - n_tokens, nullptr, nullptr, - skip_gdn_intermediate, - supports_qwen35_fused_kernels(cache.backend)); + cur = w.is_bailingmoe3 + ? build_bailingmoe3_kda_block( + ctx, gf, w, L, cur, cache.conv_state[dn_idx], + cache.ssm_state[dn_idx], n_tokens) + : build_delta_net_block( + ctx, gf, w, L, cur, + cache.conv_state[dn_idx], cache.ssm_state[dn_idx], + n_tokens, nullptr, nullptr, skip_gdn_intermediate, + supports_qwen35_fused_kernels(cache.backend)); } cur = ggml_add(ctx, cur, inpSA); out.residual = cur; out.post = rms_norm_mul(ctx, cur, L.attn_post_norm, eps); - if (w.is_moe) { + if (L.ffn_gate_inp) { // selected/weights are read back by the host (hybrid hot/cold expert // compute), not consumed in-graph. argsort_top_k yields a strided view // whose raw packed readback returns garbage ids for tokens > 0 (crash diff --git a/server/src/qwen35moe/qwen35moe_ffn.cpp b/server/src/qwen35moe/qwen35moe_ffn.cpp index a71a94a4a..513e46233 100644 --- a/server/src/qwen35moe/qwen35moe_ffn.cpp +++ b/server/src/qwen35moe/qwen35moe_ffn.cpp @@ -30,6 +30,51 @@ Qwen35MoeRouterOutputs build_qwen35moe_router( break; } + // Bailing/DeepSeek correction bias changes expert selection only. The + // selected experts retain their unbiased sigmoid weights. + ggml_tensor * selection_probs = probs; + if (L.ffn_exp_probs_b) { + selection_probs = ggml_add(ctx, probs, L.ffn_exp_probs_b); + } + + // Ling 3 routes hierarchically: rank each group by the sum of its two + // strongest experts, keep the configured top groups, then select the + // final experts from that masked set. This is the DeepSeek-V3 grouping + // rule used by llama.cpp's generic MoE builder. + if (w.n_expert_groups > 1) { + GGML_ASSERT(n_expert % w.n_expert_groups == 0); + GGML_ASSERT(w.n_expert_groups_used > 0 && + w.n_expert_groups_used <= w.n_expert_groups); + const int n_expert_per_group = n_expert / w.n_expert_groups; + ggml_tensor * selection_groups = ggml_reshape_3d( + ctx, selection_probs, n_expert_per_group, + w.n_expert_groups, n_tokens); + + ggml_tensor * group_scores = ggml_argsort_top_k( + ctx, selection_groups, 2); + group_scores = ggml_get_rows( + ctx, + ggml_reshape_4d(ctx, selection_groups, 1, + n_expert_per_group, w.n_expert_groups, + n_tokens), + group_scores); + group_scores = ggml_sum_rows( + ctx, ggml_reshape_3d(ctx, group_scores, 2, + w.n_expert_groups, n_tokens)); + group_scores = ggml_reshape_2d( + ctx, group_scores, w.n_expert_groups, n_tokens); + + ggml_tensor * selected_groups = ggml_argsort_top_k( + ctx, group_scores, w.n_expert_groups_used); + ggml_tensor * kept_groups = ggml_get_rows( + ctx, selection_groups, selected_groups); + selection_groups = ggml_set_rows( + ctx, ggml_fill(ctx, selection_groups, -INFINITY), + kept_groups, selected_groups); + selection_probs = ggml_reshape_2d( + ctx, selection_groups, n_expert, n_tokens); + } + // ggml_argsort_top_k emits GGML_OP_ARGSORT (+view), which ggml-cuda's // topk-moe fusion (ggml_cuda_topk_moe_fusion) recognizes and fuses the whole // softmax->topk->get_rows->norm router into ~1 kernel. ggml_top_k emits @@ -38,18 +83,16 @@ Qwen35MoeRouterOutputs build_qwen35moe_router( // Same top-k selection -> bit-identical. DFLASH_NO_MOE_ROUTER_FUSE=1 = old path. static const bool router_fuse = (std::getenv("DFLASH_NO_MOE_ROUTER_FUSE") == nullptr); ggml_tensor * selected = (router_fuse && allow_fused_router) - ? ggml_argsort_top_k(ctx, probs, n_used) - : ggml_top_k(ctx, probs, n_used); + ? ggml_argsort_top_k(ctx, selection_probs, n_used) + : ggml_top_k(ctx, selection_probs, n_used); ggml_tensor * probs_3d = ggml_reshape_3d(ctx, probs, 1, n_expert, n_tokens); ggml_tensor * weights = ggml_get_rows(ctx, probs_3d, selected); weights = ggml_reshape_2d(ctx, weights, n_used, n_tokens); - // Always normalize selected expert weights by their sum (matches - // llama.cpp's norm_w=true for qwen35moe). Without this, top-k softmax - // weights sum to much less than 1.0, causing systematically underscaled - // FFN output. - { + // Qwen3.5 MoE and Ling 3 both ship norm_w=true. Keep the metadata switch + // explicit because Bailing-family checkpoints can legally disable it. + if (w.expert_weights_norm) { ggml_tensor * w_sum = ggml_sum_rows(ctx, weights); w_sum = ggml_clamp(ctx, w_sum, 6.103515625e-5f, INFINITY); weights = ggml_div(ctx, weights, w_sum); @@ -95,7 +138,7 @@ ggml_tensor * build_qwen35moe_ffn( if (L.ffn_gate_up_exps) { ggml_tensor * gate_up_e = apply_scale2( ctx, ggml_mul_mat_id(ctx, L.ffn_gate_up_exps, cur_3d, selected), L.ffn_gate_up_exps_s); - if (moe_swiglu_fuse) { + if (moe_swiglu_fuse && L.ffn_swiglu_clamp_exp <= 0.0f) { gu = ggml_swiglu(ctx, gate_up_e); // silu(gate) * up, no views/conts } else { ggml_tensor * gate_e = ggml_view_3d(ctx, gate_up_e, @@ -107,14 +150,30 @@ ggml_tensor * build_qwen35moe_ffn( (size_t)n_ff_exp * ggml_element_size(gate_up_e)); gate_e = ggml_cont(ctx, gate_e); up_e = ggml_cont(ctx, up_e); - gu = ggml_swiglu_split(ctx, gate_e, up_e); + if (L.ffn_swiglu_clamp_exp > 0.0f) { + const float limit = L.ffn_swiglu_clamp_exp; + up_e = ggml_clamp(ctx, up_e, -limit, limit); + gate_e = ggml_clamp( + ctx, ggml_silu(ctx, gate_e), -INFINITY, limit); + gu = ggml_mul(ctx, gate_e, up_e); + } else { + gu = ggml_swiglu_split(ctx, gate_e, up_e); + } } } else { ggml_tensor * gate_e = apply_scale2( ctx, ggml_mul_mat_id(ctx, L.ffn_gate_exps, cur_3d, selected), L.ffn_gate_exps_s); ggml_tensor * up_e = apply_scale2( ctx, ggml_mul_mat_id(ctx, L.ffn_up_exps, cur_3d, selected), L.ffn_up_exps_s); - gu = ggml_swiglu_split(ctx, gate_e, up_e); + if (L.ffn_swiglu_clamp_exp > 0.0f) { + const float limit = L.ffn_swiglu_clamp_exp; + up_e = ggml_clamp(ctx, up_e, -limit, limit); + gate_e = ggml_clamp( + ctx, ggml_silu(ctx, gate_e), -INFINITY, limit); + gu = ggml_mul(ctx, gate_e, up_e); + } else { + gu = ggml_swiglu_split(ctx, gate_e, up_e); + } } ggml_tensor * experts = apply_scale2( @@ -132,7 +191,16 @@ ggml_tensor * build_qwen35moe_ffn( ctx, ggml_mul_mat(ctx, L.ffn_gate_shexp, cur), L.ffn_gate_shexp_s); ggml_tensor * sh_up = apply_scale2( ctx, ggml_mul_mat(ctx, L.ffn_up_shexp, cur), L.ffn_up_shexp_s); - ggml_tensor * sh_gu = ggml_swiglu_split(ctx, sh_gate, sh_up); + ggml_tensor * sh_gu = nullptr; + if (L.ffn_swiglu_clamp_shexp > 0.0f) { + const float limit = L.ffn_swiglu_clamp_shexp; + sh_up = ggml_clamp(ctx, sh_up, -limit, limit); + sh_gate = ggml_clamp( + ctx, ggml_silu(ctx, sh_gate), -INFINITY, limit); + sh_gu = ggml_mul(ctx, sh_gate, sh_up); + } else { + sh_gu = ggml_swiglu_split(ctx, sh_gate, sh_up); + } ggml_tensor * shared = apply_scale2( ctx, ggml_mul_mat(ctx, L.ffn_down_shexp, sh_gu), L.ffn_down_shexp_s); diff --git a/server/src/server/chat_template.cpp b/server/src/server/chat_template.cpp index b9cf36eff..753574946 100644 --- a/server/src/server/chat_template.cpp +++ b/server/src/server/chat_template.cpp @@ -67,6 +67,7 @@ ChatFormat chat_format_for_arch(const std::string & arch) { if (arch == "deepseek4") return ChatFormat::DEEPSEEK4; if (arch == "laguna") return ChatFormat::LAGUNA; if (arch == "gemma4") return ChatFormat::GEMMA4; + if (arch == "bailingmoe3") return ChatFormat::BAILINGMOE3; // qwen35, qwen3 use the Qwen3/ChatML format return ChatFormat::QWEN3; } @@ -83,6 +84,109 @@ std::string render_chat_template( bool has_tools = !tools_json.empty() && tools_json != "[]" && tools_json != "null"; switch (format) { + case ChatFormat::BAILINGMOE3: { + // AntLing's shipped Bailing V3 template. The system turn is always + // present because it carries the model's thinking-mode directive. + const bool has_system = + !messages.empty() && messages[0].role == "system"; + const std::string system_content = has_system + ? messages[0].content : std::string(); + const char * thinking_option = enable_thinking ? "on" : "off"; + const std::string thinking_directive = + std::string("detailed thinking ") + thinking_option; + const bool system_sets_requested_thinking = + system_content.find(thinking_directive) != std::string::npos; + + result += "SYSTEM"; + if (has_tools) { + if (!system_content.empty()) { + result += system_content; + result += '\n'; + } + result += + "# Tools\n\n" + "You may call one or more functions to assist with the user query.\n\n" + "You are provided with function signatures within XML tags:\n" + ""; + try { + const nlohmann::json tools = nlohmann::json::parse(tools_json); + for (const auto & tool : tools) { + result += '\n'; + result += tool.dump(); + } + } catch (const std::exception &) { + result += '\n'; + result += tools_json; + } + result += + "\n\n\n" + "If none of the functions can be used, point it out. If the given question lacks the parameters required by the function, also point it out.\n" + "If you need to use a function, for each function call, output the function name and arguments within the following XML format:\n" + "{function-name}\n" + "{arg-key-1}\n" + "{arg-value-1}\n" + "{arg-key-2}\n" + "{arg-value-2}\n" + "...\n" + "\n"; + if (!system_sets_requested_thinking) { + result += thinking_directive; + } + result += "<|role_end|>"; + } else if (has_system) { + result += system_content; + if (!system_sets_requested_thinking) { + result += '\n'; + result += thinking_directive; + } + result += "<|role_end|>"; + } else { + result += thinking_directive; + result += "<|role_end|>"; + } + + bool in_tool_response = false; + for (size_t i = has_system ? 1 : 0; i < messages.size(); ++i) { + const ChatMessage & msg = messages[i]; + if (msg.role == "user") { + result += "HUMAN"; + result += msg.content; + result += "<|role_end|>"; + } else if (msg.role == "system") { + result += "SYSTEM"; + result += msg.content; + result += "<|role_end|>"; + } else if (msg.role == "assistant") { + result += "ASSISTANT\n"; + if (msg.content.find("") == std::string::npos) { + result += ""; + } + result += msg.content; + result += "<|role_end|>"; + } else if (msg.role == "tool") { + if (!in_tool_response) { + result += "OBSERVATION"; + in_tool_response = true; + } + result += "\n\n"; + result += msg.content; + result += "\n"; + const bool next_is_tool = + i + 1 < messages.size() && messages[i + 1].role == "tool"; + if (!next_is_tool) { + result += "<|role_end|>"; + in_tool_response = false; + } + } + } + + if (add_generation_prompt) { + result += "ASSISTANT\n"; + if (!enable_thinking) result += ""; + } + break; + } + case ChatFormat::QWEN3: { // Qwen3/3.5 ChatML format: // <|im_start|>system\n[tool preamble +] content<|im_end|>\n diff --git a/server/src/server/chat_template.h b/server/src/server/chat_template.h index f93119906..c3f1f534b 100644 --- a/server/src/server/chat_template.h +++ b/server/src/server/chat_template.h @@ -3,7 +3,8 @@ // Renders chat messages (system/user/assistant/tool) into the model-specific // token format. Hard-coded for supported architectures: // - Qwen3/3.5: <|im_start|>role\ncontent<|im_end|>\n -// - Laguna: XML-style <|begin_of_sentence|><|User|>...<|Assistant|> +// - BailingMoE3: SYSTEM/HUMAN/ASSISTANT...<|role_end|> +// - Laguna: XML-style role blocks #pragma once @@ -23,6 +24,7 @@ struct ChatMessage { // Chat template format. enum class ChatFormat { QWEN3, // <|im_start|>role\n...<|im_end|>\n + BAILINGMOE3, // SYSTEM/HUMAN/ASSISTANT...<|role_end|> LAGUNA, // <|begin_of_sentence|><|User|>...<|Assistant|> GEMMA4, // <|turn>role\n...\n DEEPSEEK4, // <|begin▁of▁sentence|>...<|User|>...<|Assistant|> diff --git a/server/src/server/http_server.cpp b/server/src/server/http_server.cpp index a909344e8..a5a8032ae 100644 --- a/server/src/server/http_server.cpp +++ b/server/src/server/http_server.cpp @@ -2221,12 +2221,16 @@ enum class TokenDelivery { // MUST classify identically, or the reasoning/content split diverges // between streamed and non-streamed responses. TokenDelivery classify_generated_token( - Tokenizer & tokenizer, int32_t token, std::string & text) { + Tokenizer & tokenizer, SseEmitter & emitter, + int32_t token, std::string & text) { if (token == tokenizer.eos_id() || token == tokenizer.eos_chat_id()) { return TokenDelivery::kSkip; } const std::string & raw = tokenizer.raw_token(token); + if (emitter.suppress_undeclared_tool_protocol_token(raw)) { + return TokenDelivery::kSkip; + } // Gemma4 thinking channel (<|channel> / ) and Qwen3.6 // thinking markers share one mapped dialect. The Qwen markers @@ -2244,6 +2248,18 @@ TokenDelivery classify_generated_token( return TokenDelivery::kThinkTag; } + // Bailing V3 stores its tool-call XML delimiters as added special + // tokens. They are part of the model's public output protocol, not chat + // control markers: keep them so the shared parser can reconstruct an + // OpenAI `tool_calls` object. Stripping them leaves only three unrelated + // text lines (function name, argument name, value). + if (raw == "" || raw == "" || + raw == "" || raw == "" || + raw == "" || raw == "") { + text = raw; + return TokenDelivery::kText; + } + // Other special tokens are internal control markers. Byte-fallback // tokens such as <0xAB> are text and must still reach the emitter. if (raw.size() >= 2 && raw[0] == '<' && raw[1] == '|') { @@ -2264,7 +2280,7 @@ CompletionTokenCounts feed_non_streaming_tokens( for (int32_t token : tokens) { std::string text; const TokenDelivery delivery = - classify_generated_token(tokenizer, token, text); + classify_generated_token(tokenizer, emitter, token, text); if (delivery == TokenDelivery::kSkip) continue; emitter.emit_token(text); @@ -3806,7 +3822,7 @@ void HttpServer::configure_generation_io( std::string text; const TokenDelivery delivery = - classify_generated_token(tokenizer_, token, text); + classify_generated_token(tokenizer_, emitter, token, text); if (delivery == TokenDelivery::kSkip) return true; if (!text.empty()) { @@ -3835,7 +3851,7 @@ bool HttpServer::deliver_generation_token( std::string text; const TokenDelivery delivery = - classify_generated_token(tokenizer_, token, text); + classify_generated_token(tokenizer_, emitter, token, text); if (delivery == TokenDelivery::kSkip) return true; // Non-stream replay counts every non-skipped token, including tokens diff --git a/server/src/server/model_card.cpp b/server/src/server/model_card.cpp index 522963572..6c272c36f 100644 --- a/server/src/server/model_card.cpp +++ b/server/src/server/model_card.cpp @@ -278,6 +278,20 @@ static bool family_fallback(const std::string & arch, ModelCard & out) { out.source_label = "family:" + arch; return true; } + if (arch == "bailingmoe3") { + out.max_tokens = 32768; + out.complex_problem_max_tokens = 0; + out.hard_limit_reply_budget = 4096; + // Official Ling 3.0 Flash sampling defaults. + out.sampling.temperature = 0.6f; + out.sampling.top_p = 0.95f; + out.sampling.top_k = 20; + out.sampling.has_temperature = true; + out.sampling.has_top_p = true; + out.sampling.has_top_k = true; + out.source_label = "family:bailingmoe3"; + return true; + } if (arch == "gemma4") { // Gemma4 verified value: see Gemma model card; conservative // 16384 keeps us inside published recommendations. diff --git a/server/src/server/sse_emitter.cpp b/server/src/server/sse_emitter.cpp index 6c99ced7e..040a5fea1 100644 --- a/server/src/server/sse_emitter.cpp +++ b/server/src/server/sse_emitter.cpp @@ -115,6 +115,27 @@ SseEmitter::SseEmitter(ApiFormat format, } } +bool SseEmitter::suppress_undeclared_tool_protocol_token( + const std::string & raw_token) { + if (has_request_tools(tools_)) return false; + + if (raw_token == "") { + suppress_undeclared_tool_protocol_ = true; + return true; + } + if (suppress_undeclared_tool_protocol_) { + if (raw_token == "") { + suppress_undeclared_tool_protocol_ = false; + } + return true; + } + + // Stray protocol delimiters are control tokens, not assistant content. + return raw_token == "" || + raw_token == "" || raw_token == "" || + raw_token == "" || raw_token == ""; +} + // ─── SSE formatting helpers ───────────────────────────────────────────── std::string SseEmitter::sse_data(const std::string & json_str) { diff --git a/server/src/server/sse_emitter.h b/server/src/server/sse_emitter.h index ed2d6fca8..f22ff283c 100644 --- a/server/src/server/sse_emitter.h +++ b/server/src/server/sse_emitter.h @@ -139,6 +139,12 @@ class SseEmitter { // counter; the difference is the natural-close content suffix. int emit_token_count() const { return emit_token_count_; } + // Discard a model-emitted Bailing tool block when the request did not + // declare tools. Returns true for every token from through + // , including the delimiters. + bool suppress_undeclared_tool_protocol_token( + const std::string & raw_token); + private: // Format helpers std::string format_openai_delta(const json & delta, const char * finish = nullptr); @@ -164,6 +170,7 @@ class SseEmitter { StreamMode mode_; bool tool_from_reasoning_ = false; + bool suppress_undeclared_tool_protocol_ = false; std::string window_; // holdback buffer // Incomplete trailing UTF-8 bytes from the previous token piece. // BPE tokens can split a multi-byte codepoint (emoji arrive as two diff --git a/server/test/test_chain_rollback_policy.cpp b/server/test/test_chain_rollback_policy.cpp index 7a17d6aff..c1a558ac0 100644 --- a/server/test/test_chain_rollback_policy.cpp +++ b/server/test/test_chain_rollback_policy.cpp @@ -166,6 +166,50 @@ TEST_CASE(ChainRollbackPolicyFixture, split_checkpoint_dtype_is_gated_at_allocat ggml_backend_free(backend); } +TEST_CASE(ChainRollbackPolicyFixture, asymmetric_v_cache_is_ling_only) { + const luce_test::ScopedEnvVar kv_f16("DFLASH27B_KV_F16", "1"); + const luce_test::ScopedEnvVar kv_q4("DFLASH27B_KV_Q4", nullptr); + const luce_test::ScopedEnvVar kv_tq3("DFLASH27B_KV_TQ3", nullptr); + const luce_test::ScopedEnvVar kv_k("DFLASH27B_KV_K", nullptr); + const luce_test::ScopedEnvVar kv_v("DFLASH27B_KV_V", nullptr); + + ggml_backend_t backend = ggml_backend_cpu_init(); + CHECK(backend != nullptr); + if (!backend) return; + + dflash::common::TargetWeights weights; + weights.n_layer = 1; + weights.full_attention_interval = 1; + weights.n_embd_head_k = 32; + weights.n_embd_head_v = 24; + weights.n_head = 1; + weights.n_head_kv = 1; + weights.n_embd = 32; + weights.n_capture_layers = 0; + + const auto check_width = [&](bool is_bailingmoe3, int expected) { + weights.is_bailingmoe3 = is_bailingmoe3; + dflash::common::TargetCache cache; + // This fixture has no recurrent layers, so no rollback cache is needed. + const bool ok = dflash::common::create_target_cache_partial( + weights, /*max_ctx=*/1, /*max_verify_tokens=*/1, backend, cache, + /*prefill_only=*/true, /*layer_begin=*/0, /*layer_end=*/1, + /*allocate_target_feat=*/false, /*ctx_alloc=*/0, + /*f32_ssm_intermediates=*/false); + CHECK(ok); + if (ok) { + CHECK(cache.attn_v.size() == 1); + CHECK(cache.attn_v[0] != nullptr); + if (cache.attn_v[0]) CHECK(cache.attn_v[0]->ne[0] == expected); + } + dflash::common::free_target_cache(cache); + }; + + check_width(/*is_bailingmoe3=*/false, /*expected=*/32); + check_width(/*is_bailingmoe3=*/true, /*expected=*/24); + ggml_backend_free(backend); +} + TEST_CASE(ChainRollbackPolicyFixture, diagnostics_accumulator_and_print_contract) { const luce_test::ScopedEnvVar checkpoint("DFLASH_SINGLE_CHAIN_CHECKPOINT_F32", nullptr); const luce_test::ScopedEnvVar threshold("DFLASH_FAST_ROLLBACK_THRESHOLD", nullptr); diff --git a/server/test/test_server_unit.cpp b/server/test/test_server_unit.cpp index 698b865be..94710665e 100644 --- a/server/test/test_server_unit.cpp +++ b/server/test/test_server_unit.cpp @@ -37,6 +37,7 @@ #include "common/gguf_bounds.h" #include "common/gguf_inspect.h" #include "qwen35/prefill_helpers.h" +#include "qwen35moe/qwen35moe_ffn.h" #include "ggml-cpu.h" #include "server/prompt_normalize.h" #include "qwen3_drafter_model.h" @@ -53,6 +54,7 @@ #include #include #include +#include #include #include #include @@ -3401,6 +3403,143 @@ TEST_CASE(ServerUnitFixture, test_deepseek4_render_reasoning_effort_prefixes) { TEST_ASSERT(!ends_with(completed_turn, "<|Assistant|>")); } +TEST_CASE(ServerUnitFixture, test_bailingmoe3_render_official_role_format) { + TEST_ASSERT(chat_format_for_arch("bailingmoe3") == ChatFormat::BAILINGMOE3); + const std::vector msgs = {{"user", "Hello", ""}}; + const std::string out = render_chat_template( + msgs, ChatFormat::BAILINGMOE3, + /*add_generation_prompt=*/true, + /*enable_thinking=*/false, + /*tools_json=*/""); + TEST_ASSERT(out == + "SYSTEMdetailed thinking off<|role_end|>" + "HUMANHello<|role_end|>" + "ASSISTANT\n"); +} + +TEST_CASE(ServerUnitFixture, test_bailingmoe3_render_thinking_and_tools) { + const std::vector msgs = { + {"system", "Be concise.", ""}, + {"user", "Check Rome", ""}, + }; + const std::string tools = + R"([{"type":"function","function":{"name":"weather","parameters":{"type":"object"}}}])"; + const std::string out = render_chat_template( + msgs, ChatFormat::BAILINGMOE3, + /*add_generation_prompt=*/true, + /*enable_thinking=*/true, + tools); + TEST_ASSERT(out.find("SYSTEMBe concise.\n# Tools") == 0); + TEST_ASSERT(out.find("\n{\"function\":") != std::string::npos); + TEST_ASSERT(out.find("detailed thinking on<|role_end|>") != std::string::npos); + TEST_ASSERT(out.find("HUMANCheck Rome<|role_end|>") != std::string::npos); + const std::string suffix = "ASSISTANT\n"; + TEST_ASSERT(out.size() >= suffix.size()); + TEST_ASSERT(out.compare(out.size() - suffix.size(), suffix.size(), suffix) == 0); +} + +TEST_CASE(ServerUnitFixture, test_bailingmoe3_request_overrides_system_thinking) { + const std::vector thinking_on = { + {"system", "Keep this note. detailed thinking on", ""}, + {"user", "Hello", ""}, + }; + const std::string disabled = render_chat_template( + thinking_on, ChatFormat::BAILINGMOE3, + /*add_generation_prompt=*/true, + /*enable_thinking=*/false, + /*tools_json=*/""); + const size_t prior_on = disabled.find("detailed thinking on"); + const size_t requested_off = disabled.rfind("detailed thinking off"); + TEST_ASSERT(prior_on != std::string::npos); + TEST_ASSERT(requested_off != std::string::npos && requested_off > prior_on); + + const std::vector thinking_off = { + {"system", "Keep this note. detailed thinking off", ""}, + {"user", "Hello", ""}, + }; + const std::string tools = + R"([{"type":"function","function":{"name":"weather"}}])"; + const std::string enabled = render_chat_template( + thinking_off, ChatFormat::BAILINGMOE3, + /*add_generation_prompt=*/true, + /*enable_thinking=*/true, + tools); + const size_t prior_off = enabled.find("detailed thinking off"); + const size_t requested_on = enabled.rfind("detailed thinking on"); + TEST_ASSERT(prior_off != std::string::npos); + TEST_ASSERT(requested_on != std::string::npos && requested_on > prior_off); +} + +TEST_CASE(ServerUnitFixture, test_emitter_suppresses_undeclared_bailing_tool_block) { + auto em = make_emitter(ApiFormat::OPENAI_CHAT); + em.emit_start(); + + const auto deliver = [&](const std::string & raw_token) { + if (!em.suppress_undeclared_tool_protocol_token(raw_token)) { + em.emit_token(raw_token); + } + }; + deliver(""); + deliver("weather"); + deliver(""); + deliver("city"); + deliver(""); + deliver(""); + deliver("Rome"); + deliver(""); + deliver(""); + deliver("Visible answer"); + em.emit_finish(10); + + TEST_ASSERT(em.accumulated_text() == "Visible answer"); + TEST_ASSERT(em.accumulated_text().find("weather") == std::string::npos); + TEST_ASSERT(em.accumulated_text().find("Rome") == std::string::npos); +} + +TEST_CASE(ServerUnitFixture, test_bailingmoe3_router_builds_group_mask) { + ggml_init_params params{}; + params.mem_size = 2 * 1024 * 1024; + params.no_alloc = true; + ggml_context * ctx = ggml_init(params); + TEST_ASSERT(ctx != nullptr); + + TargetWeights weights; + weights.n_expert = 512; + weights.n_expert_used = 8; + weights.n_expert_groups = 8; + weights.n_expert_groups_used = 4; + weights.expert_gating_func = 2; + weights.expert_weights_norm = true; + weights.expert_weights_scale = 2.5f; + + TargetLayer layer; + layer.ffn_gate_inp = ggml_new_tensor_2d( + ctx, GGML_TYPE_F32, 16, weights.n_expert); + layer.ffn_exp_probs_b = ggml_new_tensor_1d( + ctx, GGML_TYPE_F32, weights.n_expert); + ggml_tensor * input = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 16, 2); + + const Qwen35MoeRouterOutputs router = build_qwen35moe_router( + ctx, input, weights, layer); + TEST_ASSERT(router.selected != nullptr); + TEST_ASSERT(router.weights != nullptr); + + std::unordered_set visited; + const auto contains_op = [&](const auto & self, + const ggml_tensor * tensor, + ggml_op op) -> bool { + if (!tensor || !visited.insert(tensor).second) return false; + if (tensor->op == op) return true; + for (const ggml_tensor * source : tensor->src) { + if (self(self, source, op)) return true; + } + return false; + }; + TEST_ASSERT(contains_op(contains_op, router.selected, GGML_OP_SET_ROWS)); + + ggml_free(ctx); +} + TEST_CASE(ServerUnitFixture, test_jinja_render_basic) { std::vector msgs = { {"system", "you are helpful", ""}, @@ -5397,6 +5536,18 @@ TEST_CASE(ServerUnitFixture, test_model_card_family_fallback_deepseek4) { TEST_ASSERT(unknown.source_label != "family:not-a-real-arch"); } +TEST_CASE(ServerUnitFixture, test_model_card_family_fallback_bailingmoe3) { + auto card = dflash::common::resolve_model_card("", "", "bailingmoe3", ""); + TEST_ASSERT(card.source_label == "family:bailingmoe3"); + TEST_ASSERT(card.max_tokens == 32768); + TEST_ASSERT(card.sampling.has_temperature); + TEST_ASSERT(std::abs(card.sampling.temperature - 0.6f) < 1.0e-6f); + TEST_ASSERT(card.sampling.has_top_p); + TEST_ASSERT(std::abs(card.sampling.top_p - 0.95f) < 1.0e-6f); + TEST_ASSERT(card.sampling.has_top_k); + TEST_ASSERT(card.sampling.top_k == 20); +} + TEST_CASE(ServerUnitFixture, test_props_model_card_wholesale_sidecar) { // When a sidecar was loaded, /props.model_card should be the parsed // sidecar JSON verbatim — *all* fields from the file, not just the @@ -6407,6 +6558,10 @@ TEST_CASE(ServerUnitFixture, test_qwen35_embedded_mtp_target_layer_count) { "qwen35moe", 81, 1, target_layers, error)); TEST_ASSERT(target_layers == 80); + TEST_ASSERT(derive_effective_target_layer_count( + "bailingmoe3", 43, 1, target_layers, error)); + TEST_ASSERT(target_layers == 42); + TEST_ASSERT(!derive_effective_target_layer_count( "qwen35", 1, 1, target_layers, error)); TEST_ASSERT(error.find("smaller than block_count") != std::string::npos);