From 3b46593e652b803521d713237691b5bb84a3bada Mon Sep 17 00:00:00 2001 From: mrciffa <49000955+davide221@users.noreply.github.com> Date: Mon, 24 Aug 2026 19:31:44 +0200 Subject: [PATCH 1/5] feat(server): add native Ling 3 Flash backend --- README.md | 8 + server/CMakeLists.txt | 3 + server/docs/LING3_FLASH_DGX_SPARK.md | 202 +++++++ server/scripts/bench_ling3_flash.py | 291 ++++++++++ .../src/bailingmoe3/bailingmoe3_backend.cpp | 47 ++ server/src/bailingmoe3/bailingmoe3_backend.h | 28 + server/src/bailingmoe3/bailingmoe3_loader.cpp | 497 ++++++++++++++++++ server/src/common/backend_factory.cpp | 15 + server/src/common/gguf_inspect.cpp | 7 +- server/src/common/model_capabilities.h | 9 +- server/src/internal.h | 44 ++ server/src/qwen35/qwen35_backend.cpp | 13 +- server/src/qwen35/qwen35_target_graph.cpp | 428 ++++++++++++--- server/src/qwen35moe/qwen35moe_ffn.cpp | 90 +++- server/src/server/chat_template.cpp | 106 ++++ server/src/server/chat_template.h | 4 +- server/src/server/http_server.cpp | 12 + server/src/server/model_card.cpp | 14 + server/test/test_server_unit.cpp | 97 ++++ 19 files changed, 1816 insertions(+), 99 deletions(-) create mode 100644 server/docs/LING3_FLASH_DGX_SPARK.md create mode 100644 server/scripts/bench_ling3_flash.py create mode 100644 server/src/bailingmoe3/bailingmoe3_backend.cpp create mode 100644 server/src/bailingmoe3/bailingmoe3_backend.h create mode 100644 server/src/bailingmoe3/bailingmoe3_loader.cpp diff --git a/README.md b/README.md index 842800967..f6aa075cc 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,14 @@ Model links open the exact weights used by the measured setup. Drafter links ope | [Gemma 4 31B IT Q4_K_M](https://huggingface.co/bartowski/google_gemma-4-31B-it-GGUF/blob/main/google_gemma-4-31B-it-Q4_K_M.gguf) + [DFlash Q8_0 drafter](https://huggingface.co/Lucebox/gemma-4-31B-it-DFlash-GGUF/blob/main/gemma-4-31B-it-DFlash-q8_0.gguf) | Decode | **3.2×** | | [DeepSeek V4 Flash ROCmFPX MIX Strix](https://huggingface.co/Lucebox/DeepSeek-V4-Flash-0731-ROCmFP3/blob/main/DeepSeek-V4-Flash-0731-ROCMFPX-MIX-STRIX.gguf) + [DSpark Q4RMFP4 drafter](https://huggingface.co/Lucebox/DeepSeek-V4-Flash-0731-DSpark-GGUF/blob/main/DeepSeek-V4-Flash-0731-DSpark-draft-Q4RMFP4-denseF16.gguf) | Decode | Up to **1.81×** vs target-only, **32.7 vs 18.1 tok/s** | +Experimental native `BailingMoE3` support is available for +[`Ling 3.0 Flash`](https://huggingface.co/inclusionAI/Ling-3.0-flash). On a +single DGX Spark, LuceBox serves the Q4_K_M GGUF at **34.6 tok/s** median +autoregressive decode with Q4_0 KV cache. See the +[DGX Spark integration and benchmark](server/docs/LING3_FLASH_DGX_SPARK.md) +for the exact configuration, context sweep, tool-call test, and current +limitations. + ## Tested Machines (GPU/APU) The engine is not tied to one reference card. NVIDIA architectures are selected by CMake; HIP builds should target the device's exact `gfx` architecture. diff --git a/server/CMakeLists.txt b/server/CMakeLists.txt index a20f168ab..5081a21be 100644 --- a/server/CMakeLists.txt +++ b/server/CMakeLists.txt @@ -400,6 +400,7 @@ set(DFLASH27B_SRC_INCLUDE_DIRS ${CMAKE_CURRENT_SOURCE_DIR}/src/draft ${CMAKE_CURRENT_SOURCE_DIR}/src/qwen35 ${CMAKE_CURRENT_SOURCE_DIR}/src/qwen35moe + ${CMAKE_CURRENT_SOURCE_DIR}/src/bailingmoe3 ${CMAKE_CURRENT_SOURCE_DIR}/src/laguna ${CMAKE_CURRENT_SOURCE_DIR}/src/qwen3 ${CMAKE_CURRENT_SOURCE_DIR}/src/gemma4 @@ -412,6 +413,8 @@ add_library(dflash_common STATIC src/qwen35/gguf_target_loader.cpp src/qwen35/qwen35_target_graph.cpp src/qwen35/qwen35_roctx.cpp + src/bailingmoe3/bailingmoe3_loader.cpp + src/bailingmoe3/bailingmoe3_backend.cpp src/draft/draft_gguf_loader.cpp src/draft/draft_safetensors_loader.cpp src/draft/draft_graph.cpp diff --git a/server/docs/LING3_FLASH_DGX_SPARK.md b/server/docs/LING3_FLASH_DGX_SPARK.md new file mode 100644 index 000000000..1ceff425a --- /dev/null +++ b/server/docs/LING3_FLASH_DGX_SPARK.md @@ -0,0 +1,202 @@ +# Ling 3.0 Flash on DGX Spark + +LuceBox has an experimental native backend for AntLing's 124B-parameter +`BailingMoE3` model, Ling 3.0 Flash. This is a LuceBox graph implementation, +not a proxy to vLLM or llama.cpp. + +The first tested configuration serves a Q4_K_M GGUF on one NVIDIA DGX Spark +at 34.6 generated tokens/s. It also renders the official chat template, +emits OpenAI-compatible tool calls, consumes tool results, and returns a final +answer. + +## Tested configuration + +| Item | Value | +|---|---| +| Machine | NVIDIA DGX Spark | +| GPU | NVIDIA GB10, `sm_121` | +| Host architecture | `aarch64` | +| Unified memory | 121 GiB visible to the OS | +| Driver / CUDA | 580.173.02 / CUDA 13.0 | +| Model | `inclusionAI/Ling-3.0-flash` | +| GGUF | `Ling-3.0-flash-Q4_K_M.gguf`, 72.9 GiB | +| Weight quantization | Q4_K_M | +| KV cache | Q4_0 keys and values | +| Server context | 32,768 tokens | +| Request concurrency | One active decode slot | +| Prefix cache | Disabled for measurement | +| Benchmark date | 2026-08-24 | + +The GGUF used for this bring-up came from +[`bloomer010/Ling-3.0-flash-GGUF`](https://huggingface.co/bloomer010/Ling-3.0-flash-GGUF). +The official model and tokenizer are at +[`inclusionAI/Ling-3.0-flash`](https://huggingface.co/inclusionAI/Ling-3.0-flash). + +## Build and serve + +Download the tested quantization: + +```bash +hf download bloomer010/Ling-3.0-flash-GGUF \ + Ling-3.0-flash-Q4_K_M.gguf \ + --local-dir models/ling-3.0-flash-gguf +``` + +Build the CUDA server for GB10: + +```bash +cmake -S server -B server/build-ling3-cuda \ + -DCMAKE_BUILD_TYPE=Release \ + -DDFLASH27B_GPU_BACKEND=cuda \ + -DGGML_NATIVE=OFF \ + -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc \ + -DCMAKE_CUDA_ARCHITECTURES=121 + +cmake --build server/build-ling3-cuda \ + --target dflash_server -j"$(nproc)" +``` + +Start the measured configuration: + +```bash +server/build-ling3-cuda/dflash_server \ + models/ling-3.0-flash-gguf/Ling-3.0-flash-Q4_K_M.gguf \ + --host 127.0.0.1 \ + --port 18081 \ + --max-ctx 32768 \ + --default-max-tokens 256 \ + --cache-type-k q4_0 \ + --cache-type-v q4_0 \ + --prefix-cache-slots 0 \ + --disk-prefix-cache off \ + --model-name ling-3.0-flash-lucebox +``` + +The server exposes the OpenAI-compatible `/v1/chat/completions` route. To use +a Spark reached over SSH without exposing the API on the network, create a +local tunnel: + +```bash +ssh -N -L 18081:127.0.0.1:18081 @ +``` + +Then send requests to `http://127.0.0.1:18081` on the client machine. + +## Benchmark method + +The benchmark uses [`bench_ling3_flash.py`](../scripts/bench_ling3_flash.py), +which depends only on the Python standard library. It records the timings +reported by the LuceBox server and checks output determinism. Prefix caching is +disabled, temperature is zero, and concurrency is reported separately from +single-request decode. + +```bash +python3 server/scripts/bench_ling3_flash.py \ + --url http://127.0.0.1:18081 \ + --model ling-3.0-flash-lucebox \ + --weights-quant Q4_K_M \ + --kv-quant Q4_0 \ + --server-max-context 32768 \ + --decode-tokens 128 \ + --decode-runs 3 \ + --warmups 1 \ + --context-tokens 256,1024,4096,8192 \ + --context-runs 2 \ + --concurrency-levels 1,2 \ + --concurrency-tokens 64 \ + --output ling3-flash-lucebox-bench.json +``` + +### Decode + +The decode prompt requests a deterministic repeated-token sequence and each +measured run generates 128 tokens. All three measured responses have the same +SHA-256 digest. + +| Run | Prompt tokens | Output tokens | Prefill | Decode | +|---:|---:|---:|---:|---:| +| 1 | 57 | 128 | 157.8 ms | 34.6 tok/s | +| 2 | 57 | 128 | 157.4 ms | 34.6 tok/s | +| 3 | 57 | 128 | 157.2 ms | 34.5 tok/s | +| **Median** | **57** | **128** | **157.4 ms** | **34.6 tok/s** | + +### Context prefill + +Each row is the median of two uncached requests. The response is restricted to +one token so the measurement isolates prompt evaluation. + +| Actual prompt tokens | Median prefill | Median prefill rate | +|---:|---:|---:| +| 267 | 277.55 ms | 962.0 tok/s | +| 1,035 | 810.80 ms | 1,277.0 tok/s | +| 4,107 | 2,642.15 ms | 1,554.4 tok/s | +| 8,203 | 5,238.60 ms | 1,565.9 tok/s | + +The model advertises a 262,144-token context. This run capped the server at +32,768 and exercised prompt depth only through 8,203 tokens, so it is not a +claim of validated 262K behavior. + +### Concurrency + +| Submitted requests | Total output tokens | Wall time | Aggregate rate | +|---:|---:|---:|---:| +| 1 | 64 | 2.041 s | 31.35 tok/s | +| 2 | 128 | 4.033 s | 31.74 tok/s | + +Two simultaneous HTTP requests complete safely, but the current LuceBox +backend queues them through one active decode slot. The second request took +4.032 seconds, versus 2.016 seconds for the first. This is serialization, not +continuous batching, and throughput does not scale with request concurrency. + +### Tool-call round trip + +The test gives the model a `get_weather(location)` function and asks for the +current weather in Rome. The model emits exactly one structured call: + +```json +{"name":"get_weather","arguments":{"location":"Rome"}} +``` + +The harness supplies a deterministic fixture: + +```json +{"temperature_c":29,"condition":"sunny","source":"test fixture"} +``` + +Ling then returns: + +> The current weather in Rome is sunny with a temperature of 29°C. + +The call completed with `finish_reason: "tool_calls"`; the final answer used +both fixture values. Decode measured 35.0 tok/s for the tool call and 34.9 +tok/s for the final answer. + +## Correctness checks + +The backend implements Ling's mixed 35-layer KDA / 7-layer MLA attention, +512-expert MoE, top-8 routing, and grouped expert selection. The embedded MTP +layer is recognized and excluded from the autoregressive target graph. + +For an oracle check, the same GGUF and exact chat prompt were run against the +Ling support branch proposed to llama.cpp. LuceBox and the oracle match the +opening text through `Running an open-weight language model locally offers`. +Their first different greedy token was a near tie: oracle `greater` at +-0.89073 log probability and LuceBox `three`, with oracle `three` at -0.96373. +That 0.073-natural-log-unit margin is consistent with GPU kernel numerical +drift rather than a prompt-template or graph-structure mismatch. A short exact +answer test produced `PARIS` in both engines. + +The full LuceBox CTest suite passes: 477 tests, zero failures. Four tests that +require unavailable model assets or NCCL are skipped by configuration. + +## Current verdict + +Use this backend today for single-user, local Ling 3.0 Flash evaluation on a +DGX Spark, especially chat, tool-use, and agent experiments that need an +OpenAI-compatible endpoint. The 73 GiB Q4_K_M artifact fits comfortably in the +Spark's unified memory, and 34.6 tok/s is interactive. + +Do not use this baseline yet for a multi-user serving claim, a validated 262K +context claim, or a headline comparison against another runtime. Continuous +batching, full-depth context validation, the model's MTP speculative path, and +matched-condition cross-runtime benchmarks remain follow-up work. diff --git a/server/scripts/bench_ling3_flash.py b/server/scripts/bench_ling3_flash.py new file mode 100644 index 000000000..8f19fb971 --- /dev/null +++ b/server/scripts/bench_ling3_flash.py @@ -0,0 +1,291 @@ +#!/usr/bin/env python3 +"""Reproducible Ling 3.0 Flash checks against a running dflash_server. + +The script uses only the Python standard library. It records server-reported +prefill/decode timings, a short context-depth sweep, request-level concurrency, +and a real OpenAI-style tool-call round trip. +""" + +from __future__ import annotations + +import argparse +import concurrent.futures +import hashlib +import json +import statistics +import time +import urllib.error +import urllib.request +from pathlib import Path + + +DECODE_PROMPT = ( + "Continue this exact sequence until the output limit. Output only the word " + "BETA separated by single spaces: BETA BETA BETA BETA BETA BETA BETA BETA" +) + + +def post_json(url: str, payload: dict, timeout: float) -> dict: + request = urllib.request.Request( + url, + data=json.dumps(payload, separators=(",", ":")).encode(), + headers={"Content-Type": "application/json"}, + ) + with urllib.request.urlopen(request, timeout=timeout) as response: + return json.load(response) + + +def chat(base_url: str, model: str, messages: list[dict], max_tokens: int, + timeout: float, *, tools: list[dict] | None = None) -> dict: + payload: dict = { + "model": model, + "messages": messages, + "temperature": 0, + "max_tokens": max_tokens, + "stream": False, + } + if tools is not None: + payload["tools"] = tools + started = time.monotonic() + result = post_json( + f"{base_url.rstrip('/')}/v1/chat/completions", payload, timeout) + result["_wall_seconds"] = time.monotonic() - started + return result + + +def measurement(result: dict) -> dict: + usage = result["usage"] + timings = usage.get("timings", {}) + content = result["choices"][0]["message"].get("content") or "" + return { + "prompt_tokens": usage["prompt_tokens"], + "completion_tokens": usage["completion_tokens"], + "prefill_ms": timings.get("prefill_ms"), + "prefill_tokens": timings.get( + "effective_prompt_tokens", usage["prompt_tokens"]), + "decode_ms": timings.get("decode_ms"), + "decode_tokens_per_second": timings.get("decode_tokens_per_sec"), + "cache_hit": timings.get("cache_hit"), + "cached_prefix_tokens": timings.get("cached_prefix_tokens", 0), + "wall_seconds": result["_wall_seconds"], + "finish_reason": result["choices"][0].get("finish_reason"), + "output_sha256": hashlib.sha256(content.encode()).hexdigest(), + "output_excerpt": content[:160], + } + + +def median(values: list[float | int | None]) -> float | None: + present = [float(value) for value in values if value is not None] + return statistics.median(present) if present else None + + +def run_decode(args: argparse.Namespace) -> dict: + for _ in range(args.warmups): + chat(args.url, args.model, + [{"role": "user", "content": DECODE_PROMPT}], + args.decode_tokens, args.timeout) + + runs = [ + measurement(chat( + args.url, args.model, + [{"role": "user", "content": DECODE_PROMPT}], + args.decode_tokens, args.timeout)) + for _ in range(args.decode_runs) + ] + return { + "prompt": DECODE_PROMPT, + "temperature": 0, + "max_tokens": args.decode_tokens, + "warmups": args.warmups, + "runs": runs, + "median_decode_tokens_per_second": median( + [run["decode_tokens_per_second"] for run in runs]), + "median_prefill_ms": median([run["prefill_ms"] for run in runs]), + } + + +def synthetic_context(target_tokens: int, run: int) -> str: + # " alpha" is one token in the Ling tokenizer. The fixed chat framing and + # instruction account for the small difference between target and actual. + body = "alpha " * max(1, target_tokens - 32) + return ( + "Read this synthetic context, then reply with only OK.\n" + f"{body}\nBenchmark run {run}; reply OK." + ) + + +def run_context_sweep(args: argparse.Namespace) -> list[dict]: + rows = [] + for target in args.context_tokens: + runs = [] + for run_index in range(args.context_runs): + result = chat( + args.url, args.model, + [{"role": "user", + "content": synthetic_context(target, run_index)}], + 1, args.timeout) + row = measurement(result) + if row["prefill_ms"]: + row["prefill_tokens_per_second"] = ( + row["prefill_tokens"] / (row["prefill_ms"] / 1000.0)) + else: + row["prefill_tokens_per_second"] = None + runs.append(row) + rows.append({ + "target_tokens": target, + "runs": runs, + "actual_prompt_tokens": int(median( + [run["prompt_tokens"] for run in runs]) or 0), + "median_prefill_ms": median([run["prefill_ms"] for run in runs]), + "median_prefill_tokens_per_second": median( + [run["prefill_tokens_per_second"] for run in runs]), + }) + return rows + + +def run_concurrency(args: argparse.Namespace) -> list[dict]: + rows = [] + for level in args.concurrency_levels: + started = time.monotonic() + with concurrent.futures.ThreadPoolExecutor(max_workers=level) as pool: + futures = [ + pool.submit( + chat, args.url, args.model, + [{"role": "user", "content": + f"{DECODE_PROMPT}\nConcurrent request {index}."}], + args.concurrency_tokens, args.timeout) + for index in range(level) + ] + runs = [measurement(future.result()) for future in futures] + wall_seconds = time.monotonic() - started + completion_tokens = sum(run["completion_tokens"] for run in runs) + rows.append({ + "concurrency": level, + "max_tokens_per_request": args.concurrency_tokens, + "wall_seconds": wall_seconds, + "completion_tokens": completion_tokens, + "aggregate_completion_tokens_per_second": ( + completion_tokens / wall_seconds if wall_seconds else None), + "requests": runs, + }) + return rows + + +def run_tool_round_trip(args: argparse.Namespace) -> dict: + tools = [{ + "type": "function", + "function": { + "name": "get_weather", + "description": "Get current weather for a city.", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string"}, + }, + "required": ["location"], + "additionalProperties": False, + }, + }, + }] + user_message = { + "role": "user", + "content": "What is the current weather in Rome? Use the tool.", + } + first = chat( + args.url, args.model, [user_message], 96, args.timeout, tools=tools) + choice = first["choices"][0] + assistant = choice["message"] + calls = assistant.get("tool_calls") or [] + if choice.get("finish_reason") != "tool_calls" or len(calls) != 1: + raise RuntimeError(f"expected one tool call, got: {choice}") + function = calls[0]["function"] + arguments = json.loads(function["arguments"]) + if function["name"] != "get_weather" or "rome" not in ( + arguments.get("location") or "").lower(): + raise RuntimeError(f"unexpected tool call: {function}") + + tool_message = { + "role": "tool", + "tool_call_id": calls[0]["id"], + "content": json.dumps({ + "temperature_c": 29, + "condition": "sunny", + "source": "test fixture", + }, separators=(",", ":")), + } + second = chat( + args.url, args.model, + [user_message, assistant, tool_message], + 96, args.timeout, tools=tools) + answer = second["choices"][0]["message"].get("content") or "" + if "29" not in answer or "sunny" not in answer.lower(): + raise RuntimeError(f"tool result was not used in the answer: {answer}") + return { + "call_finish_reason": choice.get("finish_reason"), + "tool_name": function["name"], + "arguments": arguments, + "call_measurement": measurement(first), + "answer": answer, + "answer_measurement": measurement(second), + "passed": True, + } + + +def csv_ints(value: str) -> list[int]: + values = [int(item) for item in value.split(",") if item.strip()] + if not values or any(item <= 0 for item in values): + raise argparse.ArgumentTypeError("expected comma-separated positive integers") + return values + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--url", default="http://127.0.0.1:18081") + parser.add_argument("--model", default="ling-3.0-flash-lucebox") + parser.add_argument("--weights-quant", default="Q4_K_M") + parser.add_argument("--kv-quant", default="Q4_0") + parser.add_argument("--server-max-context", type=int, default=32768) + parser.add_argument("--decode-tokens", type=int, default=128) + parser.add_argument("--decode-runs", type=int, default=3) + parser.add_argument("--warmups", type=int, default=1) + parser.add_argument("--context-tokens", type=csv_ints, + default=csv_ints("256,1024,4096,8192")) + parser.add_argument("--context-runs", type=int, default=2) + parser.add_argument("--concurrency-levels", type=csv_ints, + default=csv_ints("1,2")) + parser.add_argument("--concurrency-tokens", type=int, default=64) + parser.add_argument("--timeout", type=float, default=900.0) + parser.add_argument("--output", type=Path) + args = parser.parse_args() + + try: + with urllib.request.urlopen( + f"{args.url.rstrip('/')}/health", timeout=10) as response: + health = response.read().decode() + report = { + "engine": "LuceBox dflash_server", + "model": args.model, + "weights_quant": args.weights_quant, + "kv_quant": args.kv_quant, + "server_max_context": args.server_max_context, + "health": health, + "decode": run_decode(args), + "context_sweep": run_context_sweep(args), + "concurrency": run_concurrency(args), + "tool_round_trip": run_tool_round_trip(args), + } + except (KeyError, TypeError, ValueError, RuntimeError, + urllib.error.URLError) as error: + print(f"benchmark failed: {error}") + return 1 + + rendered = json.dumps(report, indent=2, sort_keys=True) + print(rendered) + if args.output: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(rendered + "\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/server/src/bailingmoe3/bailingmoe3_backend.cpp b/server/src/bailingmoe3/bailingmoe3_backend.cpp new file mode 100644 index 000000000..feb7f0367 --- /dev/null +++ b/server/src/bailingmoe3/bailingmoe3_backend.cpp @@ -0,0 +1,47 @@ +#include "bailingmoe3_backend.h" + +#include + +namespace dflash::common { +namespace { + +Qwen35Config make_qwen_runtime_config(const BailingMoe3Config & cfg) { + Qwen35Config runtime; + runtime.target_path = cfg.model_path; + runtime.device = cfg.device; + runtime.stream_fd = cfg.stream_fd; + // The Ling baseline uses the ordinary contiguous F16/Q4 KV cache and the + // proven single-sequence AR loop. No DFlash draft or paged serving yet. + // Its compressed MLA head is 576-wide, whose CUDA kernel contract uses a + // 256-row K/V span and an explicit visibility mask even for decode. + runtime.kq_stride_pad = 256; + runtime.paged_attention = false; + runtime.max_concurrency = 1; + return runtime; +} + +} // namespace + +BailingMoe3Backend::BailingMoe3Backend(const BailingMoe3Config & cfg) + : Qwen35Backend(make_qwen_runtime_config(cfg)) {} + +bool BailingMoe3Backend::load_target_model(ggml_backend_t backend, + TargetWeights & out) { + return load_bailingmoe3_gguf(cfg_.target_path, backend, out); +} + +void BailingMoe3Backend::print_ready_banner() const { + const TargetWeights & weights = target_weights(); + std::printf( + "[bailingmoe3-daemon] ready layers=%d kda=%d mla=%d " + "experts=%d/%d groups=%d/%d ctx=%d\n", + weights.n_layer, + weights.n_layer - weights.n_layer / weights.full_attention_interval, + weights.n_layer / weights.full_attention_interval, + weights.n_expert_used, weights.n_expert, + weights.n_expert_groups_used, weights.n_expert_groups, + cfg_.device.max_ctx); + std::fflush(stdout); +} + +} // namespace dflash::common diff --git a/server/src/bailingmoe3/bailingmoe3_backend.h b/server/src/bailingmoe3/bailingmoe3_backend.h new file mode 100644 index 000000000..61e775753 --- /dev/null +++ b/server/src/bailingmoe3/bailingmoe3_backend.h @@ -0,0 +1,28 @@ +#pragma once + +#include "qwen35_backend.h" + +namespace dflash::common { + +// Configuration intentionally exposes only the features the first native +// Ling backend implements. Speculative decode and expert offload can be added +// after the autoregressive path has a logits-equivalent baseline. +struct BailingMoe3Config { + const char * model_path = nullptr; + DevicePlacement device; + int stream_fd = -1; +}; + +class BailingMoe3Backend final : public Qwen35Backend { +public: + explicit BailingMoe3Backend(const BailingMoe3Config & cfg); + + void print_ready_banner() const override; + bool supports_dflash_spec_decode() const override { return false; } + bool supports_remote_draft() const override { return false; } + +protected: + bool load_target_model(ggml_backend_t backend, TargetWeights & out) override; +}; + +} // namespace dflash::common diff --git a/server/src/bailingmoe3/bailingmoe3_loader.cpp b/server/src/bailingmoe3/bailingmoe3_loader.cpp new file mode 100644 index 000000000..bd5af37a4 --- /dev/null +++ b/server/src/bailingmoe3/bailingmoe3_loader.cpp @@ -0,0 +1,497 @@ +// Native GGUF loader for BailingMoE3 (Ling 3.x). +// +// Ling GGUFs contain one embedded NextN/MTP block after the 42-layer +// autoregressive trunk. This loader deliberately loads only the trunk; MTP is +// a decode optimization and is not required for a correctness-first backend. + +#include "internal.h" + +#include "common/gguf_bounds.h" +#include "common/gguf_mmap.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace dflash::common { +namespace { + +constexpr const char * kArch = "bailingmoe3"; + +uint32_t get_u32_or(const gguf_context * g, const std::string & key, + uint32_t fallback) { + const int64_t id = gguf_find_key(g, key.c_str()); + if (id < 0) return fallback; + if (gguf_get_kv_type(g, id) == GGUF_TYPE_ARRAY) { + if (gguf_get_arr_n(g, id) == 0) return fallback; + const gguf_type type = gguf_get_arr_type(g, id); + const void * data = gguf_get_arr_data(g, id); + if (type == GGUF_TYPE_UINT32) return static_cast(data)[0]; + if (type == GGUF_TYPE_INT32) { + const int32_t value = static_cast(data)[0]; + return value < 0 ? fallback : static_cast(value); + } + return fallback; + } + return gguf_get_val_u32(g, id); +} + +float get_f32_or(const gguf_context * g, const std::string & key, + float fallback) { + const int64_t id = gguf_find_key(g, key.c_str()); + if (id < 0) return fallback; + if (gguf_get_kv_type(g, id) == GGUF_TYPE_ARRAY) { + if (gguf_get_arr_n(g, id) == 0 || + gguf_get_arr_type(g, id) != GGUF_TYPE_FLOAT32) { + return fallback; + } + return static_cast(gguf_get_arr_data(g, id))[0]; + } + return gguf_get_val_f32(g, id); +} + +bool get_bool_or(const gguf_context * g, const std::string & key, + bool fallback) { + const int64_t id = gguf_find_key(g, key.c_str()); + if (id < 0 || gguf_get_kv_type(g, id) != GGUF_TYPE_BOOL) return fallback; + return gguf_get_val_bool(g, id); +} + +std::vector get_u32_array(const gguf_context * g, + const std::string & key) { + const int64_t id = gguf_find_key(g, key.c_str()); + if (id < 0 || gguf_get_kv_type(g, id) != GGUF_TYPE_ARRAY) return {}; + const gguf_type type = gguf_get_arr_type(g, id); + if (type != GGUF_TYPE_UINT32 && type != GGUF_TYPE_INT32) return {}; + const size_t n = gguf_get_arr_n(g, id); + const void * raw = gguf_get_arr_data(g, id); + std::vector result(n); + for (size_t i = 0; i < n; ++i) { + if (type == GGUF_TYPE_UINT32) { + result[i] = static_cast(raw)[i]; + } else { + const int32_t value = static_cast(raw)[i]; + if (value < 0) return {}; + result[i] = static_cast(value); + } + } + return result; +} + +std::vector get_f32_array(const gguf_context * g, + const std::string & key, + size_t count) { + std::vector result(count, 0.0f); + const int64_t id = gguf_find_key(g, key.c_str()); + if (id < 0) return result; + if (gguf_get_kv_type(g, id) == GGUF_TYPE_FLOAT32) { + std::fill(result.begin(), result.end(), gguf_get_val_f32(g, id)); + return result; + } + if (gguf_get_kv_type(g, id) != GGUF_TYPE_ARRAY || + gguf_get_arr_type(g, id) != GGUF_TYPE_FLOAT32) { + return result; + } + const size_t n = std::min(count, gguf_get_arr_n(g, id)); + const float * values = static_cast(gguf_get_arr_data(g, id)); + std::copy(values, values + n, result.begin()); + return result; +} + +size_t align_up(size_t value, size_t alignment) { + if (alignment == 0) return value; + const size_t remainder = value % alignment; + return remainder == 0 ? value : value + alignment - remainder; +} + +struct TensorAllocation { + ggml_tensor * tensor = nullptr; + size_t file_offset = 0; + size_t file_size = 0; + size_t buffer_offset = 0; +}; + +} // namespace + +bool load_bailingmoe3_gguf(const std::string & path, + ggml_backend_t backend, + TargetWeights & out) { + ggml_context * meta_ctx = nullptr; + gguf_init_params params{}; + params.no_alloc = true; + params.ctx = &meta_ctx; + gguf_context * gctx = gguf_init_from_file(path.c_str(), params); + if (!gctx) { + set_last_error("bailingmoe3: gguf_init_from_file failed: " + path); + return false; + } + + auto fail = [&](const std::string & message) { + set_last_error("bailingmoe3: " + message); + gguf_free(gctx); + if (meta_ctx) { + ggml_free(meta_ctx); + if (out.ctx == meta_ctx) out.ctx = nullptr; + meta_ctx = nullptr; + } + return false; + }; + + const int64_t arch_id = gguf_find_key(gctx, "general.architecture"); + const char * arch = arch_id >= 0 ? gguf_get_val_str(gctx, arch_id) : nullptr; + if (!arch || std::strcmp(arch, kArch) != 0) { + return fail(std::string("unexpected architecture '") + + (arch ? arch : "") + "'"); + } + + const std::string prefix = std::string(kArch) + "."; + const uint32_t block_count = get_u32_or(gctx, prefix + "block_count", 0); + const uint32_t nextn = get_u32_or(gctx, prefix + "nextn_predict_layers", 0); + if (block_count == 0 || nextn >= block_count) { + return fail("invalid block_count/nextn_predict_layers"); + } + const uint32_t n_layer = block_count - nextn; + const uint32_t n_embd = get_u32_or(gctx, prefix + "embedding_length", 0); + const uint32_t n_head = get_u32_or(gctx, prefix + "attention.head_count", 0); + const uint32_t kda_head_dim = get_u32_or(gctx, prefix + "kda.head_dim", 0); + const uint32_t ssm_conv = get_u32_or(gctx, prefix + "ssm.conv_kernel", 0); + const uint32_t rope_dim = get_u32_or(gctx, prefix + "rope.dimension_count", 0); + const uint32_t qk_mla = get_u32_or(gctx, prefix + "attention.key_length_mla", 0); + const uint32_t v_mla = get_u32_or(gctx, prefix + "attention.value_length_mla", 0); + const uint32_t kv_lora = get_u32_or(gctx, prefix + "attention.kv_lora_rank", 0); + const uint32_t q_lora = get_u32_or(gctx, prefix + "attention.q_lora_rank", 0); + const uint32_t n_ff = get_u32_or(gctx, prefix + "feed_forward_length", 0); + const uint32_t n_ff_exp = get_u32_or(gctx, prefix + "expert_feed_forward_length", 0); + const uint32_t n_ff_shexp = get_u32_or( + gctx, prefix + "expert_shared_feed_forward_length", 0); + const uint32_t n_expert = get_u32_or(gctx, prefix + "expert_count", 0); + const uint32_t n_expert_used = get_u32_or(gctx, prefix + "expert_used_count", 0); + const uint32_t n_expert_groups = get_u32_or( + gctx, prefix + "expert_group_count", 1); + const uint32_t n_expert_groups_used = get_u32_or( + gctx, prefix + "expert_group_used_count", 1); + const uint32_t dense_lead = get_u32_or(gctx, prefix + "leading_dense_block_count", 0); + const float kda_lower = get_f32_or(gctx, prefix + "kda.gate_lower_bound", 0.0f); + + if (n_layer == 0 || n_embd == 0 || n_head == 0 || kda_head_dim == 0 || + ssm_conv < 2 || rope_dim == 0 || qk_mla <= rope_dim || v_mla == 0 || + kv_lora == 0 || q_lora != 0 || n_ff == 0 || n_ff_exp == 0 || + n_ff_shexp == 0 || n_expert == 0 || n_expert_used == 0 || + n_expert_used > n_expert || n_expert_groups == 0 || + n_expert_groups_used == 0 || + n_expert_groups_used > n_expert_groups || + n_expert % n_expert_groups != 0 || dense_lead > n_layer || + kda_lower >= 0.0f) { + char message[640]; + std::snprintf(message, sizeof(message), + "invalid hparams: layers=%u(+%u MTP) embd=%u heads=%u " + "kda{head=%u conv=%u lower=%g} mla{qk=%u v=%u rope=%u kv_lora=%u q_lora=%u} " + "ff{dense=%u exp=%u shared=%u dense_lead=%u} " + "experts=%u used=%u groups=%u/%u", + n_layer, nextn, n_embd, n_head, kda_head_dim, ssm_conv, + static_cast(kda_lower), qk_mla, v_mla, rope_dim, + kv_lora, q_lora, n_ff, n_ff_exp, n_ff_shexp, dense_lead, + n_expert, n_expert_used, n_expert_groups_used, n_expert_groups); + return fail(message); + } + + // The per-layer KV-head array is the GGUF's authoritative recurrent/MLA + // pattern: zero means KDA, non-zero means MLA. LuceBox's shared hybrid + // cache uses a fixed interval, so verify that Ling's pattern is regular. + const std::vector kv_heads = + get_u32_array(gctx, prefix + "attention.head_count_kv"); + if (kv_heads.size() < n_layer) { + return fail("missing or short attention.head_count_kv array"); + } + int full_attention_interval = 0; + for (uint32_t il = 0; il < n_layer; ++il) { + if (kv_heads[il] != 0) { + const int interval = static_cast(il) + 1; + if (full_attention_interval == 0) full_attention_interval = interval; + if (interval % full_attention_interval != 0) { + return fail("irregular KDA/MLA layer pattern is not supported"); + } + } + } + if (full_attention_interval == 0 || n_layer % full_attention_interval != 0) { + return fail("no regular MLA layers found"); + } + for (uint32_t il = 0; il < n_layer; ++il) { + const bool expected_mla = ((il + 1) % full_attention_interval) == 0; + if ((kv_heads[il] != 0) != expected_mla) { + return fail("attention.head_count_kv does not match the inferred interval"); + } + } + + out.ctx = meta_ctx; + out.backend = backend; + out.n_layer = static_cast(n_layer); + out.n_embd = static_cast(n_embd); + out.n_head = static_cast(n_head); + out.n_head_kv = 1; // latent MLA is MQA + out.n_ff = static_cast(n_ff); + out.n_ff_exp = static_cast(n_ff_exp); + out.n_ff_shexp = static_cast(n_ff_shexp); + out.n_expert = static_cast(n_expert); + out.n_expert_used = static_cast(n_expert_used); + out.n_expert_groups = static_cast(n_expert_groups); + out.n_expert_groups_used = static_cast(n_expert_groups_used); + out.n_layer_dense_lead = static_cast(dense_lead); + out.full_attention_interval = full_attention_interval; + out.rope_dimension_count = static_cast(rope_dim); + out.rope_theta = get_f32_or(gctx, prefix + "rope.freq_base", 1000000.0f); + out.rms_eps = get_f32_or( + gctx, prefix + "attention.layer_norm_rms_epsilon", 1.0e-6f); + out.kda_head_dim = static_cast(kda_head_dim); + out.mla_qk_head_dim = static_cast(qk_mla); + out.mla_v_head_dim = static_cast(v_mla); + out.kv_lora_rank = static_cast(kv_lora); + out.q_lora_rank = static_cast(q_lora); + out.kda_gate_lower_bound = kda_lower; + out.n_embd_head_k = static_cast(kv_lora + rope_dim); + out.n_embd_head_v = static_cast(kv_lora); + out.ssm_d_conv = static_cast(ssm_conv); + out.ssm_d_inner = static_cast(n_head * kda_head_dim); + out.ssm_d_state = static_cast(kda_head_dim); + out.ssm_dt_rank = static_cast(n_head); + out.ssm_n_group = static_cast(n_head); + out.expert_gating_func = static_cast( + get_u32_or(gctx, prefix + "expert_gating_func", 2)); + out.expert_weights_scale = get_f32_or( + gctx, prefix + "expert_weights_scale", 1.0f); + out.expert_weights_norm = get_bool_or( + gctx, prefix + "expert_weights_norm", true); + out.is_moe = true; + out.is_bailingmoe3 = true; + + const uint32_t missing_token = 0xFFFFFFFFu; + const uint32_t eos = get_u32_or(gctx, "tokenizer.ggml.eos_token_id", missing_token); + const uint32_t eot = get_u32_or(gctx, "tokenizer.ggml.eot_token_id", missing_token); + out.eos_id = eos == missing_token ? -1 : static_cast(eos); + out.eos_chat_id = eot == missing_token ? -1 : static_cast(eot); + + out.layers.assign(n_layer, TargetLayer{}); + const std::vector clamp_exp = + get_f32_array(gctx, prefix + "swiglu_clamp_exp", n_layer); + const std::vector clamp_shexp = + get_f32_array(gctx, prefix + "swiglu_clamp_shexp", n_layer); + + auto tensor = [&](const char * name) { return ggml_get_tensor(meta_ctx, name); }; + auto layer_tensor = [&](uint32_t il, const char * suffix) { + char name[160]; + std::snprintf(name, sizeof(name), "blk.%u.%s", il, suffix); + return ggml_get_tensor(meta_ctx, name); + }; + + out.tok_embd = tensor("token_embd.weight"); + out.out_norm = tensor("output_norm.weight"); + out.output = tensor("output.weight"); + if (!out.tok_embd || !out.out_norm || !out.output) { + return fail("missing token_embd/output_norm/output tensor"); + } + out.n_vocab = static_cast(out.tok_embd->ne[1]); + + for (uint32_t il = 0; il < n_layer; ++il) { + TargetLayer & layer = out.layers[il]; + layer.attn_norm = layer_tensor(il, "attn_norm.weight"); + layer.ffn_norm = layer_tensor(il, "ffn_norm.weight"); + layer.attn_post_norm = layer.ffn_norm; + layer.ffn_swiglu_clamp_exp = clamp_exp[il]; + layer.ffn_swiglu_clamp_shexp = clamp_shexp[il]; + if (!layer.attn_norm || !layer.ffn_norm) { + return fail("layer " + std::to_string(il) + " missing norm tensor"); + } + + const bool mla = ((il + 1) % full_attention_interval) == 0; + layer.wo = layer_tensor(il, "attn_output.weight"); + if (mla) { + layer.wq = layer_tensor(il, "attn_q.weight"); + layer.attn_q_a = layer_tensor(il, "attn_q_a.weight"); + layer.attn_q_a_norm = layer_tensor(il, "attn_q_a_norm.weight"); + layer.attn_q_b = layer_tensor(il, "attn_q_b.weight"); + layer.attn_kv_a_mqa = layer_tensor(il, "attn_kv_a_mqa.weight"); + layer.attn_kv_a_norm = layer_tensor(il, "attn_kv_a_norm.weight"); + layer.attn_k_b = layer_tensor(il, "attn_k_b.weight"); + layer.attn_v_b = layer_tensor(il, "attn_v_b.weight"); + layer.wqkv_gate = layer_tensor(il, "attn_gate.weight"); + const bool has_q = layer.wq || + (layer.attn_q_a && layer.attn_q_a_norm && layer.attn_q_b); + if (!has_q || !layer.attn_kv_a_mqa || !layer.attn_kv_a_norm || + !layer.attn_k_b || !layer.attn_v_b || !layer.wqkv_gate || !layer.wo) { + return fail("layer " + std::to_string(il) + " missing MLA tensor"); + } + } else { + layer.wq = layer_tensor(il, "attn_q.weight"); + layer.wk = layer_tensor(il, "attn_k.weight"); + layer.wv = layer_tensor(il, "attn_v.weight"); + layer.ssm_conv1d_q = layer_tensor(il, "ssm_conv1d_q.weight"); + layer.ssm_conv1d_k = layer_tensor(il, "ssm_conv1d_k.weight"); + layer.ssm_conv1d_v = layer_tensor(il, "ssm_conv1d_v.weight"); + layer.ssm_f_a = layer_tensor(il, "ssm_f_a.weight"); + layer.ssm_beta = layer_tensor(il, "ssm_beta.weight"); + layer.ssm_a = layer_tensor(il, "ssm_a"); + layer.ssm_dt_bias = layer_tensor(il, "ssm_dt.bias"); + layer.ssm_g_a = layer_tensor(il, "ssm_g_a.weight"); + layer.ssm_norm = layer_tensor(il, "ssm_norm.weight"); + if (!layer.wq || !layer.wk || !layer.wv || !layer.wo || + !layer.ssm_conv1d_q || !layer.ssm_conv1d_k || + !layer.ssm_conv1d_v || !layer.ssm_f_a || !layer.ssm_beta || + !layer.ssm_a || !layer.ssm_dt_bias || !layer.ssm_g_a || + !layer.ssm_norm) { + return fail("layer " + std::to_string(il) + " missing KDA tensor"); + } + } + + if (il < dense_lead) { + layer.w_gate = layer_tensor(il, "ffn_gate.weight"); + layer.w_up = layer_tensor(il, "ffn_up.weight"); + layer.w_down = layer_tensor(il, "ffn_down.weight"); + if (!layer.w_gate || !layer.w_up || !layer.w_down) { + return fail("layer " + std::to_string(il) + " missing dense FFN tensor"); + } + } else { + layer.ffn_gate_inp = layer_tensor(il, "ffn_gate_inp.weight"); + layer.ffn_exp_probs_b = layer_tensor(il, "exp_probs_b.bias"); + layer.ffn_gate_exps = layer_tensor(il, "ffn_gate_exps.weight"); + layer.ffn_up_exps = layer_tensor(il, "ffn_up_exps.weight"); + layer.ffn_down_exps = layer_tensor(il, "ffn_down_exps.weight"); + layer.ffn_gate_shexp = layer_tensor(il, "ffn_gate_shexp.weight"); + layer.ffn_up_shexp = layer_tensor(il, "ffn_up_shexp.weight"); + layer.ffn_down_shexp = layer_tensor(il, "ffn_down_shexp.weight"); + if (!layer.ffn_gate_inp || !layer.ffn_exp_probs_b || + !layer.ffn_gate_exps || !layer.ffn_up_exps || + !layer.ffn_down_exps || !layer.ffn_gate_shexp || + !layer.ffn_up_shexp || !layer.ffn_down_shexp) { + return fail("layer " + std::to_string(il) + " missing MoE tensor"); + } + } + } + + // Allocate exactly the tensors referenced by the trunk. Prefix-based + // selection would also upload the 124B model's embedded MTP block. + std::unordered_set wanted; + auto add = [&](ggml_tensor * value) { if (value) wanted.insert(value); }; + add(out.out_norm); + add(out.output); + for (TargetLayer & layer : out.layers) { + add(layer.attn_norm); add(layer.ffn_norm); + add(layer.w_gate); add(layer.w_up); add(layer.w_down); + add(layer.wq); add(layer.wk); add(layer.wv); add(layer.wo); + add(layer.ssm_conv1d_q); add(layer.ssm_conv1d_k); add(layer.ssm_conv1d_v); + add(layer.ssm_f_a); add(layer.ssm_beta); add(layer.ssm_a); + add(layer.ssm_dt_bias); add(layer.ssm_g_a); add(layer.ssm_norm); + add(layer.attn_q_a); add(layer.attn_q_a_norm); add(layer.attn_q_b); + add(layer.attn_kv_a_mqa); add(layer.attn_kv_a_norm); + add(layer.attn_k_b); add(layer.attn_v_b); add(layer.wqkv_gate); + add(layer.ffn_gate_inp); add(layer.ffn_exp_probs_b); + add(layer.ffn_gate_exps); add(layer.ffn_up_exps); add(layer.ffn_down_exps); + add(layer.ffn_gate_shexp); add(layer.ffn_up_shexp); add(layer.ffn_down_shexp); + } + + const int64_t n_tensors = gguf_get_n_tensors(gctx); + ggml_backend_buffer_type_t buffer_type = + ggml_backend_get_default_buffer_type(backend); + const size_t alignment = ggml_backend_buft_get_alignment(buffer_type); + std::vector allocations; + allocations.reserve(wanted.size()); + size_t allocation_size = 0; + for (int64_t tid = 0; tid < n_tensors; ++tid) { + const char * name = gguf_get_tensor_name(gctx, tid); + ggml_tensor * value = ggml_get_tensor(meta_ctx, name); + if (!value || wanted.find(value) == wanted.end()) continue; + allocation_size = align_up(allocation_size, alignment); + TensorAllocation allocation; + allocation.tensor = value; + allocation.file_offset = + gguf_get_data_offset(gctx) + gguf_get_tensor_offset(gctx, tid); + allocation.file_size = gguf_get_tensor_size(gctx, tid); + allocation.buffer_offset = allocation_size; + allocation_size += ggml_backend_buft_get_alloc_size(buffer_type, value); + allocations.push_back(allocation); + } + if (allocations.size() != wanted.size()) { + return fail("failed to resolve every trunk tensor in the GGUF table"); + } + + out.buf = ggml_backend_alloc_buffer(backend, allocation_size); + if (!out.buf) return fail("weight buffer allocation failed"); + ggml_backend_buffer_set_usage(out.buf, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); + char * base = static_cast(ggml_backend_buffer_get_base(out.buf)); + for (const TensorAllocation & allocation : allocations) { + if (ggml_backend_tensor_alloc(out.buf, allocation.tensor, + base + allocation.buffer_offset) != GGML_STATUS_SUCCESS) { + ggml_backend_buffer_free(out.buf); + out.buf = nullptr; + return fail("weight tensor allocation failed"); + } + } + + GgufMmap mmap; + std::string mmap_error; + if (!mmap.open(path, mmap_error)) { + ggml_backend_buffer_free(out.buf); + out.buf = nullptr; + return fail(mmap_error); + } + const uint8_t * bytes = static_cast(mmap.data()); + const size_t file_size = mmap.size(); + for (const TensorAllocation & allocation : allocations) { + if (allocation.file_offset + allocation.file_size < allocation.file_offset || + allocation.file_offset + allocation.file_size > file_size) { + ggml_backend_buffer_free(out.buf); + out.buf = nullptr; + return fail("truncated tensor data for " + + std::string(allocation.tensor->name)); + } + ggml_backend_tensor_set(allocation.tensor, + bytes + allocation.file_offset, 0, allocation.file_size); + } + + const int64_t token_tid = gguf_find_tensor(gctx, "token_embd.weight"); + if (token_tid < 0) { + ggml_backend_buffer_free(out.buf); + out.buf = nullptr; + return fail("token_embd.weight missing from tensor table"); + } + const size_t token_relative_offset = gguf_get_tensor_offset(gctx, token_tid); + const size_t token_size = gguf_get_tensor_size(gctx, token_tid); + const size_t data_offset = gguf_get_data_offset(gctx); + if (!gguf_tensor_in_file(data_offset, token_relative_offset, token_size, file_size)) { + ggml_backend_buffer_free(out.buf); + out.buf = nullptr; + return fail("truncated token_embd.weight"); + } + out.embedder.tok_embd_owned.resize(token_size); + std::memcpy(out.embedder.tok_embd_owned.data(), + bytes + data_offset + token_relative_offset, token_size); + out.embedder.tok_embd_bytes = out.embedder.tok_embd_owned.data(); + out.embedder.tok_embd_type = gguf_get_tensor_type(gctx, token_tid); + out.embedder.n_embd = out.n_embd; + out.embedder.n_vocab = out.n_vocab; + out.embedder.row_bytes = token_size / static_cast(out.n_vocab); + + gguf_free(gctx); + gctx = nullptr; + meta_ctx = nullptr; // owned by out.ctx from here on + + char summary[384]; + std::snprintf(summary, sizeof(summary), + "bailingmoe3 trunk loaded: %d layers (%d KDA + %d MLA), " + "%zu tensors %.2f GiB, experts=%d/%d groups=%d/%d, " + "MTP blocks ignored=%u, eos=%d", + out.n_layer, out.n_layer - out.n_layer / out.full_attention_interval, + out.n_layer / out.full_attention_interval, allocations.size(), + allocation_size / (1024.0 * 1024.0 * 1024.0), + out.n_expert_used, out.n_expert, + out.n_expert_groups_used, out.n_expert_groups, + nextn, out.eos_id); + set_last_error(summary); + std::fprintf(stderr, "[bailingmoe3] %s\n", summary); + return true; +} + +} // namespace dflash::common diff --git a/server/src/common/backend_factory.cpp b/server/src/common/backend_factory.cpp index ee9b198bf..d462be899 100644 --- a/server/src/common/backend_factory.cpp +++ b/server/src/common/backend_factory.cpp @@ -6,6 +6,7 @@ #include "qwen35_backend.h" #include "qwen35moe_backend.h" +#include "bailingmoe3_backend.h" #include "laguna_backend.h" #include "laguna_layer_split_adapter.h" #include "qwen3_backend.h" @@ -92,6 +93,7 @@ constexpr bool layer_split_carries(FeatureSupport support) { DFLASH_CHECK_ARCH("qwen35", Qwen35Config, Qwen35LayerSplitAdapterConfig); DFLASH_CHECK_ARCH("qwen35moe", Qwen35Config, NoLayerSplitConfig); +DFLASH_CHECK_ARCH("bailingmoe3", BailingMoe3Config, NoLayerSplitConfig); DFLASH_CHECK_ARCH("laguna", LagunaBackendArgs, LagunaLayerSplitAdapterConfig); DFLASH_CHECK_ARCH("qwen3", Qwen3BackendConfig, NoLayerSplitConfig); DFLASH_CHECK_ARCH("gemma4", Gemma4BackendConfig, Gemma4LayerSplitAdapterConfig); @@ -321,6 +323,19 @@ std::unique_ptr create_backend( } return backend; + } else if (arch == "bailingmoe3") { + BailingMoe3Config cfg; + cfg.model_path = args.model_path; + cfg.device = args.device; + cfg.stream_fd = args.stream_fd; + + auto backend = std::make_unique(cfg); + if (!backend->init()) { + std::fprintf(stderr, "[backend_factory] BailingMoe3Backend init failed\n"); + return nullptr; + } + return backend; + } else if (arch == "laguna") { if (args.device.is_layer_split()) { LagunaLayerSplitAdapterConfig cfg; diff --git a/server/src/common/gguf_inspect.cpp b/server/src/common/gguf_inspect.cpp index 9c3b80143..58444be95 100644 --- a/server/src/common/gguf_inspect.cpp +++ b/server/src/common/gguf_inspect.cpp @@ -26,9 +26,10 @@ bool derive_effective_target_layer_count(const std::string & arch, return false; } - // Embedded NextN blocks are currently defined by the Qwen3.5/3.6 GGUF - // layout. Do not reinterpret similarly named metadata on other arches. - if (arch != "qwen35" && arch != "qwen35moe") { + // Embedded NextN blocks are defined by the Qwen3.5/3.6 and BailingMoE3 + // GGUF layouts. Do not reinterpret similarly named metadata elsewhere. + if (arch != "qwen35" && arch != "qwen35moe" && + arch != "bailingmoe3") { return true; } if (nextn_predict_layers == 0) { diff --git a/server/src/common/model_capabilities.h b/server/src/common/model_capabilities.h index f62087141..ebc7524c0 100644 --- a/server/src/common/model_capabilities.h +++ b/server/src/common/model_capabilities.h @@ -73,10 +73,11 @@ inline constexpr ArchCapabilities kArchCapabilities[] = { // arch split rdraft pflash offload draft ddtree vwidth dblock fa_win dswa paged {"qwen35", true, true, true, false, kBoth, kBoth, kNever, kMono, kBoth, kBoth, kMono}, {"qwen35moe", false, false, false, true, kMono, kMono, kNever, kNever, kMono, kMono, kNever}, - {"laguna", true, false, false, true, kMono, kMono, kMono, kNever, kNever, kNever, kNever}, - {"qwen3", false, false, true, false, kNever, kNever, kNever, kNever, kNever, kNever, kNever}, - {"gemma4", true, false, false, false, kMono, kNever, kNever, kNever, kBoth, kNever, kNever}, - {"deepseek4", true, false, false, false, kNever, kNever, kNever, kNever, kNever, kNever, kNever}, + {"bailingmoe3",false, false, false, false, kNever,kNever,kNever, kNever, kNever,kNever, kNever}, + {"laguna", true, false, false, true, kMono, kMono, kMono, kNever, kNever,kNever, kNever}, + {"qwen3", false, false, true, false, kNever, kNever, kNever, kNever, kNever,kNever, kNever}, + {"gemma4", true, false, false, false, kMono, kNever, kNever, kNever, kBoth,kNever, kNever}, + {"deepseek4", true, false, false, false, kNever, kNever, kNever, kNever, kNever,kNever, kNever}, }; inline constexpr std::size_t kArchCount = diff --git a/server/src/internal.h b/server/src/internal.h index 792be9fcc..07d6b89a0 100644 --- a/server/src/internal.h +++ b/server/src/internal.h @@ -87,11 +87,31 @@ struct TargetLayer { // one small GPU tensor per DeltaNet layer (src[9] of the GDN op). ggml_tensor * ssm_gate_ba = nullptr; + // BailingMoE3 / Ling 3 KDA. Unlike Qwen3.5's fused projection and + // convolution, Ling projects and convolves Q, K, and V independently and + // uses a vector-valued decay gate (KDA) per head dimension. + ggml_tensor * ssm_conv1d_q = nullptr; // [conv, 1, d_inner, 1] + ggml_tensor * ssm_conv1d_k = nullptr; // [conv, 1, d_inner, 1] + ggml_tensor * ssm_conv1d_v = nullptr; // [conv, 1, d_inner, 1] + ggml_tensor * ssm_f_a = nullptr; // [hidden, d_inner] + ggml_tensor * ssm_g_a = nullptr; // [hidden, d_inner] + + // BailingMoE3 / Ling 3 MLA. The latent K/V cache stores + // [kv_lora_rank + rope_dim] for K and [kv_lora_rank] for V. + ggml_tensor * attn_q_a = nullptr; + ggml_tensor * attn_q_a_norm = nullptr; + ggml_tensor * attn_q_b = nullptr; + ggml_tensor * attn_kv_a_mqa = nullptr; + ggml_tensor * attn_kv_a_norm = nullptr; + ggml_tensor * attn_k_b = nullptr; + ggml_tensor * attn_v_b = nullptr; + // MoE FFN (qwen35moe only; nullptr on dense qwen35) ggml_tensor * ffn_gate_inp = nullptr; // [hidden, n_expert] router ggml_tensor * ffn_gate_exps = nullptr; // [hidden, n_ff_exp, n_expert] ggml_tensor * ffn_up_exps = nullptr; // [hidden, n_ff_exp, n_expert] ggml_tensor * ffn_down_exps = nullptr; // [n_ff_exp, hidden, n_expert] + ggml_tensor * ffn_exp_probs_b = nullptr; // [n_expert] router correction bias ggml_tensor * ffn_gate_up_exps = nullptr; // [hidden, 2*n_ff_exp, n_expert] optional fused gate/up ggml_tensor * ffn_gate_inp_shexp = nullptr; // [hidden] shared-expert scalar gate ggml_tensor * ffn_gate_shexp = nullptr; // [hidden, n_ff_shexp] @@ -126,6 +146,11 @@ struct TargetLayer { float ffn_gate_shexp_s = 1.0f; float ffn_up_shexp_s = 1.0f; float ffn_down_shexp_s = 1.0f; + + // Optional per-layer activation limits used by late Ling 3 blocks. + // Zero means ordinary SwiGLU. + float ffn_swiglu_clamp_exp = 0.0f; + float ffn_swiglu_clamp_shexp = 0.0f; }; // CPU-side embedder: keeps a mmap of the GGUF alive and knows how to @@ -186,6 +211,8 @@ struct TargetWeights { int n_ff_shexp = 0; int n_expert = 0; int n_expert_used = 0; + int n_expert_groups = 1; + int n_expert_groups_used = 1; int n_vocab = DFLASH27B_TARGET_VOCAB; int rope_dimension_count = 64; float rope_theta = 10000000.0f; @@ -193,6 +220,17 @@ struct TargetWeights { float expert_weights_scale = 1.0f; int expert_gating_func = 1; // 1=softmax, 2=sigmoid (llama.cpp enum values) bool is_moe = false; + bool is_bailingmoe3 = false; + bool expert_weights_norm = true; + int n_layer_dense_lead = 0; + + // BailingMoE3 / Ling 3 architecture parameters. + int kda_head_dim = 0; + int mla_qk_head_dim = 0; + int mla_v_head_dim = 0; + int kv_lora_rank = 0; + int q_lora_rank = 0; + float kda_gate_lower_bound = 0.0f; int ssm_d_conv = 4; int ssm_d_inner = 6144; int ssm_d_state = 128; @@ -243,6 +281,12 @@ bool load_target_gguf_partial(const std::string & path, const TargetLoadPlan & plan, TargetWeights & out); +// Load the autoregressive trunk of a BailingMoE3 GGUF (Ling 3.x). Embedded +// NextN/MTP blocks are intentionally ignored by this baseline backend. +bool load_bailingmoe3_gguf(const std::string & path, + ggml_backend_t backend, + TargetWeights & out); + void free_target_weights(TargetWeights & w); // ─── Draft weights (z-lab DFlash, bf16) ─────────────────────────── diff --git a/server/src/qwen35/qwen35_backend.cpp b/server/src/qwen35/qwen35_backend.cpp index 20a4b4f1a..57d0f23d5 100644 --- a/server/src/qwen35/qwen35_backend.cpp +++ b/server/src/qwen35/qwen35_backend.cpp @@ -1616,7 +1616,8 @@ GenerateResult Qwen35Backend::restore_and_generate_impl(int slot, const bool pool = kvflash_active(); if (!build_target_step(sg_, w_, cache_, target_backend_, /*kv_start=*/cache_.cur_pos, /*n_tokens=*/1, - /*with_mask=*/pool, /*capture=*/false, + /*with_mask=*/pool || w_.is_bailingmoe3, + /*capture=*/false, /*capture_delta_intermediate=*/false, /*fa_window=*/0, /*logits_tail_rows=*/0, @@ -2255,7 +2256,8 @@ bool Qwen35Backend::do_ar_decode(int committed, int n_gen, const bool paged = cfg_.paged_attention; if (!build_target_step(sg_, w_, cache_, target_backend_, /*kv_start=*/committed, /*n_tokens=*/1, - /*with_mask=*/pool, /*capture=*/false, + /*with_mask=*/pool || w_.is_bailingmoe3, + /*capture=*/false, /*capture_delta_intermediate=*/false, /*fa_window=*/0, /*logits_tail_rows=*/0, @@ -2291,7 +2293,12 @@ bool Qwen35Backend::do_ar_decode(int committed, int n_gen, ggml_backend_tensor_set(sg_.kv_write_rows, row_vals.data(), 0, sizeof(int64_t) * n_head_kv); } - if (pool) kvflash_upload_mask(); + if (pool) { + kvflash_upload_mask(); + } else if (w_.is_bailingmoe3) { + upload_qwen35_causal_mask( + sg_.attn_mask, committed, 1, cfg_.kq_stride_pad); + } auto st = ggml_backend_graph_compute(target_backend_, sg_.gf); if (st != GGML_STATUS_SUCCESS) return false; diff --git a/server/src/qwen35/qwen35_target_graph.cpp b/server/src/qwen35/qwen35_target_graph.cpp index 0e87c1a36..5e4937e09 100644 --- a/server/src/qwen35/qwen35_target_graph.cpp +++ b/server/src/qwen35/qwen35_target_graph.cpp @@ -184,10 +184,13 @@ bool create_target_cache_partial(const TargetWeights & w, // The rotation costs two extra launches per attention layer. Decision // logic lives in common/kv_rotation.h and is shared with the disk // prefix cache identity salt: a cache written under one rotation basis - // must never be adopted by a session using the other. - out.kv_k_rotated = dflash_kv_k_rotation_enabled(ggml_type_name(kv_k_type)); + // must never be adopted by a session using the other. Ling's latent MLA + // cache uses unequal K/V widths, so it cannot use the qwen FWHT path. + out.kv_k_rotated = + !w.is_bailingmoe3 && + dflash_kv_k_rotation_enabled(ggml_type_name(kv_k_type)); - const bool needs_256_stride = + const bool needs_256_stride = w.is_bailingmoe3 || kv_k_type == GGML_TYPE_TQ3_0 || kv_v_type == GGML_TYPE_TQ3_0; // kvflash mode: attention tensors are allocated at the (smaller) // physical pool capacity; logical positions are mapped to pool slots @@ -231,7 +234,7 @@ bool create_target_cache_partial(const TargetWeights & w, ggml_tensor * K = ggml_new_tensor_3d(out.base_ctx, kv_k_type, head_dim, max_ctx_alloc, w.n_head_kv); ggml_tensor * V = ggml_new_tensor_3d(out.base_ctx, kv_v_type, - head_dim, max_ctx_alloc, w.n_head_kv); + w.n_embd_head_v, max_ctx_alloc, w.n_head_kv); char name[64]; std::snprintf(name, sizeof(name), "cache_k_%d", il); ggml_set_name(K, name); @@ -1463,6 +1466,240 @@ static ggml_tensor * build_full_attn_block( return attn; } +// Ling 3 stores MLA K/V in compressed latent space. K contains the normalized +// 512-wide latent plus the 64 RoPE dimensions; V is the latent alone. Query +// absorption through attn_k_b turns the 128 non-RoPE Q dimensions into the +// same latent space, and attn_v_b expands the attention result back to 128 +// value dimensions per head. +static ggml_tensor * build_bailingmoe3_mla_block( + ggml_context * ctx, + ggml_cgraph * gf, + const TargetWeights & w, + const TargetLayer & L, + ggml_tensor * cur, + ggml_tensor * positions, + ggml_tensor * cache_k, + ggml_tensor * cache_v, + ggml_tensor * attn_mask, + int kv_start, + int n_tokens) { + const int n_head = w.n_head; + const int qk_dim = w.mla_qk_head_dim; + const int rope_dim = w.rope_dimension_count; + const int nope_dim = qk_dim - rope_dim; + const int kv_rank = w.kv_lora_rank; + const int v_dim = w.mla_v_head_dim; + GGML_ASSERT(attn_mask != nullptr); + + ggml_tensor * q_all = nullptr; + if (L.attn_q_a) { + q_all = ggml_mul_mat(ctx, L.attn_q_a, cur); + q_all = rms_norm_mul(ctx, q_all, L.attn_q_a_norm, w.rms_eps); + q_all = ggml_mul_mat(ctx, L.attn_q_b, q_all); + } else { + q_all = ggml_mul_mat(ctx, L.wq, cur); + } + q_all = ggml_reshape_3d(ctx, q_all, qk_dim, n_head, n_tokens); + ggml_tensor * q_nope = ggml_view_3d( + ctx, q_all, nope_dim, n_head, n_tokens, + q_all->nb[1], q_all->nb[2], 0); + ggml_tensor * q_pe = ggml_view_3d( + ctx, q_all, rope_dim, n_head, n_tokens, + q_all->nb[1], q_all->nb[2], + static_cast(nope_dim) * ggml_element_size(q_all)); + + ggml_tensor * kv_all = ggml_mul_mat(ctx, L.attn_kv_a_mqa, cur); + ggml_tensor * kv = ggml_view_2d( + ctx, kv_all, kv_rank, n_tokens, kv_all->nb[1], 0); + ggml_tensor * k_pe = ggml_view_3d( + ctx, kv_all, rope_dim, 1, n_tokens, + kv_all->nb[1], kv_all->nb[1], + static_cast(kv_rank) * ggml_element_size(kv_all)); + + // The shared Qwen graph input reserves four position lanes for M-RoPE. + // Bailing uses ordinary RoPE, so expose just the first lane to ggml_rope. + ggml_tensor * rope_positions = positions; + if (positions->ne[0] != n_tokens) { + GGML_ASSERT(positions->ne[0] >= n_tokens); + rope_positions = ggml_view_1d(ctx, positions, n_tokens, 0); + } + + // Bailing V3 uses interleaved rotary pairs, i.e. GGML's NORMAL layout. + q_pe = ggml_cont(ctx, q_pe); + k_pe = ggml_cont(ctx, k_pe); + q_pe = ggml_rope_ext(ctx, q_pe, rope_positions, nullptr, + rope_dim, GGML_ROPE_TYPE_NORMAL, 0, + w.rope_theta, 1.0f, + 0.0f, 1.0f, 0.0f, 0.0f); + k_pe = ggml_rope_ext(ctx, k_pe, rope_positions, nullptr, + rope_dim, GGML_ROPE_TYPE_NORMAL, 0, + w.rope_theta, 1.0f, + 0.0f, 1.0f, 0.0f, 0.0f); + kv = rms_norm_mul(ctx, kv, L.attn_kv_a_norm, w.rms_eps); + + // Absorb the K projection into Q: [nope,T,H] -> [latent,T,H]. + q_nope = ggml_permute(ctx, q_nope, 0, 2, 1, 3); + q_nope = ggml_mul_mat(ctx, L.attn_k_b, q_nope); + q_nope = ggml_permute(ctx, q_nope, 0, 2, 1, 3); + ggml_tensor * q = ggml_concat(ctx, q_nope, q_pe, 0); + + ggml_tensor * kv_3d = ggml_reshape_3d(ctx, kv, kv_rank, 1, n_tokens); + ggml_tensor * k_cur = ggml_concat(ctx, kv_3d, k_pe, 0); + ggml_tensor * v_cur = kv_3d; + + // Persistent latent cache: [D,T,1]. + ggml_tensor * k_write = ggml_permute(ctx, k_cur, 0, 2, 1, 3); + ggml_tensor * v_write = ggml_permute(ctx, v_cur, 0, 2, 1, 3); + ggml_tensor * k_slot = ggml_view_3d( + ctx, cache_k, kv_rank + rope_dim, n_tokens, 1, + cache_k->nb[1], cache_k->nb[2], + static_cast(kv_start) * cache_k->nb[1]); + ggml_tensor * v_slot = ggml_view_3d( + ctx, cache_v, kv_rank, n_tokens, 1, + cache_v->nb[1], cache_v->nb[2], + static_cast(kv_start) * cache_v->nb[1]); + ggml_build_forward_expand(gf, ggml_cpy(ctx, k_write, k_slot)); + ggml_build_forward_expand(gf, ggml_cpy(ctx, v_write, v_slot)); + + const int kv_len = kv_start + n_tokens; + // The 576x512 CUDA flash-attention specialization requires the compressed + // K/V span to be padded to its 256-token launch stride. The caller always + // supplies a causal mask for Bailing, so zero-initialized future cache rows + // remain invisible. + const int kv_len_padded = std::min( + ((kv_len + 255) / 256) * 256, static_cast(cache_k->ne[1])); + ggml_tensor * k_read = ggml_view_3d( + ctx, cache_k, kv_rank + rope_dim, kv_len_padded, 1, + cache_k->nb[1], cache_k->nb[2], 0); + ggml_tensor * v_read = ggml_view_3d( + ctx, cache_v, kv_rank, kv_len_padded, 1, + cache_v->nb[1], cache_v->nb[2], 0); + ggml_tensor * q_fa = ggml_cont(ctx, ggml_permute(ctx, q, 0, 2, 1, 3)); + ggml_tensor * attn = ggml_flash_attn_ext( + ctx, q_fa, k_read, v_read, attn_mask, + 1.0f / std::sqrt(static_cast(qk_dim)), 0.0f, 0.0f); + ggml_flash_attn_ext_set_prec(attn, GGML_PREC_F32); + + // Flash attention returns [latent,H,T]. Apply the per-head V expansion + // as a batched matmul in [latent,T,H] layout. + attn = ggml_permute(ctx, attn, 0, 2, 1, 3); + attn = ggml_mul_mat(ctx, L.attn_v_b, attn); + attn = ggml_cont(ctx, ggml_permute(ctx, attn, 0, 2, 1, 3)); + + ggml_tensor * gate = ggml_mul_mat(ctx, L.wqkv_gate, cur); + gate = ggml_sigmoid(ctx, + ggml_reshape_3d(ctx, gate, 1, n_head, n_tokens)); + attn = ggml_mul(ctx, attn, gate); + attn = ggml_cont_2d(ctx, attn, v_dim * n_head, n_tokens); + return ggml_mul_mat(ctx, L.wo, attn); +} + +static ggml_tensor * build_bailingmoe3_causal_conv1d( + ggml_context * ctx, + ggml_cgraph * gf, + ggml_tensor * all_conv_state, + int qkv_index, + ggml_tensor * cur, + ggml_tensor * projection, + ggml_tensor * conv_weight, + int d_conv, + int d_inner, + int head_dim, + int n_head, + int n_tokens) { + const size_t state_element = ggml_element_size(all_conv_state); + const size_t channel_stride = all_conv_state->nb[1]; + ggml_tensor * conv_state = ggml_view_3d( + ctx, all_conv_state, d_conv - 1, d_inner, 1, + channel_stride, all_conv_state->nb[2], + static_cast(qkv_index) * d_inner * channel_stride); + + ggml_tensor * projected = ggml_mul_mat(ctx, projection, cur); + projected = ggml_reshape_3d(ctx, projected, d_inner, n_tokens, 1); + ggml_tensor * conv_input = + ggml_concat(ctx, conv_state, ggml_transpose(ctx, projected), 0); + + ggml_tensor * last = ggml_view_3d( + ctx, conv_input, d_conv - 1, d_inner, 1, + conv_input->nb[1], conv_input->nb[2], + static_cast(n_tokens) * state_element); + ggml_build_forward_expand(gf, ggml_cpy(ctx, last, conv_state)); + + ggml_tensor * conv_2d = ggml_reshape_2d(ctx, conv_weight, d_conv, d_inner); + ggml_tensor * result = ggml_ssm_conv(ctx, conv_input, conv_2d); + result = ggml_silu(ctx, ggml_reshape_2d(ctx, result, d_inner, n_tokens)); + return ggml_reshape_4d(ctx, result, head_dim, n_head, n_tokens, 1); +} + +// Ling 3 KDA block. The shared ggml CUDA primitive detects KDA from the +// vector gate's first dimension (128 rather than scalar 1). +static ggml_tensor * build_bailingmoe3_kda_block( + ggml_context * ctx, + ggml_cgraph * gf, + const TargetWeights & w, + const TargetLayer & L, + ggml_tensor * cur, + ggml_tensor * conv_state, + ggml_tensor * ssm_state, + int n_tokens) { + const int head_dim = w.kda_head_dim; + const int n_head = w.n_head; + const int d_inner = head_dim * n_head; + + ggml_tensor * q = build_bailingmoe3_causal_conv1d( + ctx, gf, conv_state, 0, cur, L.wq, L.ssm_conv1d_q, + w.ssm_d_conv, d_inner, head_dim, n_head, n_tokens); + ggml_tensor * k = build_bailingmoe3_causal_conv1d( + ctx, gf, conv_state, 1, cur, L.wk, L.ssm_conv1d_k, + w.ssm_d_conv, d_inner, head_dim, n_head, n_tokens); + ggml_tensor * v = build_bailingmoe3_causal_conv1d( + ctx, gf, conv_state, 2, cur, L.wv, L.ssm_conv1d_v, + w.ssm_d_conv, d_inner, head_dim, n_head, n_tokens); + + ggml_tensor * gate = ggml_mul_mat(ctx, L.ssm_f_a, cur); + gate = ggml_add(ctx, gate, L.ssm_dt_bias); + gate = ggml_reshape_4d(ctx, gate, head_dim, n_head, n_tokens, 1); + ggml_tensor * a = ggml_reshape_4d(ctx, L.ssm_a, 1, n_head, 1, 1); + gate = ggml_scale(ctx, ggml_sigmoid(ctx, ggml_mul(ctx, gate, a)), + w.kda_gate_lower_bound); + + ggml_tensor * beta = ggml_mul_mat(ctx, L.ssm_beta, cur); + beta = ggml_sigmoid(ctx, + ggml_reshape_4d(ctx, beta, 1, n_head, n_tokens, 1)); + q = ggml_l2_norm(ctx, q, w.rms_eps); + k = ggml_l2_norm(ctx, k, w.rms_eps); + + ggml_tensor * state = ggml_reshape_4d( + ctx, ssm_state, head_dim, head_dim, n_head, 1); + ggml_tensor * packed = + ggml_gated_delta_net(ctx, q, k, v, gate, beta, state); + ggml_gated_delta_net_set_skip_intermediate(packed, true); + + const size_t element = ggml_element_size(packed); + ggml_tensor * output = ggml_view_4d( + ctx, packed, head_dim, n_head, n_tokens, 1, + static_cast(head_dim) * element, + static_cast(head_dim) * n_head * element, + static_cast(head_dim) * n_head * n_tokens * element, + 0); + ggml_tensor * new_state = ggml_view_4d( + ctx, packed, head_dim, head_dim, n_head, 1, + static_cast(head_dim) * element, + static_cast(head_dim) * head_dim * element, + static_cast(head_dim) * head_dim * n_head * element, + static_cast(head_dim) * n_head * n_tokens * element); + ggml_build_forward_expand(gf, ggml_cpy(ctx, new_state, state)); + + ggml_tensor * output_gate = ggml_mul_mat(ctx, L.ssm_g_a, cur); + output_gate = ggml_reshape_3d( + ctx, output_gate, head_dim, n_head, n_tokens); + output = ggml_reshape_3d(ctx, output, head_dim, n_head, n_tokens); + output = rms_norm_mul(ctx, output, L.ssm_norm, w.rms_eps); + output = ggml_mul(ctx, output, ggml_sigmoid(ctx, output_gate)); + output = ggml_cont_2d(ctx, output, d_inner, n_tokens); + return ggml_mul_mat(ctx, L.wo, output); +} + // Gated DeltaNet block using the fused ggml_gated_delta_net primitive. // // Matches the semantics of llama.cpp's build_layer_attn_linear + build_delta_net_fused. @@ -2215,14 +2452,18 @@ static ggml_tensor * build_single_layer( for (int il = 0; il < layer_idx; il++) { if (((il + 1) % w.full_attention_interval) == 0) fa_idx++; } - cur = build_full_attn_block(ctx, gf, w, L, cur, positions, w.rope_sections, - cache.attn_k[fa_idx], cache.attn_v[fa_idx], - attn_mask, kv_start, n_tokens, - cache.kv_k_type, cache.kv_v_type, - cache.kv_k_rotated, - fa_window, - q_tail_capture, q_tail_start, - kv_write_rows); + cur = w.is_bailingmoe3 + ? build_bailingmoe3_mla_block( + ctx, gf, w, L, cur, positions, + cache.attn_k[fa_idx], cache.attn_v[fa_idx], + attn_mask, kv_start, n_tokens) + : build_full_attn_block( + ctx, gf, w, L, cur, positions, w.rope_sections, + cache.attn_k[fa_idx], cache.attn_v[fa_idx], + attn_mask, kv_start, n_tokens, + cache.kv_k_type, cache.kv_v_type, + cache.kv_k_rotated, fa_window, + q_tail_capture, q_tail_start, kv_write_rows); } else { int dn_idx = 0; for (int il = 0; il < layer_idx; il++) { @@ -2235,11 +2476,16 @@ static ggml_tensor * build_single_layer( cap_ptr->ssm_intermediate_states = cache.ssm_intermediate[dn_idx]; cap_ptr->conv_input = cache.conv_input_cache[dn_idx]; } - cur = build_delta_net_block(ctx, gf, w, L, cur, - cache.conv_state[dn_idx], cache.ssm_state[dn_idx], - n_tokens, cap_ptr, parent_ids, - /*skip_gdn_intermediate=*/true, - supports_qwen35_fused_kernels(cache.backend)); + cur = w.is_bailingmoe3 + ? build_bailingmoe3_kda_block( + ctx, gf, w, L, cur, cache.conv_state[dn_idx], + cache.ssm_state[dn_idx], n_tokens) + : build_delta_net_block( + ctx, gf, w, L, cur, + cache.conv_state[dn_idx], cache.ssm_state[dn_idx], + n_tokens, cap_ptr, parent_ids, + /*skip_gdn_intermediate=*/true, + supports_qwen35_fused_kernels(cache.backend)); } cur = ggml_add(ctx, cur, inpSA); @@ -2247,8 +2493,9 @@ static ggml_tensor * build_single_layer( ggml_tensor * ffn_residual = cur; ggml_tensor * post = rms_norm_mul(ctx, cur, L.attn_post_norm, eps); ggml_tensor * moe_selected = nullptr; - ggml_tensor * ffn = w.is_moe ? build_qwen35moe_ffn(ctx, post, w, L, &moe_selected) - : build_swiglu_ffn(ctx, post, L); + ggml_tensor * ffn = L.ffn_gate_inp + ? build_qwen35moe_ffn(ctx, post, w, L, &moe_selected) + : build_swiglu_ffn(ctx, post, L); if (moe_selected_out) { *moe_selected_out = moe_selected; } @@ -2350,28 +2597,38 @@ QwenGraphOutputs build_qwen35_graph( if (is_attn) { const bool want_q_cap = in.q_capture && cache.q_cap; ggml_tensor * q_fa = nullptr; - cur = build_full_attn_block(ctx, gf, w, L, cur, in.positions, w.rope_sections, - cache.attn_k[fa_idx], cache.attn_v[fa_idx], - in.attn_mask, in.kv_start, n_tokens, - cache.kv_k_type, cache.kv_v_type, - cache.kv_k_rotated, - in.fa_window, - /*q_tail_capture=*/nullptr, - /*q_tail_start=*/0, - in.kv_write_rows, - want_q_cap ? &q_fa : nullptr, - in.paged_block_table, - in.paged_kv_seq_lens, - in.paged_query_seq_ids, - in.paged_query_positions, - in.paged_max_kv_len, - in.active_slot_ids, - in.tree_sizes ? in.parent_ids : nullptr, - in.tree_sizes, - in.tree_width, - in.tree_scratch_base, - in.tree_scratch_stride, - cache.max_ctx); + if (w.is_bailingmoe3) { + GGML_ASSERT(!in.kv_write_rows && !in.paged_block_table && + !in.paged_query_seq_ids && !in.tree_sizes && + in.n_seqs == 1); + cur = build_bailingmoe3_mla_block( + ctx, gf, w, L, cur, in.positions, + cache.attn_k[fa_idx], cache.attn_v[fa_idx], + in.attn_mask, in.kv_start, n_tokens); + } else { + cur = build_full_attn_block(ctx, gf, w, L, cur, in.positions, w.rope_sections, + cache.attn_k[fa_idx], cache.attn_v[fa_idx], + in.attn_mask, in.kv_start, n_tokens, + cache.kv_k_type, cache.kv_v_type, + cache.kv_k_rotated, + in.fa_window, + /*q_tail_capture=*/nullptr, + /*q_tail_start=*/0, + in.kv_write_rows, + want_q_cap ? &q_fa : nullptr, + in.paged_block_table, + in.paged_kv_seq_lens, + in.paged_query_seq_ids, + in.paged_query_positions, + in.paged_max_kv_len, + in.active_slot_ids, + in.tree_sizes ? in.parent_ids : nullptr, + in.tree_sizes, + in.tree_width, + in.tree_scratch_base, + in.tree_scratch_stride, + cache.max_ctx); + } if (want_q_cap && q_fa) { // Last token's Q, all heads: src [head_dim, 1, n_head] view of // [head_dim, n_tokens, n_head]; dst = q_cap plane fa_idx @@ -2445,25 +2702,32 @@ QwenGraphOutputs build_qwen35_graph( ssm_st->nb[1], ssm_st->nb[2], ssm_st->nb[3], (size_t)in.seq_slot * ssm_st->nb[3]); } - cur = build_delta_net_block(ctx, gf, w, L, cur, - conv_st, ssm_st, - n_tokens, cap_ptr, in.parent_ids, - /*skip_gdn_intermediate=*/true, - supports_qwen35_fused_kernels(cache.backend), - in.n_seqs, - in.prefill_segments, - in.n_prefill_segments, - in.active_slot_ids, - in.state_slot_ids, - in.mapped_ar_seqs, - /*allow_inplace_state=*/ - in.n_prefill_tokens == 0, - in.specla_m_strict, in.specla_m_incl, - in.specla_m_eye, in.specla_hld, - in.specla_n_boundaries, - in.specla_n_chains, - in.specla_n_waves, - in.specla_max_parallel_chains); + if (w.is_bailingmoe3) { + GGML_ASSERT(!cap_ptr && !in.parent_ids && in.n_seqs == 1 && + in.n_prefill_segments == 0 && !in.active_slot_ids); + cur = build_bailingmoe3_kda_block( + ctx, gf, w, L, cur, conv_st, ssm_st, n_tokens); + } else { + cur = build_delta_net_block(ctx, gf, w, L, cur, + conv_st, ssm_st, + n_tokens, cap_ptr, in.parent_ids, + /*skip_gdn_intermediate=*/true, + supports_qwen35_fused_kernels(cache.backend), + in.n_seqs, + in.prefill_segments, + in.n_prefill_segments, + in.active_slot_ids, + in.state_slot_ids, + in.mapped_ar_seqs, + /*allow_inplace_state=*/ + in.n_prefill_tokens == 0, + in.specla_m_strict, in.specla_m_incl, + in.specla_m_eye, in.specla_hld, + in.specla_n_boundaries, + in.specla_n_chains, + in.specla_n_waves, + in.specla_max_parallel_chains); + } dn_idx++; } @@ -2476,9 +2740,10 @@ QwenGraphOutputs build_qwen35_graph( // FFN (dense SwiGLU for qwen35, MoE for qwen35moe) ggml_tensor * moe_selected = nullptr; - ggml_tensor * ffn = w.is_moe ? build_qwen35moe_ffn(ctx, post, w, L, - in.capture_moe_router ? &moe_selected : nullptr) - : build_swiglu_ffn(ctx, post, L); + ggml_tensor * ffn = L.ffn_gate_inp + ? build_qwen35moe_ffn(ctx, post, w, L, + in.capture_moe_router ? &moe_selected : nullptr) + : build_swiglu_ffn(ctx, post, L); if (in.capture_moe_router && moe_selected) { ggml_set_output(moe_selected); og_early.moe_selected[(size_t)il] = moe_selected; @@ -2669,30 +2934,39 @@ QwenLayerPrefnOutputs build_qwen35_layer_prefn( for (int il = 0; il < layer_idx; il++) { if (((il + 1) % w.full_attention_interval) == 0) fa_idx++; } - cur = build_full_attn_block(ctx, gf, w, L, cur, positions, w.rope_sections, - cache.attn_k[fa_idx], cache.attn_v[fa_idx], - attn_mask, kv_start, n_tokens, - cache.kv_k_type, cache.kv_v_type, - cache.kv_k_rotated, - fa_window, - /*q_tail_capture=*/nullptr, /*q_tail_start=*/0, - kv_write_rows); + cur = w.is_bailingmoe3 + ? build_bailingmoe3_mla_block( + ctx, gf, w, L, cur, positions, + cache.attn_k[fa_idx], cache.attn_v[fa_idx], + attn_mask, kv_start, n_tokens) + : build_full_attn_block( + ctx, gf, w, L, cur, positions, w.rope_sections, + cache.attn_k[fa_idx], cache.attn_v[fa_idx], + attn_mask, kv_start, n_tokens, + cache.kv_k_type, cache.kv_v_type, + cache.kv_k_rotated, fa_window, + /*q_tail_capture=*/nullptr, /*q_tail_start=*/0, + kv_write_rows); } else { int dn_idx = 0; for (int il = 0; il < layer_idx; il++) { if (((il + 1) % w.full_attention_interval) != 0) dn_idx++; } - cur = build_delta_net_block(ctx, gf, w, L, cur, - cache.conv_state[dn_idx], cache.ssm_state[dn_idx], - n_tokens, nullptr, nullptr, - skip_gdn_intermediate, - supports_qwen35_fused_kernels(cache.backend)); + cur = w.is_bailingmoe3 + ? build_bailingmoe3_kda_block( + ctx, gf, w, L, cur, cache.conv_state[dn_idx], + cache.ssm_state[dn_idx], n_tokens) + : build_delta_net_block( + ctx, gf, w, L, cur, + cache.conv_state[dn_idx], cache.ssm_state[dn_idx], + n_tokens, nullptr, nullptr, skip_gdn_intermediate, + supports_qwen35_fused_kernels(cache.backend)); } cur = ggml_add(ctx, cur, inpSA); out.residual = cur; out.post = rms_norm_mul(ctx, cur, L.attn_post_norm, eps); - if (w.is_moe) { + if (L.ffn_gate_inp) { // selected/weights are read back by the host (hybrid hot/cold expert // compute), not consumed in-graph. argsort_top_k yields a strided view // whose raw packed readback returns garbage ids for tokens > 0 (crash diff --git a/server/src/qwen35moe/qwen35moe_ffn.cpp b/server/src/qwen35moe/qwen35moe_ffn.cpp index a71a94a4a..513e46233 100644 --- a/server/src/qwen35moe/qwen35moe_ffn.cpp +++ b/server/src/qwen35moe/qwen35moe_ffn.cpp @@ -30,6 +30,51 @@ Qwen35MoeRouterOutputs build_qwen35moe_router( break; } + // Bailing/DeepSeek correction bias changes expert selection only. The + // selected experts retain their unbiased sigmoid weights. + ggml_tensor * selection_probs = probs; + if (L.ffn_exp_probs_b) { + selection_probs = ggml_add(ctx, probs, L.ffn_exp_probs_b); + } + + // Ling 3 routes hierarchically: rank each group by the sum of its two + // strongest experts, keep the configured top groups, then select the + // final experts from that masked set. This is the DeepSeek-V3 grouping + // rule used by llama.cpp's generic MoE builder. + if (w.n_expert_groups > 1) { + GGML_ASSERT(n_expert % w.n_expert_groups == 0); + GGML_ASSERT(w.n_expert_groups_used > 0 && + w.n_expert_groups_used <= w.n_expert_groups); + const int n_expert_per_group = n_expert / w.n_expert_groups; + ggml_tensor * selection_groups = ggml_reshape_3d( + ctx, selection_probs, n_expert_per_group, + w.n_expert_groups, n_tokens); + + ggml_tensor * group_scores = ggml_argsort_top_k( + ctx, selection_groups, 2); + group_scores = ggml_get_rows( + ctx, + ggml_reshape_4d(ctx, selection_groups, 1, + n_expert_per_group, w.n_expert_groups, + n_tokens), + group_scores); + group_scores = ggml_sum_rows( + ctx, ggml_reshape_3d(ctx, group_scores, 2, + w.n_expert_groups, n_tokens)); + group_scores = ggml_reshape_2d( + ctx, group_scores, w.n_expert_groups, n_tokens); + + ggml_tensor * selected_groups = ggml_argsort_top_k( + ctx, group_scores, w.n_expert_groups_used); + ggml_tensor * kept_groups = ggml_get_rows( + ctx, selection_groups, selected_groups); + selection_groups = ggml_set_rows( + ctx, ggml_fill(ctx, selection_groups, -INFINITY), + kept_groups, selected_groups); + selection_probs = ggml_reshape_2d( + ctx, selection_groups, n_expert, n_tokens); + } + // ggml_argsort_top_k emits GGML_OP_ARGSORT (+view), which ggml-cuda's // topk-moe fusion (ggml_cuda_topk_moe_fusion) recognizes and fuses the whole // softmax->topk->get_rows->norm router into ~1 kernel. ggml_top_k emits @@ -38,18 +83,16 @@ Qwen35MoeRouterOutputs build_qwen35moe_router( // Same top-k selection -> bit-identical. DFLASH_NO_MOE_ROUTER_FUSE=1 = old path. static const bool router_fuse = (std::getenv("DFLASH_NO_MOE_ROUTER_FUSE") == nullptr); ggml_tensor * selected = (router_fuse && allow_fused_router) - ? ggml_argsort_top_k(ctx, probs, n_used) - : ggml_top_k(ctx, probs, n_used); + ? ggml_argsort_top_k(ctx, selection_probs, n_used) + : ggml_top_k(ctx, selection_probs, n_used); ggml_tensor * probs_3d = ggml_reshape_3d(ctx, probs, 1, n_expert, n_tokens); ggml_tensor * weights = ggml_get_rows(ctx, probs_3d, selected); weights = ggml_reshape_2d(ctx, weights, n_used, n_tokens); - // Always normalize selected expert weights by their sum (matches - // llama.cpp's norm_w=true for qwen35moe). Without this, top-k softmax - // weights sum to much less than 1.0, causing systematically underscaled - // FFN output. - { + // Qwen3.5 MoE and Ling 3 both ship norm_w=true. Keep the metadata switch + // explicit because Bailing-family checkpoints can legally disable it. + if (w.expert_weights_norm) { ggml_tensor * w_sum = ggml_sum_rows(ctx, weights); w_sum = ggml_clamp(ctx, w_sum, 6.103515625e-5f, INFINITY); weights = ggml_div(ctx, weights, w_sum); @@ -95,7 +138,7 @@ ggml_tensor * build_qwen35moe_ffn( if (L.ffn_gate_up_exps) { ggml_tensor * gate_up_e = apply_scale2( ctx, ggml_mul_mat_id(ctx, L.ffn_gate_up_exps, cur_3d, selected), L.ffn_gate_up_exps_s); - if (moe_swiglu_fuse) { + if (moe_swiglu_fuse && L.ffn_swiglu_clamp_exp <= 0.0f) { gu = ggml_swiglu(ctx, gate_up_e); // silu(gate) * up, no views/conts } else { ggml_tensor * gate_e = ggml_view_3d(ctx, gate_up_e, @@ -107,14 +150,30 @@ ggml_tensor * build_qwen35moe_ffn( (size_t)n_ff_exp * ggml_element_size(gate_up_e)); gate_e = ggml_cont(ctx, gate_e); up_e = ggml_cont(ctx, up_e); - gu = ggml_swiglu_split(ctx, gate_e, up_e); + if (L.ffn_swiglu_clamp_exp > 0.0f) { + const float limit = L.ffn_swiglu_clamp_exp; + up_e = ggml_clamp(ctx, up_e, -limit, limit); + gate_e = ggml_clamp( + ctx, ggml_silu(ctx, gate_e), -INFINITY, limit); + gu = ggml_mul(ctx, gate_e, up_e); + } else { + gu = ggml_swiglu_split(ctx, gate_e, up_e); + } } } else { ggml_tensor * gate_e = apply_scale2( ctx, ggml_mul_mat_id(ctx, L.ffn_gate_exps, cur_3d, selected), L.ffn_gate_exps_s); ggml_tensor * up_e = apply_scale2( ctx, ggml_mul_mat_id(ctx, L.ffn_up_exps, cur_3d, selected), L.ffn_up_exps_s); - gu = ggml_swiglu_split(ctx, gate_e, up_e); + if (L.ffn_swiglu_clamp_exp > 0.0f) { + const float limit = L.ffn_swiglu_clamp_exp; + up_e = ggml_clamp(ctx, up_e, -limit, limit); + gate_e = ggml_clamp( + ctx, ggml_silu(ctx, gate_e), -INFINITY, limit); + gu = ggml_mul(ctx, gate_e, up_e); + } else { + gu = ggml_swiglu_split(ctx, gate_e, up_e); + } } ggml_tensor * experts = apply_scale2( @@ -132,7 +191,16 @@ ggml_tensor * build_qwen35moe_ffn( ctx, ggml_mul_mat(ctx, L.ffn_gate_shexp, cur), L.ffn_gate_shexp_s); ggml_tensor * sh_up = apply_scale2( ctx, ggml_mul_mat(ctx, L.ffn_up_shexp, cur), L.ffn_up_shexp_s); - ggml_tensor * sh_gu = ggml_swiglu_split(ctx, sh_gate, sh_up); + ggml_tensor * sh_gu = nullptr; + if (L.ffn_swiglu_clamp_shexp > 0.0f) { + const float limit = L.ffn_swiglu_clamp_shexp; + sh_up = ggml_clamp(ctx, sh_up, -limit, limit); + sh_gate = ggml_clamp( + ctx, ggml_silu(ctx, sh_gate), -INFINITY, limit); + sh_gu = ggml_mul(ctx, sh_gate, sh_up); + } else { + sh_gu = ggml_swiglu_split(ctx, sh_gate, sh_up); + } ggml_tensor * shared = apply_scale2( ctx, ggml_mul_mat(ctx, L.ffn_down_shexp, sh_gu), L.ffn_down_shexp_s); diff --git a/server/src/server/chat_template.cpp b/server/src/server/chat_template.cpp index b9cf36eff..63f644649 100644 --- a/server/src/server/chat_template.cpp +++ b/server/src/server/chat_template.cpp @@ -67,6 +67,7 @@ ChatFormat chat_format_for_arch(const std::string & arch) { if (arch == "deepseek4") return ChatFormat::DEEPSEEK4; if (arch == "laguna") return ChatFormat::LAGUNA; if (arch == "gemma4") return ChatFormat::GEMMA4; + if (arch == "bailingmoe3") return ChatFormat::BAILINGMOE3; // qwen35, qwen3 use the Qwen3/ChatML format return ChatFormat::QWEN3; } @@ -83,6 +84,111 @@ std::string render_chat_template( bool has_tools = !tools_json.empty() && tools_json != "[]" && tools_json != "null"; switch (format) { + case ChatFormat::BAILINGMOE3: { + // AntLing's shipped Bailing V3 template. The system turn is always + // present because it carries the model's thinking-mode directive. + const bool has_system = + !messages.empty() && messages[0].role == "system"; + const std::string system_content = has_system + ? messages[0].content : std::string(); + const bool system_sets_thinking = + system_content.find("detailed thinking on") != std::string::npos || + system_content.find("detailed thinking off") != std::string::npos; + const char * thinking_option = enable_thinking ? "on" : "off"; + + result += "SYSTEM"; + if (has_tools) { + if (!system_content.empty()) { + result += system_content; + result += '\n'; + } + result += + "# Tools\n\n" + "You may call one or more functions to assist with the user query.\n\n" + "You are provided with function signatures within XML tags:\n" + ""; + try { + const nlohmann::json tools = nlohmann::json::parse(tools_json); + for (const auto & tool : tools) { + result += '\n'; + result += tool.dump(); + } + } catch (const std::exception &) { + result += '\n'; + result += tools_json; + } + result += + "\n\n\n" + "If none of the functions can be used, point it out. If the given question lacks the parameters required by the function, also point it out.\n" + "If you need to use a function, for each function call, output the function name and arguments within the following XML format:\n" + "{function-name}\n" + "{arg-key-1}\n" + "{arg-value-1}\n" + "{arg-key-2}\n" + "{arg-value-2}\n" + "...\n" + "\n"; + if (!system_sets_thinking) { + result += "detailed thinking "; + result += thinking_option; + } + result += "<|role_end|>"; + } else if (has_system) { + result += system_content; + if (!system_sets_thinking) { + result += '\n'; + result += "detailed thinking "; + result += thinking_option; + } + result += "<|role_end|>"; + } else { + result += "detailed thinking "; + result += thinking_option; + result += "<|role_end|>"; + } + + bool in_tool_response = false; + for (size_t i = has_system ? 1 : 0; i < messages.size(); ++i) { + const ChatMessage & msg = messages[i]; + if (msg.role == "user") { + result += "HUMAN"; + result += msg.content; + result += "<|role_end|>"; + } else if (msg.role == "system") { + result += "SYSTEM"; + result += msg.content; + result += "<|role_end|>"; + } else if (msg.role == "assistant") { + result += "ASSISTANT\n"; + if (msg.content.find("") == std::string::npos) { + result += ""; + } + result += msg.content; + result += "<|role_end|>"; + } else if (msg.role == "tool") { + if (!in_tool_response) { + result += "OBSERVATION"; + in_tool_response = true; + } + result += "\n\n"; + result += msg.content; + result += "\n"; + const bool next_is_tool = + i + 1 < messages.size() && messages[i + 1].role == "tool"; + if (!next_is_tool) { + result += "<|role_end|>"; + in_tool_response = false; + } + } + } + + if (add_generation_prompt) { + result += "ASSISTANT\n"; + if (!enable_thinking) result += ""; + } + break; + } + case ChatFormat::QWEN3: { // Qwen3/3.5 ChatML format: // <|im_start|>system\n[tool preamble +] content<|im_end|>\n diff --git a/server/src/server/chat_template.h b/server/src/server/chat_template.h index f93119906..c3f1f534b 100644 --- a/server/src/server/chat_template.h +++ b/server/src/server/chat_template.h @@ -3,7 +3,8 @@ // Renders chat messages (system/user/assistant/tool) into the model-specific // token format. Hard-coded for supported architectures: // - Qwen3/3.5: <|im_start|>role\ncontent<|im_end|>\n -// - Laguna: XML-style <|begin_of_sentence|><|User|>...<|Assistant|> +// - BailingMoE3: SYSTEM/HUMAN/ASSISTANT...<|role_end|> +// - Laguna: XML-style role blocks #pragma once @@ -23,6 +24,7 @@ struct ChatMessage { // Chat template format. enum class ChatFormat { QWEN3, // <|im_start|>role\n...<|im_end|>\n + BAILINGMOE3, // SYSTEM/HUMAN/ASSISTANT...<|role_end|> LAGUNA, // <|begin_of_sentence|><|User|>...<|Assistant|> GEMMA4, // <|turn>role\n...\n DEEPSEEK4, // <｜begin▁of▁sentence｜>...<｜User｜>...<｜Assistant｜> diff --git a/server/src/server/http_server.cpp b/server/src/server/http_server.cpp index a909344e8..388d11e3a 100644 --- a/server/src/server/http_server.cpp +++ b/server/src/server/http_server.cpp @@ -2244,6 +2244,18 @@ TokenDelivery classify_generated_token( return TokenDelivery::kThinkTag; } + // Bailing V3 stores its tool-call XML delimiters as added special + // tokens. They are part of the model's public output protocol, not chat + // control markers: keep them so the shared parser can reconstruct an + // OpenAI `tool_calls` object. Stripping them leaves only three unrelated + // text lines (function name, argument name, value). + if (raw == "" || raw == "" || + raw == "" || raw == "" || + raw == "" || raw == "") { + text = raw; + return TokenDelivery::kText; + } + // Other special tokens are internal control markers. Byte-fallback // tokens such as <0xAB> are text and must still reach the emitter. if (raw.size() >= 2 && raw[0] == '<' && raw[1] == '|') { diff --git a/server/src/server/model_card.cpp b/server/src/server/model_card.cpp index 522963572..6c272c36f 100644 --- a/server/src/server/model_card.cpp +++ b/server/src/server/model_card.cpp @@ -278,6 +278,20 @@ static bool family_fallback(const std::string & arch, ModelCard & out) { out.source_label = "family:" + arch; return true; } + if (arch == "bailingmoe3") { + out.max_tokens = 32768; + out.complex_problem_max_tokens = 0; + out.hard_limit_reply_budget = 4096; + // Official Ling 3.0 Flash sampling defaults. + out.sampling.temperature = 0.6f; + out.sampling.top_p = 0.95f; + out.sampling.top_k = 20; + out.sampling.has_temperature = true; + out.sampling.has_top_p = true; + out.sampling.has_top_k = true; + out.source_label = "family:bailingmoe3"; + return true; + } if (arch == "gemma4") { // Gemma4 verified value: see Gemma model card; conservative // 16384 keeps us inside published recommendations. diff --git a/server/test/test_server_unit.cpp b/server/test/test_server_unit.cpp index 698b865be..6681c8536 100644 --- a/server/test/test_server_unit.cpp +++ b/server/test/test_server_unit.cpp @@ -37,6 +37,7 @@ #include "common/gguf_bounds.h" #include "common/gguf_inspect.h" #include "qwen35/prefill_helpers.h" +#include "qwen35moe/qwen35moe_ffn.h" #include "ggml-cpu.h" #include "server/prompt_normalize.h" #include "qwen3_drafter_model.h" @@ -53,6 +54,7 @@ #include #include #include +#include #include #include #include @@ -3401,6 +3403,85 @@ TEST_CASE(ServerUnitFixture, test_deepseek4_render_reasoning_effort_prefixes) { TEST_ASSERT(!ends_with(completed_turn, "<｜Assistant｜>")); } +TEST_CASE(ServerUnitFixture, test_bailingmoe3_render_official_role_format) { + TEST_ASSERT(chat_format_for_arch("bailingmoe3") == ChatFormat::BAILINGMOE3); + const std::vector msgs = {{"user", "Hello", ""}}; + const std::string out = render_chat_template( + msgs, ChatFormat::BAILINGMOE3, + /*add_generation_prompt=*/true, + /*enable_thinking=*/false, + /*tools_json=*/""); + TEST_ASSERT(out == + "SYSTEMdetailed thinking off<|role_end|>" + "HUMANHello<|role_end|>" + "ASSISTANT\n"); +} + +TEST_CASE(ServerUnitFixture, test_bailingmoe3_render_thinking_and_tools) { + const std::vector msgs = { + {"system", "Be concise.", ""}, + {"user", "Check Rome", ""}, + }; + const std::string tools = + R"([{"type":"function","function":{"name":"weather","parameters":{"type":"object"}}}])"; + const std::string out = render_chat_template( + msgs, ChatFormat::BAILINGMOE3, + /*add_generation_prompt=*/true, + /*enable_thinking=*/true, + tools); + TEST_ASSERT(out.find("SYSTEMBe concise.\n# Tools") == 0); + TEST_ASSERT(out.find("\n{\"function\":") != std::string::npos); + TEST_ASSERT(out.find("detailed thinking on<|role_end|>") != std::string::npos); + TEST_ASSERT(out.find("HUMANCheck Rome<|role_end|>") != std::string::npos); + const std::string suffix = "ASSISTANT\n"; + TEST_ASSERT(out.size() >= suffix.size()); + TEST_ASSERT(out.compare(out.size() - suffix.size(), suffix.size(), suffix) == 0); +} + +TEST_CASE(ServerUnitFixture, test_bailingmoe3_router_builds_group_mask) { + ggml_init_params params{}; + params.mem_size = 2 * 1024 * 1024; + params.no_alloc = true; + ggml_context * ctx = ggml_init(params); + TEST_ASSERT(ctx != nullptr); + + TargetWeights weights; + weights.n_expert = 512; + weights.n_expert_used = 8; + weights.n_expert_groups = 8; + weights.n_expert_groups_used = 4; + weights.expert_gating_func = 2; + weights.expert_weights_norm = true; + weights.expert_weights_scale = 2.5f; + + TargetLayer layer; + layer.ffn_gate_inp = ggml_new_tensor_2d( + ctx, GGML_TYPE_F32, 16, weights.n_expert); + layer.ffn_exp_probs_b = ggml_new_tensor_1d( + ctx, GGML_TYPE_F32, weights.n_expert); + ggml_tensor * input = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 16, 2); + + const Qwen35MoeRouterOutputs router = build_qwen35moe_router( + ctx, input, weights, layer); + TEST_ASSERT(router.selected != nullptr); + TEST_ASSERT(router.weights != nullptr); + + std::unordered_set visited; + const auto contains_op = [&](const auto & self, + const ggml_tensor * tensor, + ggml_op op) -> bool { + if (!tensor || !visited.insert(tensor).second) return false; + if (tensor->op == op) return true; + for (const ggml_tensor * source : tensor->src) { + if (self(self, source, op)) return true; + } + return false; + }; + TEST_ASSERT(contains_op(contains_op, router.selected, GGML_OP_SET_ROWS)); + + ggml_free(ctx); +} + TEST_CASE(ServerUnitFixture, test_jinja_render_basic) { std::vector msgs = { {"system", "you are helpful", ""}, @@ -5397,6 +5478,18 @@ TEST_CASE(ServerUnitFixture, test_model_card_family_fallback_deepseek4) { TEST_ASSERT(unknown.source_label != "family:not-a-real-arch"); } +TEST_CASE(ServerUnitFixture, test_model_card_family_fallback_bailingmoe3) { + auto card = dflash::common::resolve_model_card("", "", "bailingmoe3", ""); + TEST_ASSERT(card.source_label == "family:bailingmoe3"); + TEST_ASSERT(card.max_tokens == 32768); + TEST_ASSERT(card.sampling.has_temperature); + TEST_ASSERT(std::abs(card.sampling.temperature - 0.6f) < 1.0e-6f); + TEST_ASSERT(card.sampling.has_top_p); + TEST_ASSERT(std::abs(card.sampling.top_p - 0.95f) < 1.0e-6f); + TEST_ASSERT(card.sampling.has_top_k); + TEST_ASSERT(card.sampling.top_k == 20); +} + TEST_CASE(ServerUnitFixture, test_props_model_card_wholesale_sidecar) { // When a sidecar was loaded, /props.model_card should be the parsed // sidecar JSON verbatim — *all* fields from the file, not just the @@ -6407,6 +6500,10 @@ TEST_CASE(ServerUnitFixture, test_qwen35_embedded_mtp_target_layer_count) { "qwen35moe", 81, 1, target_layers, error)); TEST_ASSERT(target_layers == 80); + TEST_ASSERT(derive_effective_target_layer_count( + "bailingmoe3", 43, 1, target_layers, error)); + TEST_ASSERT(target_layers == 42); + TEST_ASSERT(!derive_effective_target_layer_count( "qwen35", 1, 1, target_layers, error)); TEST_ASSERT(error.find("smaller than block_count") != std::string::npos); From d7b4795cc3f4f3ee33effcb366fd99dfd2c9e501 Mon Sep 17 00:00:00 2001 From: mrciffa <49000955+davide221@users.noreply.github.com> Date: Fri, 28 Aug 2026 23:32:35 +0200 Subject: [PATCH 2/5] chore: keep Ling integration PR focused --- server/docs/LING3_FLASH_DGX_SPARK.md | 202 ------------------- server/scripts/bench_ling3_flash.py | 291 --------------------------- 2 files changed, 493 deletions(-) delete mode 100644 server/docs/LING3_FLASH_DGX_SPARK.md delete mode 100644 server/scripts/bench_ling3_flash.py diff --git a/server/docs/LING3_FLASH_DGX_SPARK.md b/server/docs/LING3_FLASH_DGX_SPARK.md deleted file mode 100644 index 1ceff425a..000000000 --- a/server/docs/LING3_FLASH_DGX_SPARK.md +++ /dev/null @@ -1,202 +0,0 @@ -# Ling 3.0 Flash on DGX Spark - -LuceBox has an experimental native backend for AntLing's 124B-parameter -`BailingMoE3` model, Ling 3.0 Flash. This is a LuceBox graph implementation, -not a proxy to vLLM or llama.cpp. - -The first tested configuration serves a Q4_K_M GGUF on one NVIDIA DGX Spark -at 34.6 generated tokens/s. It also renders the official chat template, -emits OpenAI-compatible tool calls, consumes tool results, and returns a final -answer. - -## Tested configuration - -| Item | Value | -|---|---| -| Machine | NVIDIA DGX Spark | -| GPU | NVIDIA GB10, `sm_121` | -| Host architecture | `aarch64` | -| Unified memory | 121 GiB visible to the OS | -| Driver / CUDA | 580.173.02 / CUDA 13.0 | -| Model | `inclusionAI/Ling-3.0-flash` | -| GGUF | `Ling-3.0-flash-Q4_K_M.gguf`, 72.9 GiB | -| Weight quantization | Q4_K_M | -| KV cache | Q4_0 keys and values | -| Server context | 32,768 tokens | -| Request concurrency | One active decode slot | -| Prefix cache | Disabled for measurement | -| Benchmark date | 2026-08-24 | - -The GGUF used for this bring-up came from -[`bloomer010/Ling-3.0-flash-GGUF`](https://huggingface.co/bloomer010/Ling-3.0-flash-GGUF). -The official model and tokenizer are at -[`inclusionAI/Ling-3.0-flash`](https://huggingface.co/inclusionAI/Ling-3.0-flash). - -## Build and serve - -Download the tested quantization: - -```bash -hf download bloomer010/Ling-3.0-flash-GGUF \ - Ling-3.0-flash-Q4_K_M.gguf \ - --local-dir models/ling-3.0-flash-gguf -``` - -Build the CUDA server for GB10: - -```bash -cmake -S server -B server/build-ling3-cuda \ - -DCMAKE_BUILD_TYPE=Release \ - -DDFLASH27B_GPU_BACKEND=cuda \ - -DGGML_NATIVE=OFF \ - -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc \ - -DCMAKE_CUDA_ARCHITECTURES=121 - -cmake --build server/build-ling3-cuda \ - --target dflash_server -j"$(nproc)" -``` - -Start the measured configuration: - -```bash -server/build-ling3-cuda/dflash_server \ - models/ling-3.0-flash-gguf/Ling-3.0-flash-Q4_K_M.gguf \ - --host 127.0.0.1 \ - --port 18081 \ - --max-ctx 32768 \ - --default-max-tokens 256 \ - --cache-type-k q4_0 \ - --cache-type-v q4_0 \ - --prefix-cache-slots 0 \ - --disk-prefix-cache off \ - --model-name ling-3.0-flash-lucebox -``` - -The server exposes the OpenAI-compatible `/v1/chat/completions` route. To use -a Spark reached over SSH without exposing the API on the network, create a -local tunnel: - -```bash -ssh -N -L 18081:127.0.0.1:18081 @ -``` - -Then send requests to `http://127.0.0.1:18081` on the client machine. - -## Benchmark method - -The benchmark uses [`bench_ling3_flash.py`](../scripts/bench_ling3_flash.py), -which depends only on the Python standard library. It records the timings -reported by the LuceBox server and checks output determinism. Prefix caching is -disabled, temperature is zero, and concurrency is reported separately from -single-request decode. - -```bash -python3 server/scripts/bench_ling3_flash.py \ - --url http://127.0.0.1:18081 \ - --model ling-3.0-flash-lucebox \ - --weights-quant Q4_K_M \ - --kv-quant Q4_0 \ - --server-max-context 32768 \ - --decode-tokens 128 \ - --decode-runs 3 \ - --warmups 1 \ - --context-tokens 256,1024,4096,8192 \ - --context-runs 2 \ - --concurrency-levels 1,2 \ - --concurrency-tokens 64 \ - --output ling3-flash-lucebox-bench.json -``` - -### Decode - -The decode prompt requests a deterministic repeated-token sequence and each -measured run generates 128 tokens. All three measured responses have the same -SHA-256 digest. - -| Run | Prompt tokens | Output tokens | Prefill | Decode | -|---:|---:|---:|---:|---:| -| 1 | 57 | 128 | 157.8 ms | 34.6 tok/s | -| 2 | 57 | 128 | 157.4 ms | 34.6 tok/s | -| 3 | 57 | 128 | 157.2 ms | 34.5 tok/s | -| **Median** | **57** | **128** | **157.4 ms** | **34.6 tok/s** | - -### Context prefill - -Each row is the median of two uncached requests. The response is restricted to -one token so the measurement isolates prompt evaluation. - -| Actual prompt tokens | Median prefill | Median prefill rate | -|---:|---:|---:| -| 267 | 277.55 ms | 962.0 tok/s | -| 1,035 | 810.80 ms | 1,277.0 tok/s | -| 4,107 | 2,642.15 ms | 1,554.4 tok/s | -| 8,203 | 5,238.60 ms | 1,565.9 tok/s | - -The model advertises a 262,144-token context. This run capped the server at -32,768 and exercised prompt depth only through 8,203 tokens, so it is not a -claim of validated 262K behavior. - -### Concurrency - -| Submitted requests | Total output tokens | Wall time | Aggregate rate | -|---:|---:|---:|---:| -| 1 | 64 | 2.041 s | 31.35 tok/s | -| 2 | 128 | 4.033 s | 31.74 tok/s | - -Two simultaneous HTTP requests complete safely, but the current LuceBox -backend queues them through one active decode slot. The second request took -4.032 seconds, versus 2.016 seconds for the first. This is serialization, not -continuous batching, and throughput does not scale with request concurrency. - -### Tool-call round trip - -The test gives the model a `get_weather(location)` function and asks for the -current weather in Rome. The model emits exactly one structured call: - -```json -{"name":"get_weather","arguments":{"location":"Rome"}} -``` - -The harness supplies a deterministic fixture: - -```json -{"temperature_c":29,"condition":"sunny","source":"test fixture"} -``` - -Ling then returns: - -> The current weather in Rome is sunny with a temperature of 29°C. - -The call completed with `finish_reason: "tool_calls"`; the final answer used -both fixture values. Decode measured 35.0 tok/s for the tool call and 34.9 -tok/s for the final answer. - -## Correctness checks - -The backend implements Ling's mixed 35-layer KDA / 7-layer MLA attention, -512-expert MoE, top-8 routing, and grouped expert selection. The embedded MTP -layer is recognized and excluded from the autoregressive target graph. - -For an oracle check, the same GGUF and exact chat prompt were run against the -Ling support branch proposed to llama.cpp. LuceBox and the oracle match the -opening text through `Running an open-weight language model locally offers`. -Their first different greedy token was a near tie: oracle `greater` at --0.89073 log probability and LuceBox `three`, with oracle `three` at -0.96373. -That 0.073-natural-log-unit margin is consistent with GPU kernel numerical -drift rather than a prompt-template or graph-structure mismatch. A short exact -answer test produced `PARIS` in both engines. - -The full LuceBox CTest suite passes: 477 tests, zero failures. Four tests that -require unavailable model assets or NCCL are skipped by configuration. - -## Current verdict - -Use this backend today for single-user, local Ling 3.0 Flash evaluation on a -DGX Spark, especially chat, tool-use, and agent experiments that need an -OpenAI-compatible endpoint. The 73 GiB Q4_K_M artifact fits comfortably in the -Spark's unified memory, and 34.6 tok/s is interactive. - -Do not use this baseline yet for a multi-user serving claim, a validated 262K -context claim, or a headline comparison against another runtime. Continuous -batching, full-depth context validation, the model's MTP speculative path, and -matched-condition cross-runtime benchmarks remain follow-up work. diff --git a/server/scripts/bench_ling3_flash.py b/server/scripts/bench_ling3_flash.py deleted file mode 100644 index 8f19fb971..000000000 --- a/server/scripts/bench_ling3_flash.py +++ /dev/null @@ -1,291 +0,0 @@ -#!/usr/bin/env python3 -"""Reproducible Ling 3.0 Flash checks against a running dflash_server. - -The script uses only the Python standard library. It records server-reported -prefill/decode timings, a short context-depth sweep, request-level concurrency, -and a real OpenAI-style tool-call round trip. -""" - -from __future__ import annotations - -import argparse -import concurrent.futures -import hashlib -import json -import statistics -import time -import urllib.error -import urllib.request -from pathlib import Path - - -DECODE_PROMPT = ( - "Continue this exact sequence until the output limit. Output only the word " - "BETA separated by single spaces: BETA BETA BETA BETA BETA BETA BETA BETA" -) - - -def post_json(url: str, payload: dict, timeout: float) -> dict: - request = urllib.request.Request( - url, - data=json.dumps(payload, separators=(",", ":")).encode(), - headers={"Content-Type": "application/json"}, - ) - with urllib.request.urlopen(request, timeout=timeout) as response: - return json.load(response) - - -def chat(base_url: str, model: str, messages: list[dict], max_tokens: int, - timeout: float, *, tools: list[dict] | None = None) -> dict: - payload: dict = { - "model": model, - "messages": messages, - "temperature": 0, - "max_tokens": max_tokens, - "stream": False, - } - if tools is not None: - payload["tools"] = tools - started = time.monotonic() - result = post_json( - f"{base_url.rstrip('/')}/v1/chat/completions", payload, timeout) - result["_wall_seconds"] = time.monotonic() - started - return result - - -def measurement(result: dict) -> dict: - usage = result["usage"] - timings = usage.get("timings", {}) - content = result["choices"][0]["message"].get("content") or "" - return { - "prompt_tokens": usage["prompt_tokens"], - "completion_tokens": usage["completion_tokens"], - "prefill_ms": timings.get("prefill_ms"), - "prefill_tokens": timings.get( - "effective_prompt_tokens", usage["prompt_tokens"]), - "decode_ms": timings.get("decode_ms"), - "decode_tokens_per_second": timings.get("decode_tokens_per_sec"), - "cache_hit": timings.get("cache_hit"), - "cached_prefix_tokens": timings.get("cached_prefix_tokens", 0), - "wall_seconds": result["_wall_seconds"], - "finish_reason": result["choices"][0].get("finish_reason"), - "output_sha256": hashlib.sha256(content.encode()).hexdigest(), - "output_excerpt": content[:160], - } - - -def median(values: list[float | int | None]) -> float | None: - present = [float(value) for value in values if value is not None] - return statistics.median(present) if present else None - - -def run_decode(args: argparse.Namespace) -> dict: - for _ in range(args.warmups): - chat(args.url, args.model, - [{"role": "user", "content": DECODE_PROMPT}], - args.decode_tokens, args.timeout) - - runs = [ - measurement(chat( - args.url, args.model, - [{"role": "user", "content": DECODE_PROMPT}], - args.decode_tokens, args.timeout)) - for _ in range(args.decode_runs) - ] - return { - "prompt": DECODE_PROMPT, - "temperature": 0, - "max_tokens": args.decode_tokens, - "warmups": args.warmups, - "runs": runs, - "median_decode_tokens_per_second": median( - [run["decode_tokens_per_second"] for run in runs]), - "median_prefill_ms": median([run["prefill_ms"] for run in runs]), - } - - -def synthetic_context(target_tokens: int, run: int) -> str: - # " alpha" is one token in the Ling tokenizer. The fixed chat framing and - # instruction account for the small difference between target and actual. - body = "alpha " * max(1, target_tokens - 32) - return ( - "Read this synthetic context, then reply with only OK.\n" - f"{body}\nBenchmark run {run}; reply OK." - ) - - -def run_context_sweep(args: argparse.Namespace) -> list[dict]: - rows = [] - for target in args.context_tokens: - runs = [] - for run_index in range(args.context_runs): - result = chat( - args.url, args.model, - [{"role": "user", - "content": synthetic_context(target, run_index)}], - 1, args.timeout) - row = measurement(result) - if row["prefill_ms"]: - row["prefill_tokens_per_second"] = ( - row["prefill_tokens"] / (row["prefill_ms"] / 1000.0)) - else: - row["prefill_tokens_per_second"] = None - runs.append(row) - rows.append({ - "target_tokens": target, - "runs": runs, - "actual_prompt_tokens": int(median( - [run["prompt_tokens"] for run in runs]) or 0), - "median_prefill_ms": median([run["prefill_ms"] for run in runs]), - "median_prefill_tokens_per_second": median( - [run["prefill_tokens_per_second"] for run in runs]), - }) - return rows - - -def run_concurrency(args: argparse.Namespace) -> list[dict]: - rows = [] - for level in args.concurrency_levels: - started = time.monotonic() - with concurrent.futures.ThreadPoolExecutor(max_workers=level) as pool: - futures = [ - pool.submit( - chat, args.url, args.model, - [{"role": "user", "content": - f"{DECODE_PROMPT}\nConcurrent request {index}."}], - args.concurrency_tokens, args.timeout) - for index in range(level) - ] - runs = [measurement(future.result()) for future in futures] - wall_seconds = time.monotonic() - started - completion_tokens = sum(run["completion_tokens"] for run in runs) - rows.append({ - "concurrency": level, - "max_tokens_per_request": args.concurrency_tokens, - "wall_seconds": wall_seconds, - "completion_tokens": completion_tokens, - "aggregate_completion_tokens_per_second": ( - completion_tokens / wall_seconds if wall_seconds else None), - "requests": runs, - }) - return rows - - -def run_tool_round_trip(args: argparse.Namespace) -> dict: - tools = [{ - "type": "function", - "function": { - "name": "get_weather", - "description": "Get current weather for a city.", - "parameters": { - "type": "object", - "properties": { - "location": {"type": "string"}, - }, - "required": ["location"], - "additionalProperties": False, - }, - }, - }] - user_message = { - "role": "user", - "content": "What is the current weather in Rome? Use the tool.", - } - first = chat( - args.url, args.model, [user_message], 96, args.timeout, tools=tools) - choice = first["choices"][0] - assistant = choice["message"] - calls = assistant.get("tool_calls") or [] - if choice.get("finish_reason") != "tool_calls" or len(calls) != 1: - raise RuntimeError(f"expected one tool call, got: {choice}") - function = calls[0]["function"] - arguments = json.loads(function["arguments"]) - if function["name"] != "get_weather" or "rome" not in ( - arguments.get("location") or "").lower(): - raise RuntimeError(f"unexpected tool call: {function}") - - tool_message = { - "role": "tool", - "tool_call_id": calls[0]["id"], - "content": json.dumps({ - "temperature_c": 29, - "condition": "sunny", - "source": "test fixture", - }, separators=(",", ":")), - } - second = chat( - args.url, args.model, - [user_message, assistant, tool_message], - 96, args.timeout, tools=tools) - answer = second["choices"][0]["message"].get("content") or "" - if "29" not in answer or "sunny" not in answer.lower(): - raise RuntimeError(f"tool result was not used in the answer: {answer}") - return { - "call_finish_reason": choice.get("finish_reason"), - "tool_name": function["name"], - "arguments": arguments, - "call_measurement": measurement(first), - "answer": answer, - "answer_measurement": measurement(second), - "passed": True, - } - - -def csv_ints(value: str) -> list[int]: - values = [int(item) for item in value.split(",") if item.strip()] - if not values or any(item <= 0 for item in values): - raise argparse.ArgumentTypeError("expected comma-separated positive integers") - return values - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--url", default="http://127.0.0.1:18081") - parser.add_argument("--model", default="ling-3.0-flash-lucebox") - parser.add_argument("--weights-quant", default="Q4_K_M") - parser.add_argument("--kv-quant", default="Q4_0") - parser.add_argument("--server-max-context", type=int, default=32768) - parser.add_argument("--decode-tokens", type=int, default=128) - parser.add_argument("--decode-runs", type=int, default=3) - parser.add_argument("--warmups", type=int, default=1) - parser.add_argument("--context-tokens", type=csv_ints, - default=csv_ints("256,1024,4096,8192")) - parser.add_argument("--context-runs", type=int, default=2) - parser.add_argument("--concurrency-levels", type=csv_ints, - default=csv_ints("1,2")) - parser.add_argument("--concurrency-tokens", type=int, default=64) - parser.add_argument("--timeout", type=float, default=900.0) - parser.add_argument("--output", type=Path) - args = parser.parse_args() - - try: - with urllib.request.urlopen( - f"{args.url.rstrip('/')}/health", timeout=10) as response: - health = response.read().decode() - report = { - "engine": "LuceBox dflash_server", - "model": args.model, - "weights_quant": args.weights_quant, - "kv_quant": args.kv_quant, - "server_max_context": args.server_max_context, - "health": health, - "decode": run_decode(args), - "context_sweep": run_context_sweep(args), - "concurrency": run_concurrency(args), - "tool_round_trip": run_tool_round_trip(args), - } - except (KeyError, TypeError, ValueError, RuntimeError, - urllib.error.URLError) as error: - print(f"benchmark failed: {error}") - return 1 - - rendered = json.dumps(report, indent=2, sort_keys=True) - print(rendered) - if args.output: - args.output.parent.mkdir(parents=True, exist_ok=True) - args.output.write_text(rendered + "\n") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) From c1cd64485fd426866d961e689caaa2762b51ce07 Mon Sep 17 00:00:00 2001 From: mrciffa <49000955+davide221@users.noreply.github.com> Date: Sat, 29 Aug 2026 00:02:23 +0200 Subject: [PATCH 3/5] refactor(ling): isolate model graph implementation --- README.md | 8 +- server/CMakeLists.txt | 1 + server/src/bailingmoe3/bailingmoe3_graph.cpp | 249 +++++++++++++++++++ server/src/bailingmoe3/bailingmoe3_graph.h | 35 +++ server/src/qwen35/qwen35_target_graph.cpp | 235 +---------------- 5 files changed, 287 insertions(+), 241 deletions(-) create mode 100644 server/src/bailingmoe3/bailingmoe3_graph.cpp create mode 100644 server/src/bailingmoe3/bailingmoe3_graph.h diff --git a/README.md b/README.md index f6aa075cc..d519bbbf0 100644 --- a/README.md +++ b/README.md @@ -53,13 +53,7 @@ Model links open the exact weights used by the measured setup. Drafter links ope | [Gemma 4 31B IT Q4_K_M](https://huggingface.co/bartowski/google_gemma-4-31B-it-GGUF/blob/main/google_gemma-4-31B-it-Q4_K_M.gguf) + [DFlash Q8_0 drafter](https://huggingface.co/Lucebox/gemma-4-31B-it-DFlash-GGUF/blob/main/gemma-4-31B-it-DFlash-q8_0.gguf) | Decode | **3.2×** | | [DeepSeek V4 Flash ROCmFPX MIX Strix](https://huggingface.co/Lucebox/DeepSeek-V4-Flash-0731-ROCmFP3/blob/main/DeepSeek-V4-Flash-0731-ROCMFPX-MIX-STRIX.gguf) + [DSpark Q4RMFP4 drafter](https://huggingface.co/Lucebox/DeepSeek-V4-Flash-0731-DSpark-GGUF/blob/main/DeepSeek-V4-Flash-0731-DSpark-draft-Q4RMFP4-denseF16.gguf) | Decode | Up to **1.81×** vs target-only, **32.7 vs 18.1 tok/s** | -Experimental native `BailingMoE3` support is available for -[`Ling 3.0 Flash`](https://huggingface.co/inclusionAI/Ling-3.0-flash). On a -single DGX Spark, LuceBox serves the Q4_K_M GGUF at **34.6 tok/s** median -autoregressive decode with Q4_0 KV cache. See the -[DGX Spark integration and benchmark](server/docs/LING3_FLASH_DGX_SPARK.md) -for the exact configuration, context sweep, tool-call test, and current -limitations. +| [Ling 3.0 Flash 124B-A5.1B Q4_K_M](https://huggingface.co/bloomer010/Ling-3.0-flash-GGUF) | Decode | **34.6 tok/s** median AR on DGX Spark | ## Tested Machines (GPU/APU) diff --git a/server/CMakeLists.txt b/server/CMakeLists.txt index 5081a21be..cab194c56 100644 --- a/server/CMakeLists.txt +++ b/server/CMakeLists.txt @@ -414,6 +414,7 @@ add_library(dflash_common STATIC src/qwen35/qwen35_target_graph.cpp src/qwen35/qwen35_roctx.cpp src/bailingmoe3/bailingmoe3_loader.cpp + src/bailingmoe3/bailingmoe3_graph.cpp src/bailingmoe3/bailingmoe3_backend.cpp src/draft/draft_gguf_loader.cpp src/draft/draft_safetensors_loader.cpp diff --git a/server/src/bailingmoe3/bailingmoe3_graph.cpp b/server/src/bailingmoe3/bailingmoe3_graph.cpp new file mode 100644 index 000000000..2c3dd2f6f --- /dev/null +++ b/server/src/bailingmoe3/bailingmoe3_graph.cpp @@ -0,0 +1,249 @@ +#include "bailingmoe3_graph.h" + +#include "internal.h" +#include "qwen35_ops.h" + +#include +#include +#include + +namespace dflash::common { +namespace { + +ggml_tensor * build_causal_conv1d( + ggml_context * ctx, + ggml_cgraph * gf, + ggml_tensor * all_conv_state, + int qkv_index, + ggml_tensor * cur, + ggml_tensor * projection, + ggml_tensor * conv_weight, + int d_conv, + int d_inner, + int head_dim, + int n_head, + int n_tokens) { + const size_t state_element = ggml_element_size(all_conv_state); + const size_t channel_stride = all_conv_state->nb[1]; + ggml_tensor * conv_state = ggml_view_3d( + ctx, all_conv_state, d_conv - 1, d_inner, 1, + channel_stride, all_conv_state->nb[2], + static_cast(qkv_index) * d_inner * channel_stride); + + ggml_tensor * projected = ggml_mul_mat(ctx, projection, cur); + projected = ggml_reshape_3d(ctx, projected, d_inner, n_tokens, 1); + ggml_tensor * conv_input = + ggml_concat(ctx, conv_state, ggml_transpose(ctx, projected), 0); + + ggml_tensor * last = ggml_view_3d( + ctx, conv_input, d_conv - 1, d_inner, 1, + conv_input->nb[1], conv_input->nb[2], + static_cast(n_tokens) * state_element); + ggml_build_forward_expand(gf, ggml_cpy(ctx, last, conv_state)); + + ggml_tensor * conv_2d = ggml_reshape_2d(ctx, conv_weight, d_conv, d_inner); + ggml_tensor * result = ggml_ssm_conv(ctx, conv_input, conv_2d); + result = ggml_silu(ctx, ggml_reshape_2d(ctx, result, d_inner, n_tokens)); + return ggml_reshape_4d(ctx, result, head_dim, n_head, n_tokens, 1); +} + +} // namespace + +// Ling 3 stores MLA K/V in compressed latent space. K contains the normalized +// 512-wide latent plus the 64 RoPE dimensions; V is the latent alone. Query +// absorption through attn_k_b turns the 128 non-RoPE Q dimensions into the +// same latent space, and attn_v_b expands the attention result back to 128 +// value dimensions per head. +ggml_tensor * build_bailingmoe3_mla_block( + ggml_context * ctx, + ggml_cgraph * gf, + const TargetWeights & w, + const TargetLayer & L, + ggml_tensor * cur, + ggml_tensor * positions, + ggml_tensor * cache_k, + ggml_tensor * cache_v, + ggml_tensor * attn_mask, + int kv_start, + int n_tokens) { + const int n_head = w.n_head; + const int qk_dim = w.mla_qk_head_dim; + const int rope_dim = w.rope_dimension_count; + const int nope_dim = qk_dim - rope_dim; + const int kv_rank = w.kv_lora_rank; + const int v_dim = w.mla_v_head_dim; + GGML_ASSERT(attn_mask != nullptr); + + ggml_tensor * q_all = nullptr; + if (L.attn_q_a) { + q_all = ggml_mul_mat(ctx, L.attn_q_a, cur); + q_all = rms_norm_mul(ctx, q_all, L.attn_q_a_norm, w.rms_eps); + q_all = ggml_mul_mat(ctx, L.attn_q_b, q_all); + } else { + q_all = ggml_mul_mat(ctx, L.wq, cur); + } + q_all = ggml_reshape_3d(ctx, q_all, qk_dim, n_head, n_tokens); + ggml_tensor * q_nope = ggml_view_3d( + ctx, q_all, nope_dim, n_head, n_tokens, + q_all->nb[1], q_all->nb[2], 0); + ggml_tensor * q_pe = ggml_view_3d( + ctx, q_all, rope_dim, n_head, n_tokens, + q_all->nb[1], q_all->nb[2], + static_cast(nope_dim) * ggml_element_size(q_all)); + + ggml_tensor * kv_all = ggml_mul_mat(ctx, L.attn_kv_a_mqa, cur); + ggml_tensor * kv = ggml_view_2d( + ctx, kv_all, kv_rank, n_tokens, kv_all->nb[1], 0); + ggml_tensor * k_pe = ggml_view_3d( + ctx, kv_all, rope_dim, 1, n_tokens, + kv_all->nb[1], kv_all->nb[1], + static_cast(kv_rank) * ggml_element_size(kv_all)); + + // The shared hybrid graph input reserves four position lanes for M-RoPE. + // Ling uses ordinary RoPE, so expose just the first lane to ggml_rope. + ggml_tensor * rope_positions = positions; + if (positions->ne[0] != n_tokens) { + GGML_ASSERT(positions->ne[0] >= n_tokens); + rope_positions = ggml_view_1d(ctx, positions, n_tokens, 0); + } + + // Bailing V3 uses interleaved rotary pairs, i.e. GGML's NORMAL layout. + q_pe = ggml_cont(ctx, q_pe); + k_pe = ggml_cont(ctx, k_pe); + q_pe = ggml_rope_ext(ctx, q_pe, rope_positions, nullptr, + rope_dim, GGML_ROPE_TYPE_NORMAL, 0, + w.rope_theta, 1.0f, + 0.0f, 1.0f, 0.0f, 0.0f); + k_pe = ggml_rope_ext(ctx, k_pe, rope_positions, nullptr, + rope_dim, GGML_ROPE_TYPE_NORMAL, 0, + w.rope_theta, 1.0f, + 0.0f, 1.0f, 0.0f, 0.0f); + kv = rms_norm_mul(ctx, kv, L.attn_kv_a_norm, w.rms_eps); + + // Absorb the K projection into Q: [nope,T,H] -> [latent,T,H]. + q_nope = ggml_permute(ctx, q_nope, 0, 2, 1, 3); + q_nope = ggml_mul_mat(ctx, L.attn_k_b, q_nope); + q_nope = ggml_permute(ctx, q_nope, 0, 2, 1, 3); + ggml_tensor * q = ggml_concat(ctx, q_nope, q_pe, 0); + + ggml_tensor * kv_3d = ggml_reshape_3d(ctx, kv, kv_rank, 1, n_tokens); + ggml_tensor * k_cur = ggml_concat(ctx, kv_3d, k_pe, 0); + ggml_tensor * v_cur = kv_3d; + + // Persistent latent cache: [D,T,1]. + ggml_tensor * k_write = ggml_permute(ctx, k_cur, 0, 2, 1, 3); + ggml_tensor * v_write = ggml_permute(ctx, v_cur, 0, 2, 1, 3); + ggml_tensor * k_slot = ggml_view_3d( + ctx, cache_k, kv_rank + rope_dim, n_tokens, 1, + cache_k->nb[1], cache_k->nb[2], + static_cast(kv_start) * cache_k->nb[1]); + ggml_tensor * v_slot = ggml_view_3d( + ctx, cache_v, kv_rank, n_tokens, 1, + cache_v->nb[1], cache_v->nb[2], + static_cast(kv_start) * cache_v->nb[1]); + ggml_build_forward_expand(gf, ggml_cpy(ctx, k_write, k_slot)); + ggml_build_forward_expand(gf, ggml_cpy(ctx, v_write, v_slot)); + + const int kv_len = kv_start + n_tokens; + // The 576x512 CUDA flash-attention specialization requires the compressed + // K/V span to be padded to its 256-token launch stride. The caller always + // supplies a causal mask for Ling, so zero-initialized future cache rows + // remain invisible. + const int kv_len_padded = std::min( + ((kv_len + 255) / 256) * 256, static_cast(cache_k->ne[1])); + ggml_tensor * k_read = ggml_view_3d( + ctx, cache_k, kv_rank + rope_dim, kv_len_padded, 1, + cache_k->nb[1], cache_k->nb[2], 0); + ggml_tensor * v_read = ggml_view_3d( + ctx, cache_v, kv_rank, kv_len_padded, 1, + cache_v->nb[1], cache_v->nb[2], 0); + ggml_tensor * q_fa = ggml_cont(ctx, ggml_permute(ctx, q, 0, 2, 1, 3)); + ggml_tensor * attn = ggml_flash_attn_ext( + ctx, q_fa, k_read, v_read, attn_mask, + 1.0f / std::sqrt(static_cast(qk_dim)), 0.0f, 0.0f); + ggml_flash_attn_ext_set_prec(attn, GGML_PREC_F32); + + // Flash attention returns [latent,H,T]. Apply the per-head V expansion + // as a batched matmul in [latent,T,H] layout. + attn = ggml_permute(ctx, attn, 0, 2, 1, 3); + attn = ggml_mul_mat(ctx, L.attn_v_b, attn); + attn = ggml_cont(ctx, ggml_permute(ctx, attn, 0, 2, 1, 3)); + + ggml_tensor * gate = ggml_mul_mat(ctx, L.wqkv_gate, cur); + gate = ggml_sigmoid(ctx, + ggml_reshape_3d(ctx, gate, 1, n_head, n_tokens)); + attn = ggml_mul(ctx, attn, gate); + attn = ggml_cont_2d(ctx, attn, v_dim * n_head, n_tokens); + return ggml_mul_mat(ctx, L.wo, attn); +} + +// Ling 3 KDA block. The shared ggml CUDA primitive detects KDA from the +// vector gate's first dimension (128 rather than scalar 1). +ggml_tensor * build_bailingmoe3_kda_block( + ggml_context * ctx, + ggml_cgraph * gf, + const TargetWeights & w, + const TargetLayer & L, + ggml_tensor * cur, + ggml_tensor * conv_state, + ggml_tensor * ssm_state, + int n_tokens) { + const int head_dim = w.kda_head_dim; + const int n_head = w.n_head; + const int d_inner = head_dim * n_head; + + ggml_tensor * q = build_causal_conv1d( + ctx, gf, conv_state, 0, cur, L.wq, L.ssm_conv1d_q, + w.ssm_d_conv, d_inner, head_dim, n_head, n_tokens); + ggml_tensor * k = build_causal_conv1d( + ctx, gf, conv_state, 1, cur, L.wk, L.ssm_conv1d_k, + w.ssm_d_conv, d_inner, head_dim, n_head, n_tokens); + ggml_tensor * v = build_causal_conv1d( + ctx, gf, conv_state, 2, cur, L.wv, L.ssm_conv1d_v, + w.ssm_d_conv, d_inner, head_dim, n_head, n_tokens); + + ggml_tensor * gate = ggml_mul_mat(ctx, L.ssm_f_a, cur); + gate = ggml_add(ctx, gate, L.ssm_dt_bias); + gate = ggml_reshape_4d(ctx, gate, head_dim, n_head, n_tokens, 1); + ggml_tensor * a = ggml_reshape_4d(ctx, L.ssm_a, 1, n_head, 1, 1); + gate = ggml_scale(ctx, ggml_sigmoid(ctx, ggml_mul(ctx, gate, a)), + w.kda_gate_lower_bound); + + ggml_tensor * beta = ggml_mul_mat(ctx, L.ssm_beta, cur); + beta = ggml_sigmoid(ctx, + ggml_reshape_4d(ctx, beta, 1, n_head, n_tokens, 1)); + q = ggml_l2_norm(ctx, q, w.rms_eps); + k = ggml_l2_norm(ctx, k, w.rms_eps); + + ggml_tensor * state = ggml_reshape_4d( + ctx, ssm_state, head_dim, head_dim, n_head, 1); + ggml_tensor * packed = + ggml_gated_delta_net(ctx, q, k, v, gate, beta, state); + ggml_gated_delta_net_set_skip_intermediate(packed, true); + + const size_t element = ggml_element_size(packed); + ggml_tensor * output = ggml_view_4d( + ctx, packed, head_dim, n_head, n_tokens, 1, + static_cast(head_dim) * element, + static_cast(head_dim) * n_head * element, + static_cast(head_dim) * n_head * n_tokens * element, + 0); + ggml_tensor * new_state = ggml_view_4d( + ctx, packed, head_dim, head_dim, n_head, 1, + static_cast(head_dim) * element, + static_cast(head_dim) * head_dim * element, + static_cast(head_dim) * head_dim * n_head * element, + static_cast(head_dim) * n_head * n_tokens * element); + ggml_build_forward_expand(gf, ggml_cpy(ctx, new_state, state)); + + ggml_tensor * output_gate = ggml_mul_mat(ctx, L.ssm_g_a, cur); + output_gate = ggml_reshape_3d( + ctx, output_gate, head_dim, n_head, n_tokens); + output = ggml_reshape_3d(ctx, output, head_dim, n_head, n_tokens); + output = rms_norm_mul(ctx, output, L.ssm_norm, w.rms_eps); + output = ggml_mul(ctx, output, ggml_sigmoid(ctx, output_gate)); + output = ggml_cont_2d(ctx, output, d_inner, n_tokens); + return ggml_mul_mat(ctx, L.wo, output); +} + +} // namespace dflash::common diff --git a/server/src/bailingmoe3/bailingmoe3_graph.h b/server/src/bailingmoe3/bailingmoe3_graph.h new file mode 100644 index 000000000..e7a4cc3a9 --- /dev/null +++ b/server/src/bailingmoe3/bailingmoe3_graph.h @@ -0,0 +1,35 @@ +#pragma once + +struct ggml_cgraph; +struct ggml_context; +struct ggml_tensor; + +namespace dflash::common { + +struct TargetLayer; +struct TargetWeights; + +ggml_tensor * build_bailingmoe3_mla_block( + ggml_context * ctx, + ggml_cgraph * gf, + const TargetWeights & weights, + const TargetLayer & layer, + ggml_tensor * cur, + ggml_tensor * positions, + ggml_tensor * cache_k, + ggml_tensor * cache_v, + ggml_tensor * attn_mask, + int kv_start, + int n_tokens); + +ggml_tensor * build_bailingmoe3_kda_block( + ggml_context * ctx, + ggml_cgraph * gf, + const TargetWeights & weights, + const TargetLayer & layer, + ggml_tensor * cur, + ggml_tensor * conv_state, + ggml_tensor * ssm_state, + int n_tokens); + +} // namespace dflash::common diff --git a/server/src/qwen35/qwen35_target_graph.cpp b/server/src/qwen35/qwen35_target_graph.cpp index 5e4937e09..2aba1355c 100644 --- a/server/src/qwen35/qwen35_target_graph.cpp +++ b/server/src/qwen35/qwen35_target_graph.cpp @@ -31,6 +31,7 @@ // conv_kernel = 4 #include "internal.h" +#include "bailingmoe3_graph.h" #include "delta_net_chunked.h" #include "delta_net_specla.h" #include "kv_quant.h" @@ -1466,240 +1467,6 @@ static ggml_tensor * build_full_attn_block( return attn; } -// Ling 3 stores MLA K/V in compressed latent space. K contains the normalized -// 512-wide latent plus the 64 RoPE dimensions; V is the latent alone. Query -// absorption through attn_k_b turns the 128 non-RoPE Q dimensions into the -// same latent space, and attn_v_b expands the attention result back to 128 -// value dimensions per head. -static ggml_tensor * build_bailingmoe3_mla_block( - ggml_context * ctx, - ggml_cgraph * gf, - const TargetWeights & w, - const TargetLayer & L, - ggml_tensor * cur, - ggml_tensor * positions, - ggml_tensor * cache_k, - ggml_tensor * cache_v, - ggml_tensor * attn_mask, - int kv_start, - int n_tokens) { - const int n_head = w.n_head; - const int qk_dim = w.mla_qk_head_dim; - const int rope_dim = w.rope_dimension_count; - const int nope_dim = qk_dim - rope_dim; - const int kv_rank = w.kv_lora_rank; - const int v_dim = w.mla_v_head_dim; - GGML_ASSERT(attn_mask != nullptr); - - ggml_tensor * q_all = nullptr; - if (L.attn_q_a) { - q_all = ggml_mul_mat(ctx, L.attn_q_a, cur); - q_all = rms_norm_mul(ctx, q_all, L.attn_q_a_norm, w.rms_eps); - q_all = ggml_mul_mat(ctx, L.attn_q_b, q_all); - } else { - q_all = ggml_mul_mat(ctx, L.wq, cur); - } - q_all = ggml_reshape_3d(ctx, q_all, qk_dim, n_head, n_tokens); - ggml_tensor * q_nope = ggml_view_3d( - ctx, q_all, nope_dim, n_head, n_tokens, - q_all->nb[1], q_all->nb[2], 0); - ggml_tensor * q_pe = ggml_view_3d( - ctx, q_all, rope_dim, n_head, n_tokens, - q_all->nb[1], q_all->nb[2], - static_cast(nope_dim) * ggml_element_size(q_all)); - - ggml_tensor * kv_all = ggml_mul_mat(ctx, L.attn_kv_a_mqa, cur); - ggml_tensor * kv = ggml_view_2d( - ctx, kv_all, kv_rank, n_tokens, kv_all->nb[1], 0); - ggml_tensor * k_pe = ggml_view_3d( - ctx, kv_all, rope_dim, 1, n_tokens, - kv_all->nb[1], kv_all->nb[1], - static_cast(kv_rank) * ggml_element_size(kv_all)); - - // The shared Qwen graph input reserves four position lanes for M-RoPE. - // Bailing uses ordinary RoPE, so expose just the first lane to ggml_rope. - ggml_tensor * rope_positions = positions; - if (positions->ne[0] != n_tokens) { - GGML_ASSERT(positions->ne[0] >= n_tokens); - rope_positions = ggml_view_1d(ctx, positions, n_tokens, 0); - } - - // Bailing V3 uses interleaved rotary pairs, i.e. GGML's NORMAL layout. - q_pe = ggml_cont(ctx, q_pe); - k_pe = ggml_cont(ctx, k_pe); - q_pe = ggml_rope_ext(ctx, q_pe, rope_positions, nullptr, - rope_dim, GGML_ROPE_TYPE_NORMAL, 0, - w.rope_theta, 1.0f, - 0.0f, 1.0f, 0.0f, 0.0f); - k_pe = ggml_rope_ext(ctx, k_pe, rope_positions, nullptr, - rope_dim, GGML_ROPE_TYPE_NORMAL, 0, - w.rope_theta, 1.0f, - 0.0f, 1.0f, 0.0f, 0.0f); - kv = rms_norm_mul(ctx, kv, L.attn_kv_a_norm, w.rms_eps); - - // Absorb the K projection into Q: [nope,T,H] -> [latent,T,H]. - q_nope = ggml_permute(ctx, q_nope, 0, 2, 1, 3); - q_nope = ggml_mul_mat(ctx, L.attn_k_b, q_nope); - q_nope = ggml_permute(ctx, q_nope, 0, 2, 1, 3); - ggml_tensor * q = ggml_concat(ctx, q_nope, q_pe, 0); - - ggml_tensor * kv_3d = ggml_reshape_3d(ctx, kv, kv_rank, 1, n_tokens); - ggml_tensor * k_cur = ggml_concat(ctx, kv_3d, k_pe, 0); - ggml_tensor * v_cur = kv_3d; - - // Persistent latent cache: [D,T,1]. - ggml_tensor * k_write = ggml_permute(ctx, k_cur, 0, 2, 1, 3); - ggml_tensor * v_write = ggml_permute(ctx, v_cur, 0, 2, 1, 3); - ggml_tensor * k_slot = ggml_view_3d( - ctx, cache_k, kv_rank + rope_dim, n_tokens, 1, - cache_k->nb[1], cache_k->nb[2], - static_cast(kv_start) * cache_k->nb[1]); - ggml_tensor * v_slot = ggml_view_3d( - ctx, cache_v, kv_rank, n_tokens, 1, - cache_v->nb[1], cache_v->nb[2], - static_cast(kv_start) * cache_v->nb[1]); - ggml_build_forward_expand(gf, ggml_cpy(ctx, k_write, k_slot)); - ggml_build_forward_expand(gf, ggml_cpy(ctx, v_write, v_slot)); - - const int kv_len = kv_start + n_tokens; - // The 576x512 CUDA flash-attention specialization requires the compressed - // K/V span to be padded to its 256-token launch stride. The caller always - // supplies a causal mask for Bailing, so zero-initialized future cache rows - // remain invisible. - const int kv_len_padded = std::min( - ((kv_len + 255) / 256) * 256, static_cast(cache_k->ne[1])); - ggml_tensor * k_read = ggml_view_3d( - ctx, cache_k, kv_rank + rope_dim, kv_len_padded, 1, - cache_k->nb[1], cache_k->nb[2], 0); - ggml_tensor * v_read = ggml_view_3d( - ctx, cache_v, kv_rank, kv_len_padded, 1, - cache_v->nb[1], cache_v->nb[2], 0); - ggml_tensor * q_fa = ggml_cont(ctx, ggml_permute(ctx, q, 0, 2, 1, 3)); - ggml_tensor * attn = ggml_flash_attn_ext( - ctx, q_fa, k_read, v_read, attn_mask, - 1.0f / std::sqrt(static_cast(qk_dim)), 0.0f, 0.0f); - ggml_flash_attn_ext_set_prec(attn, GGML_PREC_F32); - - // Flash attention returns [latent,H,T]. Apply the per-head V expansion - // as a batched matmul in [latent,T,H] layout. - attn = ggml_permute(ctx, attn, 0, 2, 1, 3); - attn = ggml_mul_mat(ctx, L.attn_v_b, attn); - attn = ggml_cont(ctx, ggml_permute(ctx, attn, 0, 2, 1, 3)); - - ggml_tensor * gate = ggml_mul_mat(ctx, L.wqkv_gate, cur); - gate = ggml_sigmoid(ctx, - ggml_reshape_3d(ctx, gate, 1, n_head, n_tokens)); - attn = ggml_mul(ctx, attn, gate); - attn = ggml_cont_2d(ctx, attn, v_dim * n_head, n_tokens); - return ggml_mul_mat(ctx, L.wo, attn); -} - -static ggml_tensor * build_bailingmoe3_causal_conv1d( - ggml_context * ctx, - ggml_cgraph * gf, - ggml_tensor * all_conv_state, - int qkv_index, - ggml_tensor * cur, - ggml_tensor * projection, - ggml_tensor * conv_weight, - int d_conv, - int d_inner, - int head_dim, - int n_head, - int n_tokens) { - const size_t state_element = ggml_element_size(all_conv_state); - const size_t channel_stride = all_conv_state->nb[1]; - ggml_tensor * conv_state = ggml_view_3d( - ctx, all_conv_state, d_conv - 1, d_inner, 1, - channel_stride, all_conv_state->nb[2], - static_cast(qkv_index) * d_inner * channel_stride); - - ggml_tensor * projected = ggml_mul_mat(ctx, projection, cur); - projected = ggml_reshape_3d(ctx, projected, d_inner, n_tokens, 1); - ggml_tensor * conv_input = - ggml_concat(ctx, conv_state, ggml_transpose(ctx, projected), 0); - - ggml_tensor * last = ggml_view_3d( - ctx, conv_input, d_conv - 1, d_inner, 1, - conv_input->nb[1], conv_input->nb[2], - static_cast(n_tokens) * state_element); - ggml_build_forward_expand(gf, ggml_cpy(ctx, last, conv_state)); - - ggml_tensor * conv_2d = ggml_reshape_2d(ctx, conv_weight, d_conv, d_inner); - ggml_tensor * result = ggml_ssm_conv(ctx, conv_input, conv_2d); - result = ggml_silu(ctx, ggml_reshape_2d(ctx, result, d_inner, n_tokens)); - return ggml_reshape_4d(ctx, result, head_dim, n_head, n_tokens, 1); -} - -// Ling 3 KDA block. The shared ggml CUDA primitive detects KDA from the -// vector gate's first dimension (128 rather than scalar 1). -static ggml_tensor * build_bailingmoe3_kda_block( - ggml_context * ctx, - ggml_cgraph * gf, - const TargetWeights & w, - const TargetLayer & L, - ggml_tensor * cur, - ggml_tensor * conv_state, - ggml_tensor * ssm_state, - int n_tokens) { - const int head_dim = w.kda_head_dim; - const int n_head = w.n_head; - const int d_inner = head_dim * n_head; - - ggml_tensor * q = build_bailingmoe3_causal_conv1d( - ctx, gf, conv_state, 0, cur, L.wq, L.ssm_conv1d_q, - w.ssm_d_conv, d_inner, head_dim, n_head, n_tokens); - ggml_tensor * k = build_bailingmoe3_causal_conv1d( - ctx, gf, conv_state, 1, cur, L.wk, L.ssm_conv1d_k, - w.ssm_d_conv, d_inner, head_dim, n_head, n_tokens); - ggml_tensor * v = build_bailingmoe3_causal_conv1d( - ctx, gf, conv_state, 2, cur, L.wv, L.ssm_conv1d_v, - w.ssm_d_conv, d_inner, head_dim, n_head, n_tokens); - - ggml_tensor * gate = ggml_mul_mat(ctx, L.ssm_f_a, cur); - gate = ggml_add(ctx, gate, L.ssm_dt_bias); - gate = ggml_reshape_4d(ctx, gate, head_dim, n_head, n_tokens, 1); - ggml_tensor * a = ggml_reshape_4d(ctx, L.ssm_a, 1, n_head, 1, 1); - gate = ggml_scale(ctx, ggml_sigmoid(ctx, ggml_mul(ctx, gate, a)), - w.kda_gate_lower_bound); - - ggml_tensor * beta = ggml_mul_mat(ctx, L.ssm_beta, cur); - beta = ggml_sigmoid(ctx, - ggml_reshape_4d(ctx, beta, 1, n_head, n_tokens, 1)); - q = ggml_l2_norm(ctx, q, w.rms_eps); - k = ggml_l2_norm(ctx, k, w.rms_eps); - - ggml_tensor * state = ggml_reshape_4d( - ctx, ssm_state, head_dim, head_dim, n_head, 1); - ggml_tensor * packed = - ggml_gated_delta_net(ctx, q, k, v, gate, beta, state); - ggml_gated_delta_net_set_skip_intermediate(packed, true); - - const size_t element = ggml_element_size(packed); - ggml_tensor * output = ggml_view_4d( - ctx, packed, head_dim, n_head, n_tokens, 1, - static_cast(head_dim) * element, - static_cast(head_dim) * n_head * element, - static_cast(head_dim) * n_head * n_tokens * element, - 0); - ggml_tensor * new_state = ggml_view_4d( - ctx, packed, head_dim, head_dim, n_head, 1, - static_cast(head_dim) * element, - static_cast(head_dim) * head_dim * element, - static_cast(head_dim) * head_dim * n_head * element, - static_cast(head_dim) * n_head * n_tokens * element); - ggml_build_forward_expand(gf, ggml_cpy(ctx, new_state, state)); - - ggml_tensor * output_gate = ggml_mul_mat(ctx, L.ssm_g_a, cur); - output_gate = ggml_reshape_3d( - ctx, output_gate, head_dim, n_head, n_tokens); - output = ggml_reshape_3d(ctx, output, head_dim, n_head, n_tokens); - output = rms_norm_mul(ctx, output, L.ssm_norm, w.rms_eps); - output = ggml_mul(ctx, output, ggml_sigmoid(ctx, output_gate)); - output = ggml_cont_2d(ctx, output, d_inner, n_tokens); - return ggml_mul_mat(ctx, L.wo, output); -} - // Gated DeltaNet block using the fused ggml_gated_delta_net primitive. // // Matches the semantics of llama.cpp's build_layer_attn_linear + build_delta_net_fused. From 711ffe2f45167781bd19d682c8ace5a3b354acc4 Mon Sep 17 00:00:00 2001 From: mrciffa <49000955+davide221@users.noreply.github.com> Date: Sat, 29 Aug 2026 00:16:09 +0200 Subject: [PATCH 4/5] fix(ling): address review findings --- README.md | 1 - server/src/qwen35/qwen35_target_graph.cpp | 7 ++- server/src/server/chat_template.cpp | 20 ++++---- server/src/server/http_server.cpp | 12 +++-- server/src/server/sse_emitter.cpp | 21 ++++++++ server/src/server/sse_emitter.h | 7 +++ server/test/test_chain_rollback_policy.cpp | 43 ++++++++++++++++ server/test/test_server_unit.cpp | 58 ++++++++++++++++++++++ 8 files changed, 151 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index d519bbbf0..64f999206 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,6 @@ Model links open the exact weights used by the measured setup. Drafter links ope | [Gemma 4 26B-A4B Q4_K_M](https://huggingface.co/bartowski/google_gemma-4-26B-A4B-it-GGUF/blob/main/google_gemma-4-26B-A4B-it-Q4_K_M.gguf) + [DFlash Q8_0 drafter](https://huggingface.co/Lucebox/gemma-4-26B-A4B-it-DFlash-GGUF/blob/main/gemma-4-26B-A4B-it-DFlash-q8_0.gguf) | Decode | **1.31×** | | [Gemma 4 31B IT Q4_K_M](https://huggingface.co/bartowski/google_gemma-4-31B-it-GGUF/blob/main/google_gemma-4-31B-it-Q4_K_M.gguf) + [DFlash Q8_0 drafter](https://huggingface.co/Lucebox/gemma-4-31B-it-DFlash-GGUF/blob/main/gemma-4-31B-it-DFlash-q8_0.gguf) | Decode | **3.2×** | | [DeepSeek V4 Flash ROCmFPX MIX Strix](https://huggingface.co/Lucebox/DeepSeek-V4-Flash-0731-ROCmFP3/blob/main/DeepSeek-V4-Flash-0731-ROCMFPX-MIX-STRIX.gguf) + [DSpark Q4RMFP4 drafter](https://huggingface.co/Lucebox/DeepSeek-V4-Flash-0731-DSpark-GGUF/blob/main/DeepSeek-V4-Flash-0731-DSpark-draft-Q4RMFP4-denseF16.gguf) | Decode | Up to **1.81×** vs target-only, **32.7 vs 18.1 tok/s** | - | [Ling 3.0 Flash 124B-A5.1B Q4_K_M](https://huggingface.co/bloomer010/Ling-3.0-flash-GGUF) | Decode | **34.6 tok/s** median AR on DGX Spark | ## Tested Machines (GPU/APU) diff --git a/server/src/qwen35/qwen35_target_graph.cpp b/server/src/qwen35/qwen35_target_graph.cpp index 2aba1355c..7e7c3b613 100644 --- a/server/src/qwen35/qwen35_target_graph.cpp +++ b/server/src/qwen35/qwen35_target_graph.cpp @@ -234,8 +234,11 @@ bool create_target_cache_partial(const TargetWeights & w, // [head_dim, max_ctx_alloc, n_head_kv] ggml_tensor * K = ggml_new_tensor_3d(out.base_ctx, kv_k_type, head_dim, max_ctx_alloc, w.n_head_kv); - ggml_tensor * V = ggml_new_tensor_3d(out.base_ctx, kv_v_type, - w.n_embd_head_v, max_ctx_alloc, w.n_head_kv); + const int v_head_dim = w.is_bailingmoe3 + ? w.n_embd_head_v : head_dim; + ggml_tensor * V = ggml_new_tensor_3d( + out.base_ctx, kv_v_type, v_head_dim, + max_ctx_alloc, w.n_head_kv); char name[64]; std::snprintf(name, sizeof(name), "cache_k_%d", il); ggml_set_name(K, name); diff --git a/server/src/server/chat_template.cpp b/server/src/server/chat_template.cpp index 63f644649..753574946 100644 --- a/server/src/server/chat_template.cpp +++ b/server/src/server/chat_template.cpp @@ -91,10 +91,11 @@ std::string render_chat_template( !messages.empty() && messages[0].role == "system"; const std::string system_content = has_system ? messages[0].content : std::string(); - const bool system_sets_thinking = - system_content.find("detailed thinking on") != std::string::npos || - system_content.find("detailed thinking off") != std::string::npos; const char * thinking_option = enable_thinking ? "on" : "off"; + const std::string thinking_directive = + std::string("detailed thinking ") + thinking_option; + const bool system_sets_requested_thinking = + system_content.find(thinking_directive) != std::string::npos; result += "SYSTEM"; if (has_tools) { @@ -128,22 +129,19 @@ std::string render_chat_template( "{arg-value-2}\n" "...\n" "\n"; - if (!system_sets_thinking) { - result += "detailed thinking "; - result += thinking_option; + if (!system_sets_requested_thinking) { + result += thinking_directive; } result += "<|role_end|>"; } else if (has_system) { result += system_content; - if (!system_sets_thinking) { + if (!system_sets_requested_thinking) { result += '\n'; - result += "detailed thinking "; - result += thinking_option; + result += thinking_directive; } result += "<|role_end|>"; } else { - result += "detailed thinking "; - result += thinking_option; + result += thinking_directive; result += "<|role_end|>"; } diff --git a/server/src/server/http_server.cpp b/server/src/server/http_server.cpp index 388d11e3a..a5a8032ae 100644 --- a/server/src/server/http_server.cpp +++ b/server/src/server/http_server.cpp @@ -2221,12 +2221,16 @@ enum class TokenDelivery { // MUST classify identically, or the reasoning/content split diverges // between streamed and non-streamed responses. TokenDelivery classify_generated_token( - Tokenizer & tokenizer, int32_t token, std::string & text) { + Tokenizer & tokenizer, SseEmitter & emitter, + int32_t token, std::string & text) { if (token == tokenizer.eos_id() || token == tokenizer.eos_chat_id()) { return TokenDelivery::kSkip; } const std::string & raw = tokenizer.raw_token(token); + if (emitter.suppress_undeclared_tool_protocol_token(raw)) { + return TokenDelivery::kSkip; + } // Gemma4 thinking channel (<|channel> / ) and Qwen3.6 // thinking markers share one mapped dialect. The Qwen markers @@ -2276,7 +2280,7 @@ CompletionTokenCounts feed_non_streaming_tokens( for (int32_t token : tokens) { std::string text; const TokenDelivery delivery = - classify_generated_token(tokenizer, token, text); + classify_generated_token(tokenizer, emitter, token, text); if (delivery == TokenDelivery::kSkip) continue; emitter.emit_token(text); @@ -3818,7 +3822,7 @@ void HttpServer::configure_generation_io( std::string text; const TokenDelivery delivery = - classify_generated_token(tokenizer_, token, text); + classify_generated_token(tokenizer_, emitter, token, text); if (delivery == TokenDelivery::kSkip) return true; if (!text.empty()) { @@ -3847,7 +3851,7 @@ bool HttpServer::deliver_generation_token( std::string text; const TokenDelivery delivery = - classify_generated_token(tokenizer_, token, text); + classify_generated_token(tokenizer_, emitter, token, text); if (delivery == TokenDelivery::kSkip) return true; // Non-stream replay counts every non-skipped token, including tokens diff --git a/server/src/server/sse_emitter.cpp b/server/src/server/sse_emitter.cpp index 6c99ced7e..040a5fea1 100644 --- a/server/src/server/sse_emitter.cpp +++ b/server/src/server/sse_emitter.cpp @@ -115,6 +115,27 @@ SseEmitter::SseEmitter(ApiFormat format, } } +bool SseEmitter::suppress_undeclared_tool_protocol_token( + const std::string & raw_token) { + if (has_request_tools(tools_)) return false; + + if (raw_token == "") { + suppress_undeclared_tool_protocol_ = true; + return true; + } + if (suppress_undeclared_tool_protocol_) { + if (raw_token == "") { + suppress_undeclared_tool_protocol_ = false; + } + return true; + } + + // Stray protocol delimiters are control tokens, not assistant content. + return raw_token == "" || + raw_token == "" || raw_token == "" || + raw_token == "" || raw_token == ""; +} + // ─── SSE formatting helpers ───────────────────────────────────────────── std::string SseEmitter::sse_data(const std::string & json_str) { diff --git a/server/src/server/sse_emitter.h b/server/src/server/sse_emitter.h index ed2d6fca8..f22ff283c 100644 --- a/server/src/server/sse_emitter.h +++ b/server/src/server/sse_emitter.h @@ -139,6 +139,12 @@ class SseEmitter { // counter; the difference is the natural-close content suffix. int emit_token_count() const { return emit_token_count_; } + // Discard a model-emitted Bailing tool block when the request did not + // declare tools. Returns true for every token from through + // , including the delimiters. + bool suppress_undeclared_tool_protocol_token( + const std::string & raw_token); + private: // Format helpers std::string format_openai_delta(const json & delta, const char * finish = nullptr); @@ -164,6 +170,7 @@ class SseEmitter { StreamMode mode_; bool tool_from_reasoning_ = false; + bool suppress_undeclared_tool_protocol_ = false; std::string window_; // holdback buffer // Incomplete trailing UTF-8 bytes from the previous token piece. // BPE tokens can split a multi-byte codepoint (emoji arrive as two diff --git a/server/test/test_chain_rollback_policy.cpp b/server/test/test_chain_rollback_policy.cpp index 7a17d6aff..ed40db245 100644 --- a/server/test/test_chain_rollback_policy.cpp +++ b/server/test/test_chain_rollback_policy.cpp @@ -166,6 +166,49 @@ TEST_CASE(ChainRollbackPolicyFixture, split_checkpoint_dtype_is_gated_at_allocat ggml_backend_free(backend); } +TEST_CASE(ChainRollbackPolicyFixture, asymmetric_v_cache_is_ling_only) { + const luce_test::ScopedEnvVar kv_f16("DFLASH27B_KV_F16", "1"); + const luce_test::ScopedEnvVar kv_q4("DFLASH27B_KV_Q4", nullptr); + const luce_test::ScopedEnvVar kv_tq3("DFLASH27B_KV_TQ3", nullptr); + const luce_test::ScopedEnvVar kv_k("DFLASH27B_KV_K", nullptr); + const luce_test::ScopedEnvVar kv_v("DFLASH27B_KV_V", nullptr); + + ggml_backend_t backend = ggml_backend_cpu_init(); + CHECK(backend != nullptr); + if (!backend) return; + + dflash::common::TargetWeights weights; + weights.n_layer = 1; + weights.full_attention_interval = 1; + weights.n_embd_head_k = 32; + weights.n_embd_head_v = 24; + weights.n_head = 1; + weights.n_head_kv = 1; + weights.n_embd = 32; + weights.n_capture_layers = 0; + + const auto check_width = [&](bool is_bailingmoe3, int expected) { + weights.is_bailingmoe3 = is_bailingmoe3; + dflash::common::TargetCache cache; + const bool ok = dflash::common::create_target_cache_partial( + weights, /*max_ctx=*/1, /*max_verify_tokens=*/1, backend, cache, + /*prefill_only=*/false, /*layer_begin=*/0, /*layer_end=*/1, + /*allocate_target_feat=*/false, /*ctx_alloc=*/0, + /*f32_ssm_intermediates=*/false); + CHECK(ok); + if (ok) { + CHECK(cache.attn_v.size() == 1); + CHECK(cache.attn_v[0] != nullptr); + if (cache.attn_v[0]) CHECK(cache.attn_v[0]->ne[0] == expected); + } + dflash::common::free_target_cache(cache); + }; + + check_width(/*is_bailingmoe3=*/false, /*expected=*/32); + check_width(/*is_bailingmoe3=*/true, /*expected=*/24); + ggml_backend_free(backend); +} + TEST_CASE(ChainRollbackPolicyFixture, diagnostics_accumulator_and_print_contract) { const luce_test::ScopedEnvVar checkpoint("DFLASH_SINGLE_CHAIN_CHECKPOINT_F32", nullptr); const luce_test::ScopedEnvVar threshold("DFLASH_FAST_ROLLBACK_THRESHOLD", nullptr); diff --git a/server/test/test_server_unit.cpp b/server/test/test_server_unit.cpp index 6681c8536..94710665e 100644 --- a/server/test/test_server_unit.cpp +++ b/server/test/test_server_unit.cpp @@ -3438,6 +3438,64 @@ TEST_CASE(ServerUnitFixture, test_bailingmoe3_render_thinking_and_tools) { TEST_ASSERT(out.compare(out.size() - suffix.size(), suffix.size(), suffix) == 0); } +TEST_CASE(ServerUnitFixture, test_bailingmoe3_request_overrides_system_thinking) { + const std::vector thinking_on = { + {"system", "Keep this note. detailed thinking on", ""}, + {"user", "Hello", ""}, + }; + const std::string disabled = render_chat_template( + thinking_on, ChatFormat::BAILINGMOE3, + /*add_generation_prompt=*/true, + /*enable_thinking=*/false, + /*tools_json=*/""); + const size_t prior_on = disabled.find("detailed thinking on"); + const size_t requested_off = disabled.rfind("detailed thinking off"); + TEST_ASSERT(prior_on != std::string::npos); + TEST_ASSERT(requested_off != std::string::npos && requested_off > prior_on); + + const std::vector thinking_off = { + {"system", "Keep this note. detailed thinking off", ""}, + {"user", "Hello", ""}, + }; + const std::string tools = + R"([{"type":"function","function":{"name":"weather"}}])"; + const std::string enabled = render_chat_template( + thinking_off, ChatFormat::BAILINGMOE3, + /*add_generation_prompt=*/true, + /*enable_thinking=*/true, + tools); + const size_t prior_off = enabled.find("detailed thinking off"); + const size_t requested_on = enabled.rfind("detailed thinking on"); + TEST_ASSERT(prior_off != std::string::npos); + TEST_ASSERT(requested_on != std::string::npos && requested_on > prior_off); +} + +TEST_CASE(ServerUnitFixture, test_emitter_suppresses_undeclared_bailing_tool_block) { + auto em = make_emitter(ApiFormat::OPENAI_CHAT); + em.emit_start(); + + const auto deliver = [&](const std::string & raw_token) { + if (!em.suppress_undeclared_tool_protocol_token(raw_token)) { + em.emit_token(raw_token); + } + }; + deliver(""); + deliver("weather"); + deliver(""); + deliver("city"); + deliver(""); + deliver(""); + deliver("Rome"); + deliver(""); + deliver(""); + deliver("Visible answer"); + em.emit_finish(10); + + TEST_ASSERT(em.accumulated_text() == "Visible answer"); + TEST_ASSERT(em.accumulated_text().find("weather") == std::string::npos); + TEST_ASSERT(em.accumulated_text().find("Rome") == std::string::npos); +} + TEST_CASE(ServerUnitFixture, test_bailingmoe3_router_builds_group_mask) { ggml_init_params params{}; params.mem_size = 2 * 1024 * 1024; From e49abac8ccc6dd7d39af44204540212a9fcb0f89 Mon Sep 17 00:00:00 2001 From: mrciffa <49000955+davide221@users.noreply.github.com> Date: Sat, 29 Aug 2026 00:19:10 +0200 Subject: [PATCH 5/5] test(ling): isolate asymmetric cache fixture --- server/test/test_chain_rollback_policy.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/test/test_chain_rollback_policy.cpp b/server/test/test_chain_rollback_policy.cpp index ed40db245..c1a558ac0 100644 --- a/server/test/test_chain_rollback_policy.cpp +++ b/server/test/test_chain_rollback_policy.cpp @@ -190,9 +190,10 @@ TEST_CASE(ChainRollbackPolicyFixture, asymmetric_v_cache_is_ling_only) { const auto check_width = [&](bool is_bailingmoe3, int expected) { weights.is_bailingmoe3 = is_bailingmoe3; dflash::common::TargetCache cache; + // This fixture has no recurrent layers, so no rollback cache is needed. const bool ok = dflash::common::create_target_cache_partial( weights, /*max_ctx=*/1, /*max_verify_tokens=*/1, backend, cache, - /*prefill_only=*/false, /*layer_begin=*/0, /*layer_end=*/1, + /*prefill_only=*/true, /*layer_begin=*/0, /*layer_end=*/1, /*allocate_target_feat=*/false, /*ctx_alloc=*/0, /*f32_ssm_intermediates=*/false); CHECK(ok);