Add GLM-5-Next (GLM-5.3-Flash): KDA + DSA hybrid attention, MoE, lightning indexer, vision

conversion/__init__.py

@@ -103,6 +103,8 @@
     "Glm4MoeLiteForCausalLM": "glm",
     "Glm4vForConditionalGeneration": "glm",
     "Glm4vMoeForConditionalGeneration": "glm",
+    "Glm5NextForCausalLM": "glm5next",
+    "Glm5NextForConditionalGeneration": "glm5next",
     "GlmForCausalLM": "chatglm",
     "GlmMoeDsaForCausalLM": "glm",
     "GlmOcrForConditionalGeneration": "glm",
@@ -293,6 +295,7 @@
     "Gemma4UnifiedForConditionalGeneration": "gemma",
     "Glm4vForConditionalGeneration": "qwen3vl",
     "Glm4vMoeForConditionalGeneration": "qwen3vl",
+    "Glm5NextForConditionalGeneration": "glm5next",
     "Glm5vForConditionalGeneration": "kimivl",
     "GlmOcrForConditionalGeneration": "qwen3vl",
     "GlmasrModel": "ultravox",

conversion/base.py

@@ -2537,6 +2537,20 @@ def set_gguf_parameters(self):
         if not self.has_vision_encoder and not self.has_audio_encoder:
             raise ValueError("MmprojModel must have either vision or audio encoder")
 
+    def prepare_tensors(self):
+        super().prepare_tensors()
+
+        # an mmproj has no vocab-only mode, so an empty one is always a silent mapping failure
+        # rather than a supported output; count and size are both checked because a tensor map
+        # that produces only zero-sized entries is just as broken as one that produces none
+        n_tensors = sum(len(t) for t in self.gguf_writer.tensors)
+        n_bytes = sum(ti.nbytes for t in self.gguf_writer.tensors for ti in t.values())
+        if n_tensors == 0 or n_bytes == 0:
+            raise ValueError(
+                f"refusing to write an mmproj with no tensor data (n_tensors = {n_tensors}, n_bytes = {n_bytes}); "
+                "check that the model's vision/audio tensors are named as the tensor map expects"
+            )
+
     def write_vocab(self):
         raise ValueError("MmprojModel does not support vocab writing")
 

conversion/glm5next.py

@@ -0,0 +1,273 @@
+from __future__ import annotations
+
+import re
+
+from typing import Callable, Iterable, TYPE_CHECKING
+
+import torch
+
+if TYPE_CHECKING:
+    from torch import Tensor
+
+from .base import LazyTorchTensor, ModelBase, TextModel, gguf
+from .qwen3vl import Glm4VVisionModel
+
+
+@ModelBase.register("Glm5NextForConditionalGeneration", "Glm5NextForCausalLM")
+# [TAG_HF_EXAMPLE_MISSING]
+class Glm5NextModel(TextModel):
+    """GLM-5.3-Flash text tower: hybrid KDA + DSA attention, nope-only MLA, mHC
+    hyper-connections, and a NextN block with its own DSA attention and indexer.
+    """
+
+    model_arch = gguf.MODEL_ARCH.GLM5NEXT
+    supports_mtp_export = True
+
+    _experts: list[dict[str, Tensor]] | None = None
+    _main_layers: int | None = None
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        nextn_layers = 0 if self.no_mtp else (self.hparams.get("num_nextn_predict_layers", 0) or 0)
+        self.block_count = self.hparams["num_hidden_layers"] + nextn_layers
+        self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count)
+
+        # two independent spellings of the same partition; disagreement means an
+        # unexpected config
+        from_types = {il for il, t in enumerate(self.hparams["layer_types"]) if t == "deepseek_sparse_attention"}
+        from_list = set(self.hparams["linear_attn_config"]["full_attn_layers"])
+        if from_types != from_list:
+            raise ValueError(f"layer_types picks DSA layers {sorted(from_types)} but full_attn_layers says {sorted(from_list)}")
+        self._full_attn_layers = from_types
+
+        dense_lead = self.hparams["first_k_dense_replace"]
+        expected = ["dense"] * dense_lead + ["sparse"] * (self.hparams["num_hidden_layers"] - dense_lead)
+        if self.hparams["mlp_layer_types"] != expected:
+            raise ValueError("mlp_layer_types does not match first_k_dense_replace")
+
+    def index_tensors(self, remote_hf_model_id: str | None = None):
+        # runs before TextModel.__init__ has hoisted text_config to the root
+        hp = self.hparams.get("text_config", self.hparams)
+        type(self)._main_layers = hp["num_hidden_layers"]
+        return super().index_tensors(remote_hf_model_id=remote_hf_model_id)
+
+    def set_vocab(self):
+        self._set_vocab_glm()
+
+    def is_full_attention(self, bid: int) -> bool:
+        return bid >= self.hparams["num_hidden_layers"] or bid in self._full_attn_layers
+
+    # -- metadata ---------------------------------------------------------
+
+    def set_gguf_parameters(self):
+        hp = self.hparams
+        linear_cfg = hp["linear_attn_config"]
+
+        # checked here, not in the loader: head_count_kv is overwritten below with
+        # the per-layer 1/0 recurrence marker
+        if hp["num_attention_heads"] != hp.get("num_key_value_heads"):
+            raise ValueError("glm5next expects MHA-shaped head counts before MLA absorption")
+        if hp["qk_rope_head_dim"] != 0 or not hp.get("mla_use_nope"):
+            raise ValueError("glm5next is nope-only: qk_rope_head_dim must be 0 and mla_use_nope true")
+        if linear_cfg["num_heads"] != hp["num_attention_heads"]:
+            raise ValueError("glm5next KDA and full attention are expected to share a head count")
+        if not hp.get("mhc"):
+            raise ValueError("glm5next without mHC is not supported")
+        if hp["index_topk"] % hp["index_kpool"] != 0:
+            raise ValueError("glm5next index_topk must be a whole number of kpool pools")
+
+        # no GGUF key carries these and the graph cannot express them off, so refuse
+        # rather than write a silently wrong file
+        if not hp.get("index_kpool_compress"):
+            raise ValueError("glm5next without the indexer kpool compressor is not supported")
+        if not hp.get("index_kpool_always_select_tail"):
+            raise ValueError("glm5next without always-select-tail kpool is not supported")
+        if not hp.get("indexer_rope_interleave"):
+            raise ValueError("glm5next without interleaved indexer rope is not supported")
+        if set(hp["indexer_types"]) != {"full"}:
+            raise ValueError("glm5next expects every indexer to be full")
+
+        # drop both: head_dim is 0 in the config, head_count_kv is written as a
+        # per-layer array below
+        hp.pop("head_dim", None)
+        hp.pop("num_key_value_heads", None)
+
+        super().set_gguf_parameters()
+
+        self.gguf_writer.add_vocab_size(hp["vocab_size"])
+
+        # n_head_kv == 0 marks a KDA (recurrent) layer, as in kimi-k3 and bailingmoe3
+        self.gguf_writer.add_head_count_kv(
+            [1 if self.is_full_attention(il) else 0 for il in range(self.block_count)])
+
+        # --- MLA ---
+        kv_lora_rank = hp["kv_lora_rank"]
+        qk_rope_head_dim = hp["qk_rope_head_dim"]
+        self.gguf_writer.add_q_lora_rank(hp["q_lora_rank"])
+        self.gguf_writer.add_kv_lora_rank(kv_lora_rank)
+        self.gguf_writer.add_rope_dimension_count(qk_rope_head_dim)
+        self.gguf_writer.add_key_length(kv_lora_rank + qk_rope_head_dim)
+        self.gguf_writer.add_value_length(kv_lora_rank)
+        self.gguf_writer.add_key_length_mla(hp["qk_nope_head_dim"] + qk_rope_head_dim)
+        self.gguf_writer.add_value_length_mla(hp["v_head_dim"])
+
+        # indexer k_norm is a LayerNorm with bias at a fixed 1e-6, not the model's
+        # RMS eps. glm-dsa omits this key and runs that norm at eps 0
+        self.gguf_writer.add_layer_norm_eps(1e-6)
+
+        # --- KDA ---
+        self.gguf_writer.add_ssm_conv_kernel(linear_cfg["short_conv_kernel_size"])
+        self.gguf_writer.add_kda_head_dim(linear_cfg["head_dim"])
+        # not a clamp: scales the sigmoid decay gate. required, a missing key
+        # silently selects the softplus branch instead
+        self.gguf_writer.add_kda_gate_lower_bound(linear_cfg["gate_lower_bound"])
+
+        # --- DSA indexer ---
+        self.gguf_writer.add_indexer_head_count(hp["index_n_heads"])
+        self.gguf_writer.add_indexer_key_length(hp["index_head_dim"])
+        self.gguf_writer.add_indexer_top_k(hp["index_topk"])
+        self.gguf_writer.add_indexer_kpool(hp["index_kpool"])
+
+        # --- mHC ---
+        self.gguf_writer.add_hyper_connection_count(hp["hc_mult"])
+        self.gguf_writer.add_hyper_connection_sinkhorn_iterations(hp["hc_sinkhorn_iters"])
+        self.gguf_writer.add_hyper_connection_epsilon(hp["hc_eps"])
+
+        # --- MoE ---
+        n_ff_exp = hp["moe_intermediate_size"]
+        self.gguf_writer.add_expert_feed_forward_length(n_ff_exp)
+        self.gguf_writer.add_expert_shared_feed_forward_length(n_ff_exp * hp["n_shared_experts"])
+        self.gguf_writer.add_expert_shared_count(hp["n_shared_experts"])
+        self.gguf_writer.add_leading_dense_block_count(hp["first_k_dense_replace"])
+        self.gguf_writer.add_expert_weights_scale(hp["routed_scaling_factor"])
+        self.gguf_writer.add_expert_weights_norm(hp["norm_topk_prob"])
+
+        # one limit for the whole model. no dense-FFN clamp key exists, so the
+        # expert arrays are sized for every layer to cover the leading dense ones
+        swiglu_limit = float(hp["swiglu_limit"])
+        self.gguf_writer.add_swiglu_clamp_exp([swiglu_limit] * self.block_count)
+        self.gguf_writer.add_swiglu_clamp_shexp([swiglu_limit] * self.block_count)
+
+        if not self.no_mtp and (nextn_layers := hp.get("num_nextn_predict_layers", 0)):
+            self.gguf_writer.add_nextn_predict_layers(nextn_layers)
+
+    # -- tensors ----------------------------------------------------------
+
+    @classmethod
+    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
+        if (titem := super().filter_tensors(item)) is None:
+            return None
+        name, gen = titem
+
+        assert cls._main_layers is not None
+        is_mtp = (m := re.match(r"model\.layers\.(\d+)\.", name)) is not None and int(m.group(1)) >= cls._main_layers
+
+        if is_mtp and cls.no_mtp:
+            return None
+        if cls.mtp_only and not is_mtp and name not in (
+            "model.embed_tokens.weight", "model.norm.weight", "lm_head.weight",
+        ):
+            return None
+
+        return name, gen
+
+    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+        # --- KDA conv1d: HF [d_inner, 1, d_conv] -> ggml ne [d_conv, 1, d_inner, 1] ---
+        if name.endswith((".q_conv1d.weight", ".k_conv1d.weight", ".v_conv1d.weight")):
+            d_inner = data_torch.shape[0]
+            d_conv = data_torch.shape[-1]
+            data_torch = data_torch.reshape(1, d_inner, 1, d_conv)
+
+        # ssm_a holds -exp(A_log), the kimi-k3 convention (bailingmoe3 stores
+        # +exp(A_log)); the wrong sign turns decay into an unchecked growing state
+        if name.endswith(".A_log"):
+            # eager: the sign is the point of the check, and A_log is one per head
+            decay = LazyTorchTensor.to_eager(torch.exp(data_torch.float()))
+            if not bool(torch.isfinite(decay).all() and (decay > 0).all()):
+                raise ValueError(f"{name}: exp(A_log) must be finite and positive")
+            data_torch = -decay
+
+        # dt_bias -> the name SSM_DT's mapping expects
+        if name.endswith(".dt_bias"):
+            name = name.rpartition(".dt_bias")[0] + ".dt_proj.bias"
+
+        # bare tensors in the checkpoint, but the GGUF names carry .weight
+        if re.search(r"\.hc_(attn|ffn)_(fn|base|scale)$", name) or name.endswith(
+                (".index_kpool_compress_gate", ".index_kpool_compress_ape")):
+            name += ".weight"
+
+        # --- routed experts ---
+        if ".mlp.experts." in name:
+            n_experts = self.hparams["n_routed_experts"]
+            assert bid is not None
+
+            if self._experts is None:
+                self._experts = [{} for _ in range(self.block_count)]
+            self._experts[bid][name] = data_torch
+
+            if len(self._experts[bid]) < n_experts * 3:
+                return
+
+            for weight_name in ("down_proj", "gate_proj", "up_proj"):
+                tensors = []
+                for expert_id in range(n_experts):
+                    expert_name = f"model.layers.{bid}.mlp.experts.{expert_id}.{weight_name}.weight"
+                    tensors.append(self._experts[bid].pop(expert_name))
+                merged_name = f"model.layers.{bid}.mlp.experts.{weight_name}.weight"
+                yield from super().modify_tensors(torch.stack(tensors, dim=0), merged_name, bid)
+            return
+
+        # --- MLA absorption ---
+        if name.endswith(".kv_b_proj.weight"):
+            assert bid is not None
+            n_head = self.hparams["num_attention_heads"]
+            v_head_dim = self.hparams["v_head_dim"]
+            qk_nope_head_dim = self.hparams["qk_nope_head_dim"]
+            assert data_torch.shape[0] == n_head * (v_head_dim + qk_nope_head_dim)
+            kv_b = data_torch.view(n_head, v_head_dim + qk_nope_head_dim, data_torch.shape[-1])
+            k_b, v_b = torch.split(kv_b, [qk_nope_head_dim, v_head_dim], dim=1)
+            yield from super().modify_tensors(k_b.transpose(1, 2), self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_K_B, bid), bid)
+            yield from super().modify_tensors(v_b, self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_V_B, bid), bid)
+            return
+
+        yield from super().modify_tensors(data_torch, name, bid)
+
+    def tensor_force_quant(self, name, new_name, bid, n_dims):
+        # learned position table, one row per pooled key; pinned for the same
+        # reason POS_EMBD is in base.py
+        if self.match_model_tensor_name(new_name, gguf.MODEL_TENSOR.INDEXER_COMPRESSOR_APE, bid):
+            return gguf.GGMLQuantizationType.F32
+        return super().tensor_force_quant(name, new_name, bid, n_dims)
+
+    def prepare_tensors(self):
+        super().prepare_tensors()
+        if self._experts is not None:
+            experts = [name for layer in self._experts for name in layer]
+            if experts:
+                raise ValueError(f"Unprocessed experts: {experts}")
+
+
+@ModelBase.register("Glm5NextForConditionalGeneration")
+# [TAG_HF_EXAMPLE_MISSING]
+class Glm5NextVisionModel(Glm4VVisionModel):
+    """The vision tower is the GLM-OCR ViT under a `model.visual.` prefix.
+
+    Every tensor already maps through the GLM-4V entries. The one structural
+    difference is a clamp on the SwiGLU gate and up projections, applied in the
+    per-block MLP and again in the merger, so it gets its own projector type
+    rather than a flag on glm4v.
+    """
+
+    clip_projector_type = gguf.VisionProjectorType.GLM5NEXT
+
+    def set_gguf_parameters(self):
+        super().set_gguf_parameters()
+        assert self.hparams_vision is not None
+
+        # Glm4VVisionModel bypasses Qwen3VLVisionModel entirely, which is also where
+        # the merge size is written, so no GLM4V-family mmproj carries this key and
+        # clip.cpp falls back to a hardcoded 2. Write it rather than rely on that
+        self.gguf_writer.add_vision_spatial_merge_size(int(self.hparams_vision["spatial_merge_size"]))
+
+        self.gguf_writer.add_vision_swiglu_limit(float(self.hparams_vision["swiglu_limit"]))

conversion/qwen3vl.py

@@ -228,10 +228,13 @@ class Qwen3ASRMmprojModel(Qwen3OmniMmprojModel):
 @ModelBase.register("Glm4vForConditionalGeneration", "Glm4vMoeForConditionalGeneration", "GlmOcrForConditionalGeneration")
 @ModelBase.example("zai-org/GLM-4.1V-9B-Thinking", "zai-org/GLM-4.5V")
 class Glm4VVisionModel(Qwen3VLVisionModel):
+    # subclasses that share this tower but need their own clip graph override this
+    clip_projector_type = gguf.VisionProjectorType.GLM4V
+
     def set_gguf_parameters(self):
         MmprojModel.set_gguf_parameters(self) # skip Qwen3VLVisionModel parameters
         assert self.hparams_vision is not None
-        self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.GLM4V)
+        self.gguf_writer.add_clip_projector_type(self.clip_projector_type)
 
         hidden_act = str(self.hparams_vision.get("hidden_act", "")).lower()
         if hidden_act == "gelu":

examples/embedding/embedding.cpp

@@ -144,6 +144,12 @@ int main(int argc, char ** argv) {
         return 1;
     }
 
+    // a context can fail on its own, and llama_n_ctx below dereferences it
+    if (ctx == NULL) {
+        LOG_ERR("%s: unable to create context\n", __func__);
+        return 1;
+    }
+
     const llama_vocab * vocab = llama_model_get_vocab(model);
 
     const int n_ctx_train = llama_model_n_ctx_train(model);