glm: fix [gMASK]<sop> tokenization without BOS metadata

Makefile

@@ -484,6 +484,12 @@ tests/test_sampling.o: tests/test_sampling.c ds4.h
 tests/test_sampling: tests/test_sampling.o ds4_cpu_test_hooks.o ds4_image.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o
 	$(CC) $(CFLAGS) -o $@ $^ $(LDLIBS)
 
+tests/test_tokenizer.o: tests/test_tokenizer.c ds4.h
+	$(CC) $(CFLAGS) -DDS4_TEST_HOOKS -I. -c -o $@ $<
+
+tests/test_tokenizer: tests/test_tokenizer.o ds4_cpu_test_hooks.o ds4_image.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o
+	$(CC) $(CFLAGS) -o $@ $^ $(LDLIBS)
+
 ifneq ($(UNAME_S),Darwin)
 tests/test_gpu_xdev.o: tests/test_gpu_xdev.c ds4_gpu.h ds4_gpu_mgpu.h
 	$(CC) $(CFLAGS) -I. -I$(CUDA_HOME)/include -c -o $@ $<
@@ -565,7 +571,7 @@ tests/test_prompt_prefix: tests/test_prompt_prefix.o ds4_prompt_prefix.o
 
 test: ds4_test ds4_agent_test ds4-eval q4k-dot-test mxfp4-dot-test \
 	tests/test_layer_pack tests/test_engine_mgpu_placement tests/test_gpu_args \
-	tests/test_deepseek4_vision_image tests/test_prompt_prefix $(SAMPLING_TEST) ds4 ds4-server ds4-bench ds4-agent
+	tests/test_deepseek4_vision_image tests/test_prompt_prefix tests/test_tokenizer $(SAMPLING_TEST) ds4 ds4-server ds4-bench ds4-agent
 	./ds4-eval --self-test-extractors
 	./ds4_agent_test
 	./ds4_test
@@ -575,6 +581,7 @@ test: ds4_test ds4_agent_test ds4-eval q4k-dot-test mxfp4-dot-test \
 	./tests/test_gpu_args_cli.sh
 	./tests/test_prompt_prefix
 	./tests/test_sampling
+	./tests/test_tokenizer
 	./tests/test_deepseek4_vision_image
 
 dspark-acceptance: ds4
@@ -611,4 +618,4 @@ mxfp4-dot-test: tests/test_mxfp4_dot.c
 	./tests/test_mxfp4_dot
 
 clean:
-	rm -f ds4 ds4-server ds4-bench ds4-eval ds4-agent ds4_cpu ds4_native ds4_server_test ds4_test ds4_agent_test gguf-tools/quality-testing/score_official gguf-tools/quality-testing/score_official.o speed-bench/metal_decode_schedule_bench speed-bench/metal_prefill_variant_bench speed-bench/*.o tests/test_q4k_dot tests/test_mxfp4_dot tests/test_mxfp4_metal tests/test_mxfp4_rocm tests/test_mxfp4_cuda tests/test_metal_session_batch tests/test_glm53_kda tests/test_glm53_kda_rocm tests/test_glm53_vision_engine tests/test_glm53_vision_prompt tests/test_deepseek4_vision_image tests/test_prompt_prefix tests/test_gpu_xdev tests/test_gpu_model_cache tests/test_gpu_lookup_cache_strict tests/test_engine_mgpu_refusal tests/test_engine_mgpu_runtime tests/test_engine_correctness tests/test_sampling tests/test_cuda_session_batch tests/test_cuda_mixed_batch tests/*.o *.o tests/cuda_long_context_smoke tests/cuda_long_context_smoke.o
+	rm -f ds4 ds4-server ds4-bench ds4-eval ds4-agent ds4_cpu ds4_native ds4_server_test ds4_test ds4_agent_test gguf-tools/quality-testing/score_official gguf-tools/quality-testing/score_official.o speed-bench/metal_decode_schedule_bench speed-bench/metal_prefill_variant_bench speed-bench/*.o tests/test_q4k_dot tests/test_mxfp4_dot tests/test_mxfp4_metal tests/test_mxfp4_rocm tests/test_mxfp4_cuda tests/test_metal_session_batch tests/test_glm53_kda tests/test_glm53_kda_rocm tests/test_glm53_vision_engine tests/test_glm53_vision_prompt tests/test_deepseek4_vision_image tests/test_prompt_prefix tests/test_gpu_xdev tests/test_gpu_model_cache tests/test_gpu_lookup_cache_strict tests/test_engine_mgpu_refusal tests/test_engine_mgpu_runtime tests/test_engine_correctness tests/test_sampling tests/test_tokenizer tests/test_cuda_session_batch tests/test_cuda_mixed_batch tests/*.o *.o tests/cuda_long_context_smoke tests/cuda_long_context_smoke.o

ds4.c

@@ -39515,6 +39515,10 @@ static int vocab_lookup_optional(const ds4_vocab *vocab, const char *text) {
     return token;
 }
 
+static int glm_chat_bos_fallback_id(int gmask_id, int sop_id) {
+    return gmask_id >= 0 ? gmask_id : sop_id;
+}
+
 /* Load token strings, special token ids, and merge ranks from GGUF metadata. */
 
 static void vocab_load(ds4_vocab *vocab, const ds4_model *model) {
@@ -39552,7 +39556,9 @@ static void vocab_load(ds4_vocab *vocab, const ds4_model *model) {
 
     if (DS4_MODEL_FAMILY == DS4_MODEL_FAMILY_GLM_DSA) {
         if (!model_get_token_id(model, "tokenizer.ggml.bos_token_id", &vocab->bos_id)) {
-            vocab->bos_id = vocab_lookup_optional(vocab, "<sop>");
+            vocab->bos_id = glm_chat_bos_fallback_id(
+                    vocab_lookup_optional(vocab, "[gMASK]"),
+                    vocab_lookup_optional(vocab, "<sop>"));
         }
         if (!model_get_token_id(model, "tokenizer.ggml.eos_token_id", &vocab->eos_id)) {
             vocab->eos_id = vocab_lookup_optional(vocab, "<|endoftext|>");
@@ -39608,7 +39614,8 @@ static void vocab_free(ds4_vocab *vocab) {
  * thinking is only a prompt prefix: the model still enters through <think>. */
 static void chat_push_bos_sequence(const ds4_vocab *vocab, token_vec *out) {
     token_vec_push(out, vocab->bos_id);
-    if (DS4_MODEL_FAMILY == DS4_MODEL_FAMILY_GLM_DSA && vocab->sop_id >= 0)
+    if (DS4_MODEL_FAMILY == DS4_MODEL_FAMILY_GLM_DSA &&
+        vocab->sop_id >= 0 && vocab->sop_id != vocab->bos_id)
         token_vec_push(out, vocab->sop_id);
 }
 
@@ -39754,6 +39761,50 @@ void ds4_tokenize_rendered_chat(ds4_engine *e, const char *text, ds4_tokens *out
     tokenize_rendered_chat_vocab(&e->vocab, text, out);
 }
 
+#ifdef DS4_TEST_HOOKS
+bool ds4_test_glm_chat_preamble(void) {
+    static const ds4_str gmask = {"[gMASK]", 7};
+    static const ds4_str sop = {"<sop>", 5};
+    const ds4_shape saved_shape = g_ds4_shape;
+    ds4_vocab vocab = {0};
+    ds4_vocab sop_only = {0};
+    token_vec direct = {0};
+    token_vec rendered = {0};
+    token_vec shared = {0};
+
+    table_init(&vocab.token_to_id, 2);
+    table_put(&vocab.token_to_id, gmask, 154822);
+    table_put(&vocab.token_to_id, sop, 154824);
+    vocab.bos_id = glm_chat_bos_fallback_id(
+            vocab_lookup_optional(&vocab, "[gMASK]"),
+            vocab_lookup_optional(&vocab, "<sop>"));
+    vocab.sop_id = vocab_lookup_optional(&vocab, "<sop>");
+
+    g_ds4_shape.family = DS4_MODEL_FAMILY_GLM_DSA;
+    chat_push_bos_sequence(&vocab, &direct);
+    tokenize_rendered_chat_vocab(&vocab, "[gMASK]<sop>", &rendered);
+    bool ok = direct.len == 2 && direct.v[0] == 154822 && direct.v[1] == 154824 &&
+              rendered.len == 2 && rendered.v[0] == 154822 && rendered.v[1] == 154824;
+
+    table_init(&sop_only.token_to_id, 1);
+    table_put(&sop_only.token_to_id, sop, 154824);
+    sop_only.bos_id = glm_chat_bos_fallback_id(
+            vocab_lookup_optional(&sop_only, "[gMASK]"),
+            vocab_lookup_optional(&sop_only, "<sop>"));
+    sop_only.sop_id = vocab_lookup_optional(&sop_only, "<sop>");
+    chat_push_bos_sequence(&sop_only, &shared);
+    ok = ok && shared.len == 1 && shared.v[0] == 154824;
+
+    token_vec_free(&shared);
+    token_vec_free(&rendered);
+    token_vec_free(&direct);
+    table_free(&sop_only.token_to_id);
+    table_free(&vocab.token_to_id);
+    g_ds4_shape = saved_shape;
+    return ok;
+}
+#endif
+
 void ds4_chat_begin(ds4_engine *e, ds4_tokens *tokens) {
     chat_push_bos_sequence(&e->vocab, tokens);
 }

ds4.h

@@ -478,6 +478,7 @@ int ds4_test_speculative_delta_sample(const float *target_logits,
 int ds4_test_argmax_excluding_logits(const float *logits, uint32_t n_vocab,
                                      int excluded_id);
 uint64_t ds4_test_mixed_native_count(void);
+bool ds4_test_glm_chat_preamble(void);
 #endif
 int ds4_session_top_logprobs(ds4_session *s, ds4_token_score *out, int k);
 int ds4_session_token_logprob(ds4_session *s, int token, ds4_token_score *out);

tests/test_tokenizer.c

@@ -0,0 +1,12 @@
+#include "../ds4.h"
+
+#include <stdio.h>
+
+int main(void) {
+    if (!ds4_test_glm_chat_preamble()) {
+        fputs("GLM chat preamble test failed\n", stderr);
+        return 1;
+    }
+    puts("tokenizer tests: OK");
+    return 0;
+}