From 59cd9a1a5713ba5762f32aac821551a9afe4aa69 Mon Sep 17 00:00:00 2001 From: Mantas Vidutis Date: Tue, 1 Sep 2026 15:28:47 -0700 Subject: [PATCH] glm: fix metadata-less chat preamble --- Makefile | 11 +++++++-- ds4.c | 55 ++++++++++++++++++++++++++++++++++++++++-- ds4.h | 1 + tests/test_tokenizer.c | 12 +++++++++ 4 files changed, 75 insertions(+), 4 deletions(-) create mode 100644 tests/test_tokenizer.c diff --git a/Makefile b/Makefile index c58acb939..1b8bf97d9 100644 --- a/Makefile +++ b/Makefile @@ -484,6 +484,12 @@ tests/test_sampling.o: tests/test_sampling.c ds4.h tests/test_sampling: tests/test_sampling.o ds4_cpu_test_hooks.o ds4_image.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o $(CC) $(CFLAGS) -o $@ $^ $(LDLIBS) +tests/test_tokenizer.o: tests/test_tokenizer.c ds4.h + $(CC) $(CFLAGS) -DDS4_TEST_HOOKS -I. -c -o $@ $< + +tests/test_tokenizer: tests/test_tokenizer.o ds4_cpu_test_hooks.o ds4_image.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o + $(CC) $(CFLAGS) -o $@ $^ $(LDLIBS) + ifneq ($(UNAME_S),Darwin) tests/test_gpu_xdev.o: tests/test_gpu_xdev.c ds4_gpu.h ds4_gpu_mgpu.h $(CC) $(CFLAGS) -I. -I$(CUDA_HOME)/include -c -o $@ $< @@ -565,7 +571,7 @@ tests/test_prompt_prefix: tests/test_prompt_prefix.o ds4_prompt_prefix.o test: ds4_test ds4_agent_test ds4-eval q4k-dot-test mxfp4-dot-test \ tests/test_layer_pack tests/test_engine_mgpu_placement tests/test_gpu_args \ - tests/test_deepseek4_vision_image tests/test_prompt_prefix $(SAMPLING_TEST) ds4 ds4-server ds4-bench ds4-agent + tests/test_deepseek4_vision_image tests/test_prompt_prefix tests/test_tokenizer $(SAMPLING_TEST) ds4 ds4-server ds4-bench ds4-agent ./ds4-eval --self-test-extractors ./ds4_agent_test ./ds4_test @@ -575,6 +581,7 @@ test: ds4_test ds4_agent_test ds4-eval q4k-dot-test mxfp4-dot-test \ ./tests/test_gpu_args_cli.sh ./tests/test_prompt_prefix ./tests/test_sampling + ./tests/test_tokenizer ./tests/test_deepseek4_vision_image dspark-acceptance: ds4 @@ -611,4 +618,4 @@ mxfp4-dot-test: tests/test_mxfp4_dot.c ./tests/test_mxfp4_dot clean: - rm -f ds4 ds4-server ds4-bench ds4-eval ds4-agent ds4_cpu ds4_native ds4_server_test ds4_test ds4_agent_test gguf-tools/quality-testing/score_official gguf-tools/quality-testing/score_official.o speed-bench/metal_decode_schedule_bench speed-bench/metal_prefill_variant_bench speed-bench/*.o tests/test_q4k_dot tests/test_mxfp4_dot tests/test_mxfp4_metal tests/test_mxfp4_rocm tests/test_mxfp4_cuda tests/test_metal_session_batch tests/test_glm53_kda tests/test_glm53_kda_rocm tests/test_glm53_vision_engine tests/test_glm53_vision_prompt tests/test_deepseek4_vision_image tests/test_prompt_prefix tests/test_gpu_xdev tests/test_gpu_model_cache tests/test_gpu_lookup_cache_strict tests/test_engine_mgpu_refusal tests/test_engine_mgpu_runtime tests/test_engine_correctness tests/test_sampling tests/test_cuda_session_batch tests/test_cuda_mixed_batch tests/*.o *.o tests/cuda_long_context_smoke tests/cuda_long_context_smoke.o + rm -f ds4 ds4-server ds4-bench ds4-eval ds4-agent ds4_cpu ds4_native ds4_server_test ds4_test ds4_agent_test gguf-tools/quality-testing/score_official gguf-tools/quality-testing/score_official.o speed-bench/metal_decode_schedule_bench speed-bench/metal_prefill_variant_bench speed-bench/*.o tests/test_q4k_dot tests/test_mxfp4_dot tests/test_mxfp4_metal tests/test_mxfp4_rocm tests/test_mxfp4_cuda tests/test_metal_session_batch tests/test_glm53_kda tests/test_glm53_kda_rocm tests/test_glm53_vision_engine tests/test_glm53_vision_prompt tests/test_deepseek4_vision_image tests/test_prompt_prefix tests/test_gpu_xdev tests/test_gpu_model_cache tests/test_gpu_lookup_cache_strict tests/test_engine_mgpu_refusal tests/test_engine_mgpu_runtime tests/test_engine_correctness tests/test_sampling tests/test_tokenizer tests/test_cuda_session_batch tests/test_cuda_mixed_batch tests/*.o *.o tests/cuda_long_context_smoke tests/cuda_long_context_smoke.o diff --git a/ds4.c b/ds4.c index 91ab214ab..bbae0571e 100644 --- a/ds4.c +++ b/ds4.c @@ -39515,6 +39515,10 @@ static int vocab_lookup_optional(const ds4_vocab *vocab, const char *text) { return token; } +static int glm_chat_bos_fallback_id(int gmask_id, int sop_id) { + return gmask_id >= 0 ? gmask_id : sop_id; +} + /* Load token strings, special token ids, and merge ranks from GGUF metadata. */ static void vocab_load(ds4_vocab *vocab, const ds4_model *model) { @@ -39552,7 +39556,9 @@ static void vocab_load(ds4_vocab *vocab, const ds4_model *model) { if (DS4_MODEL_FAMILY == DS4_MODEL_FAMILY_GLM_DSA) { if (!model_get_token_id(model, "tokenizer.ggml.bos_token_id", &vocab->bos_id)) { - vocab->bos_id = vocab_lookup_optional(vocab, ""); + vocab->bos_id = glm_chat_bos_fallback_id( + vocab_lookup_optional(vocab, "[gMASK]"), + vocab_lookup_optional(vocab, "")); } if (!model_get_token_id(model, "tokenizer.ggml.eos_token_id", &vocab->eos_id)) { vocab->eos_id = vocab_lookup_optional(vocab, "<|endoftext|>"); @@ -39608,7 +39614,8 @@ static void vocab_free(ds4_vocab *vocab) { * thinking is only a prompt prefix: the model still enters through . */ static void chat_push_bos_sequence(const ds4_vocab *vocab, token_vec *out) { token_vec_push(out, vocab->bos_id); - if (DS4_MODEL_FAMILY == DS4_MODEL_FAMILY_GLM_DSA && vocab->sop_id >= 0) + if (DS4_MODEL_FAMILY == DS4_MODEL_FAMILY_GLM_DSA && + vocab->sop_id >= 0 && vocab->sop_id != vocab->bos_id) token_vec_push(out, vocab->sop_id); } @@ -39754,6 +39761,50 @@ void ds4_tokenize_rendered_chat(ds4_engine *e, const char *text, ds4_tokens *out tokenize_rendered_chat_vocab(&e->vocab, text, out); } +#ifdef DS4_TEST_HOOKS +bool ds4_test_glm_chat_preamble(void) { + static const ds4_str gmask = {"[gMASK]", 7}; + static const ds4_str sop = {"", 5}; + const ds4_shape saved_shape = g_ds4_shape; + ds4_vocab vocab = {0}; + ds4_vocab sop_only = {0}; + token_vec direct = {0}; + token_vec rendered = {0}; + token_vec shared = {0}; + + table_init(&vocab.token_to_id, 2); + table_put(&vocab.token_to_id, gmask, 154822); + table_put(&vocab.token_to_id, sop, 154824); + vocab.bos_id = glm_chat_bos_fallback_id( + vocab_lookup_optional(&vocab, "[gMASK]"), + vocab_lookup_optional(&vocab, "")); + vocab.sop_id = vocab_lookup_optional(&vocab, ""); + + g_ds4_shape.family = DS4_MODEL_FAMILY_GLM_DSA; + chat_push_bos_sequence(&vocab, &direct); + tokenize_rendered_chat_vocab(&vocab, "[gMASK]", &rendered); + bool ok = direct.len == 2 && direct.v[0] == 154822 && direct.v[1] == 154824 && + rendered.len == 2 && rendered.v[0] == 154822 && rendered.v[1] == 154824; + + table_init(&sop_only.token_to_id, 1); + table_put(&sop_only.token_to_id, sop, 154824); + sop_only.bos_id = glm_chat_bos_fallback_id( + vocab_lookup_optional(&sop_only, "[gMASK]"), + vocab_lookup_optional(&sop_only, "")); + sop_only.sop_id = vocab_lookup_optional(&sop_only, ""); + chat_push_bos_sequence(&sop_only, &shared); + ok = ok && shared.len == 1 && shared.v[0] == 154824; + + token_vec_free(&shared); + token_vec_free(&rendered); + token_vec_free(&direct); + table_free(&sop_only.token_to_id); + table_free(&vocab.token_to_id); + g_ds4_shape = saved_shape; + return ok; +} +#endif + void ds4_chat_begin(ds4_engine *e, ds4_tokens *tokens) { chat_push_bos_sequence(&e->vocab, tokens); } diff --git a/ds4.h b/ds4.h index e6dae1b9f..7c103b578 100644 --- a/ds4.h +++ b/ds4.h @@ -478,6 +478,7 @@ int ds4_test_speculative_delta_sample(const float *target_logits, int ds4_test_argmax_excluding_logits(const float *logits, uint32_t n_vocab, int excluded_id); uint64_t ds4_test_mixed_native_count(void); +bool ds4_test_glm_chat_preamble(void); #endif int ds4_session_top_logprobs(ds4_session *s, ds4_token_score *out, int k); int ds4_session_token_logprob(ds4_session *s, int token, ds4_token_score *out); diff --git a/tests/test_tokenizer.c b/tests/test_tokenizer.c new file mode 100644 index 000000000..5854648c0 --- /dev/null +++ b/tests/test_tokenizer.c @@ -0,0 +1,12 @@ +#include "../ds4.h" + +#include + +int main(void) { + if (!ds4_test_glm_chat_preamble()) { + fputs("GLM chat preamble test failed\n", stderr); + return 1; + } + puts("tokenizer tests: OK"); + return 0; +}