Skip to content

Commit da3f990

Browse files
sfallahngxson
andauthored
mtmd: Add DeepSeekOCR 2 Support (ggml-org#20975)
* mtmd: DeepSeek-OCR 2 support, with multi-tile dynamic resolution * introduced clip_image_f32::add_viewsep * address PR review - drop redundant ggml_cpy ops in both deepseekocr versions build - drop no-op ggml_cont in build_sam - assert num_image_tokens deepseekocr2 - view_seperator as (1, n_embd) at conversion (for both versions) - drop redundant ggml_reshape_2d * Update tools/mtmd/models/deepseekocr2.cpp Co-authored-by: Xuan-Son Nguyen <thichthat@gmail.com> --------- Co-authored-by: Xuan-Son Nguyen <thichthat@gmail.com>
1 parent 6ed481e commit da3f990

16 files changed

Lines changed: 504 additions & 89 deletions

conversion/__init__.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -237,6 +237,7 @@
237237
MMPROJ_MODEL_MAP: dict[str, str] = {
238238
"AudioFlamingo3ForConditionalGeneration": "ultravox",
239239
"CogVLMForCausalLM": "cogvlm",
240+
"DeepseekOCR2ForCausalLM": "deepseek",
240241
"DeepseekOCRForCausalLM": "deepseek",
241242
"DotsOCRForCausalLM": "dotsocr",
242243
"Gemma3ForConditionalGeneration": "gemma",

conversion/base.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1140,7 +1140,7 @@ def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Ca
11401140
# Skip multimodal tensors
11411141
if name.startswith(("mlp", "vit.", "vpm.", "siglip2.", "conformer.", "merger.", "resampler.", "sound_encoder.", "sound_projection.", "speech_embeddings.")) \
11421142
or "visual." in name or "vision." in name or "audio." in name or "talker." in name \
1143-
or "vision_" in name or "audio_" in name or "sam_model" in name \
1143+
or "vision_" in name or "audio_" in name \
11441144
or "token2wav." in name or "code2wav." in name \
11451145
or "projector." in name or "pre_mm_projector_norm" in name \
11461146
or "image_newline" in name or "view_seperator" in name \

conversion/deepseek.py

Lines changed: 56 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,14 @@
1616

1717
@ModelBase.register("DeepseekOCRForCausalLM")
1818
class DeepseekOCRVisionModel(MmprojModel):
19+
def __init__(self, *args, **kwargs):
20+
super().__init__(*args, **kwargs)
21+
self.clip_projector_type = gguf.VisionProjectorType.DEEPSEEKOCR
22+
1923
def set_gguf_parameters(self):
2024
super().set_gguf_parameters()
2125
hparams = self.hparams
22-
self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.DEEPSEEKOCR)
26+
self.gguf_writer.add_clip_projector_type(self.clip_projector_type)
2327
# default values below are taken from HF tranformers code
2428
self.gguf_writer.add_vision_attention_layernorm_eps(hparams.get("layer_norm_eps", 1e-6))
2529
self.gguf_writer.add_vision_use_gelu(True)
@@ -49,22 +53,27 @@ def get_vision_config(self) -> dict[str, Any]:
4953
raise ValueError("DeepseekOCR model requires 'vision_config' in the model configuration, but it was not found")
5054

5155
vision_config['sam'] = vision_config['width']['sam_vit_b']
52-
vision_config.update(vision_config['width']['clip-l-14-224'])
53-
vision_config['hidden_size'] = vision_config['width']
54-
vision_config['num_heads'] = vision_config['heads']
55-
vision_config['intermediate_size'] = vision_config['heads'] * 4
56+
if vision_config['width'].get('clip-l-14-224') is not None:
57+
vision_config.update(vision_config['width']['clip-l-14-224'])
58+
if isinstance(vision_config['width'], int):
59+
vision_config['hidden_size'] = vision_config['width']
60+
if vision_config.get('heads') is not None:
61+
vision_config['num_heads'] = vision_config['heads']
62+
vision_config['intermediate_size'] = vision_config['heads'] * 4
5663

5764
return vision_config
5865

5966
def tensor_force_quant(self, name, new_name, bid, n_dims):
60-
if ".embeddings." in name or 'pos_embed' in name:
61-
return gguf.GGMLQuantizationType.F32
62-
if ".rel_pos_h" in name or '.rel_pos_w' in name:
63-
return gguf.GGMLQuantizationType.F32
64-
if ".neck." in name or ".net_" in name:
65-
return gguf.GGMLQuantizationType.F32
67+
for nq_name in ('.embeddings.', 'pos_embed', '.rel_pos_h', '.rel_pos_w', '.neck.', '.net_'):
68+
if nq_name in name:
69+
return gguf.GGMLQuantizationType.F32
6670
return super().tensor_force_quant(name, new_name, bid, n_dims)
6771

72+
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
73+
if name.endswith("view_seperator"):
74+
data_torch = data_torch.unsqueeze(0)
75+
yield from super().modify_tensors(data_torch, name, bid)
76+
6877
@classmethod
6978
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
7079
name, gen = item
@@ -81,6 +90,33 @@ def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Ca
8190
return super().filter_tensors((name, gen))
8291

8392

93+
@ModelBase.register("DeepseekOCR2ForCausalLM")
94+
class DeepseekOCR2VisionModel(DeepseekOCRVisionModel):
95+
def __init__(self, *args, **kwargs):
96+
super().__init__(*args, **kwargs)
97+
self.clip_projector_type = gguf.VisionProjectorType.DEEPSEEKOCR2
98+
99+
def set_gguf_parameters(self):
100+
# the vision tower's qwen2 encoder is built from fixed defaults,
101+
# see build_qwen2_decoder_as_encoder() in deepencoderv2.py
102+
if self.hparams.get("patch_size") is None:
103+
self.hparams["patch_size"] = 16
104+
if self.hparams.get("intermediate_size") is None:
105+
self.hparams["intermediate_size"] = 4864
106+
if self.hparams.get("num_attention_heads") is None:
107+
self.hparams["num_attention_heads"] = 14
108+
super().set_gguf_parameters()
109+
# qwen2 encoder is GQA: 14 Q heads, 2 KV heads
110+
self.gguf_writer.add_vision_head_count_kv(2)
111+
112+
def get_vision_config(self) -> dict[str, Any]:
113+
vision_config = super().get_vision_config()
114+
vision_config['hidden_size'] = vision_config['width']['qwen2-0-5b']['dim']
115+
if vision_config.get('layers') is None:
116+
vision_config['layers'] = 24
117+
return vision_config
118+
119+
84120
@ModelBase.register("DeepseekForCausalLM")
85121
class DeepseekModel(TextModel):
86122
model_arch = gguf.MODEL_ARCH.DEEPSEEK
@@ -188,13 +224,21 @@ def __init__(self, *args, **kwargs):
188224
self.origin_hf_arch = hparams.get('architectures', [None])[0]
189225

190226
# special handling for Deepseek OCR
191-
if self.origin_hf_arch == "DeepseekOCRForCausalLM":
227+
if self.origin_hf_arch in ("DeepseekOCRForCausalLM", "DeepseekOCR2ForCausalLM"):
192228
self.model_arch = gguf.MODEL_ARCH.DEEPSEEK2OCR
193229
self.gguf_writer.arch = gguf.MODEL_ARCH_NAMES[self.model_arch]
194230
self.gguf_writer.add_architecture()
195231
# default jinja template
196232
self.gguf_writer.add_chat_template("{% for m in messages %}{{m['content']}}{% endfor %}")
197233

234+
@classmethod
235+
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
236+
name, _ = item
237+
# DeepSeek-OCR vision encoder (SAM + DeepSeek-OCR-2 qwen2 tower)
238+
if "sam_model" in name or "qwen2_model" in name:
239+
return None
240+
return super().filter_tensors(item)
241+
198242
def set_vocab(self):
199243
try:
200244
self._set_vocab_gpt2()

gguf-py/gguf/constants.py

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -812,6 +812,8 @@ class MODEL_TENSOR(IntEnum):
812812
V_SAM_NET_3 = auto() # Deepseek-OCR
813813
V_ENC_EMBD_IMGNL = auto() # Deepseek-OCR
814814
V_ENC_EMBD_VSEP = auto() # Deepseek-OCR
815+
V_RESMPL_QUERY_768 = auto() # Deepseek-OCR-2
816+
V_RESMPL_QUERY_1024 = auto() # Deepseek-OCR-2
815817

816818
# audio (mtmd)
817819
A_ENC_EMBD_POS = auto()
@@ -1329,6 +1331,8 @@ class MODEL_TENSOR(IntEnum):
13291331
MODEL_TENSOR.V_SAM_NET_3: "v.sam.net_3",
13301332
MODEL_TENSOR.V_ENC_EMBD_IMGNL: "v.image_newline", # Deepseek-OCR
13311333
MODEL_TENSOR.V_ENC_EMBD_VSEP: "v.view_seperator", # Deepseek-OCR
1334+
MODEL_TENSOR.V_RESMPL_QUERY_768: "v.resample_query_768", # Deepseek-OCR-2 qwen2
1335+
MODEL_TENSOR.V_RESMPL_QUERY_1024: "v.resample_query_1024", # Deepseek-OCR-2 qwen2
13321336
# audio (mtmd)
13331337
# note: all audio tensor names must use prefix "a." or "mm.a."
13341338
MODEL_TENSOR.A_ENC_EMBD_POS: "a.position_embd",
@@ -1507,6 +1511,8 @@ class MODEL_TENSOR(IntEnum):
15071511
MODEL_TENSOR.V_SAM_NECK,
15081512
MODEL_TENSOR.V_SAM_NET_2,
15091513
MODEL_TENSOR.V_SAM_NET_3,
1514+
MODEL_TENSOR.V_RESMPL_QUERY_768,
1515+
MODEL_TENSOR.V_RESMPL_QUERY_1024,
15101516
# audio
15111517
MODEL_TENSOR.A_ENC_EMBD_POS,
15121518
MODEL_TENSOR.A_ENC_EMBD_NORM,
@@ -4329,6 +4335,7 @@ class VisionProjectorType:
43294335
JANUS_PRO = "janus_pro"
43304336
DOTSOCR = "dots_ocr"
43314337
DEEPSEEKOCR = "deepseekocr"
4338+
DEEPSEEKOCR2 = "deepseekocr2"
43324339
LFM2A = "lfm2a" # audio
43334340
MUSIC_FLAMINGO = "musicflamingo" # audio
43344341
GLM4V = "glm4v"

gguf-py/gguf/tensor_mapping.py

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1485,6 +1485,7 @@ class TensorNameMap:
14851485
"siglip2.vision_model.encoder.layers.{bid}.self_attn.q_proj", # youtuvl
14861486
"model.vision_model.transformer.layers.{bid}.self_attn.q_proj", # Deepseek-OCR CLIP, generated
14871487
"vision_model.model.layers.{bid}.self_attn.q_proj.linear", # gemma4
1488+
"model.qwen2_model.model.model.layers.{bid}.self_attn.q_proj" # Deepseek-OCR-2 qwen2
14881489
),
14891490

14901491
MODEL_TENSOR.V_ENC_ATTN_Q_NORM: (
@@ -1509,6 +1510,7 @@ class TensorNameMap:
15091510
"model.vision_model.transformer.layers.{bid}.self_attn.k_proj", # Deepseek-OCR CLIP, generated
15101511
"siglip2.vision_model.encoder.layers.{bid}.self_attn.k_proj",
15111512
"vision_model.model.layers.{bid}.self_attn.k_proj.linear", # gemma4
1513+
"model.qwen2_model.model.model.layers.{bid}.self_attn.k_proj" # Deepseek-OCR-2 qwen2
15121514
),
15131515

15141516
MODEL_TENSOR.V_ENC_ATTN_K_NORM: (
@@ -1533,6 +1535,7 @@ class TensorNameMap:
15331535
"siglip2.vision_model.encoder.layers.{bid}.self_attn.v_proj",
15341536
"model.vision_model.transformer.layers.{bid}.self_attn.v_proj", # Deepseek-OCR CLIP, generated
15351537
"vision_model.model.layers.{bid}.self_attn.v_proj.linear", # gemma4
1538+
"model.qwen2_model.model.model.layers.{bid}.self_attn.v_proj" # Deepseek-OCR-2 qwen2
15361539
),
15371540

15381541
MODEL_TENSOR.V_ENC_INPUT_NORM: (
@@ -1554,6 +1557,7 @@ class TensorNameMap:
15541557
"vision_model.radio_model.model.blocks.{bid}.norm1", # Nemotron Nano v2 VL
15551558
"vision_tower.blocks.{bid}.norm1", # dots.ocr
15561559
"vision_model.transformer.resblocks.{bid}.ln_1", # Step3-VL
1560+
"model.qwen2_model.model.model.layers.{bid}.input_layernorm", # Deepseek-OCR-2 qwen2
15571561
),
15581562

15591563
MODEL_TENSOR.V_ENC_ATTN_O: (
@@ -1574,6 +1578,7 @@ class TensorNameMap:
15741578
"model.vision_model.transformer.layers.{bid}.self_attn.out_proj", # Deepseek-OCR CLIP
15751579
"siglip2.vision_model.encoder.layers.{bid}.self_attn.out_proj", # youtuvl
15761580
"vision_model.radio_model.model.blocks.{bid}.attn.proj", # Nemotron Nano v2 VL
1581+
"model.qwen2_model.model.model.layers.{bid}.self_attn.o_proj", # Deepseek-OCR-2 qwen2
15771582
"vision_model.model.layers.{bid}.self_attn.o_proj.linear", # gemma4
15781583
"vision_tower.blocks.{bid}.attn.proj", # dots.ocr
15791584
"vision_model.transformer.resblocks.{bid}.attn.out_proj", # Step3-VL
@@ -1603,6 +1608,7 @@ class TensorNameMap:
16031608
"vision_model.model.layers.{bid}.pre_feedforward_layernorm", # gemma4
16041609
"vision_tower.blocks.{bid}.norm2", # dots.ocr
16051610
"vision_model.transformer.resblocks.{bid}.ln_2", # Step3-VL
1611+
"model.qwen2_model.model.model.layers.{bid}.post_attention_layernorm", # Deepseek-OCR-2 qwen2
16061612
),
16071613

16081614
MODEL_TENSOR.V_ENC_FFN_UP: (
@@ -1625,13 +1631,15 @@ class TensorNameMap:
16251631
"vision_model.radio_model.model.blocks.{bid}.mlp.fc1", # Nemotron Nano v2 VL
16261632
"vision_model.model.layers.{bid}.mlp.up_proj", # gemma4
16271633
"vision_model.transformer.resblocks.{bid}.mlp.c_fc", # Step3-VL
1634+
"model.qwen2_model.model.model.layers.{bid}.mlp.up_proj", # Deepseek-OCR-2 qwen2
16281635
),
16291636

16301637
MODEL_TENSOR.V_ENC_FFN_GATE: (
16311638
"vision_tower.transformer.layers.{bid}.feed_forward.gate_proj", # pixtral-hf
16321639
"vision_encoder.transformer.layers.{bid}.feed_forward.w1", # pixtral
16331640
"visual.blocks.{bid}.mlp.gate_proj", # qwen2.5vl
16341641
"vision_model.model.layers.{bid}.mlp.gate_proj", # gemma4
1642+
"model.qwen2_model.model.model.layers.{bid}.mlp.gate_proj", # Deepseek-OCR-2 qwen2
16351643
),
16361644

16371645
MODEL_TENSOR.V_ENC_FFN_DOWN: (
@@ -1652,6 +1660,7 @@ class TensorNameMap:
16521660
"model.vision_model.transformer.layers.{bid}.mlp.fc2", # Deepseek-OCR CLIP
16531661
"siglip2.vision_model.encoder.layers.{bid}.mlp.fc2",
16541662
"vision_model.radio_model.model.blocks.{bid}.mlp.fc2", # Nemotron Nano v2 VL
1663+
"model.qwen2_model.model.model.layers.{bid}.mlp.down_proj" , # Deepseek-OCR-2 qwen2
16551664
"vision_model.model.layers.{bid}.mlp.down_proj", # gemma4
16561665
"vision_model.transformer.resblocks.{bid}.mlp.c_proj", # Step3-VL
16571666
),
@@ -1699,6 +1708,7 @@ class TensorNameMap:
16991708
"vision_tower.encoder.final_layernorm", # kimi-vl
17001709
"visual.post_layernorm", # glm4v
17011710
"siglip2.vision_model.post_layernorm",
1711+
"model.qwen2_model.model.model.norm", # Deepseek-OCR-2 qwen2
17021712
),
17031713

17041714
MODEL_TENSOR.V_MM_POST_NORM: (
@@ -1879,6 +1889,14 @@ class TensorNameMap:
18791889
"model.sam_model.net_3",
18801890
),
18811891

1892+
MODEL_TENSOR.V_RESMPL_QUERY_768: (
1893+
"model.qwen2_model.query_768", # Deepseek-OCR-2 qwen2
1894+
),
1895+
1896+
MODEL_TENSOR.V_RESMPL_QUERY_1024: (
1897+
"model.qwen2_model.query_1024", # Deepseek-OCR-2 qwen2
1898+
),
1899+
18821900
MODEL_TENSOR.V_MM_POST_FC_NORM: (
18831901
"model.vision.linear_proj.norm1", # cogvlm
18841902
),

tools/mtmd/CMakeLists.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ add_library(mtmd
4040
models/siglip.cpp
4141
models/whisper-enc.cpp
4242
models/deepseekocr.cpp
43+
models/deepseekocr2.cpp
4344
models/mobilenetv5.cpp
4445
models/youtuvl.cpp
4546
models/yasa2.cpp

tools/mtmd/clip-impl.h

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,8 @@
188188
#define TN_SAM_FFN_DOWN "v.sam.blk.%d.mlp.lin2.%s"
189189
#define TN_SAM_NECK "v.sam.neck.%d.%s"
190190
#define TN_SAM_NET "v.sam.net_%d.%s"
191+
// deepseek-ocr-2
192+
#define TN_RESMPL_QUERY "v.resample_query_%d.%s"
191193
// (conformer) lfm2
192194
#define TN_PRE_ENCODE_OUT "a.pre_encode.out.%s"
193195
#define TN_FFN_NORM "%s.blk.%d.ffn_norm.%s"
@@ -337,6 +339,7 @@ enum projector_type {
337339
PROJECTOR_TYPE_JANUS_PRO,
338340
PROJECTOR_TYPE_DOTS_OCR,
339341
PROJECTOR_TYPE_DEEPSEEKOCR,
342+
PROJECTOR_TYPE_DEEPSEEKOCR2,
340343
PROJECTOR_TYPE_LFM2A,
341344
PROJECTOR_TYPE_GLM4V,
342345
PROJECTOR_TYPE_YOUTUVL,
@@ -386,6 +389,7 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
386389
{ PROJECTOR_TYPE_JANUS_PRO, "janus_pro"},
387390
{ PROJECTOR_TYPE_DOTS_OCR, "dots_ocr"},
388391
{ PROJECTOR_TYPE_DEEPSEEKOCR,"deepseekocr"},
392+
{ PROJECTOR_TYPE_DEEPSEEKOCR2,"deepseekocr2"},
389393
{ PROJECTOR_TYPE_LFM2A, "lfm2a"},
390394
{ PROJECTOR_TYPE_GLM4V, "glm4v"},
391395
{ PROJECTOR_TYPE_YOUTUVL, "youtuvl"},
@@ -424,6 +428,9 @@ struct clip_image_f32 {
424428
int ny;
425429

426430
std::vector<float> buf;
431+
432+
// marks the global view in e.g., DeepSeek-OCR Models
433+
bool add_viewsep = false;
427434
};
428435

429436
//

tools/mtmd/clip-model.h

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -542,6 +542,11 @@ struct clip_model {
542542
int32_t n_sam_layers = 12; // used by deepseek-ocr sam encoder
543543

544544
std::vector<clip_layer> sam_layers;
545+
546+
// deepseek-ocr-2
547+
ggml_tensor * resample_query_768 = nullptr;
548+
ggml_tensor * resample_query_1024 = nullptr;
549+
545550
// lfm2 audio
546551
std::array<ggml_tensor *, 7> pre_encode_conv_X_w = {nullptr};
547552
std::array<ggml_tensor *, 7> pre_encode_conv_X_b = {nullptr};

0 commit comments

Comments
 (0)