Skip to content

Fix DeepSeek Vision-Exp image inference with SSD streaming on Metal #937

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
cropduster wants to merge 1 commit into
base: main
Choose a base branch
from
+132 −15
Open

Fix DeepSeek Vision-Exp image inference with SSD streaming on Metal #937

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 21 additions & 9 deletions ds4.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38394,6 +38394,10 @@ struct ds4_engine {
bool mtp_ready;
bool vision_ready;
bool vision_map_ready;
/* Generation of the Metal model-view set at the time vision_map_ready
* was last established; a mismatch means the views were cleared and the
* sidecar map must be re-registered. */
uint64_t vision_map_generation;
bool share_session_prefill_workspace;
#ifndef DS4_NO_GPU
bool shared_prefill_workspace_ready;
Expand Down Expand Up @@ -63410,23 +63414,27 @@ static int ds4_engine_open_internal(ds4_engine **out,
}
if (e->vision_ready) {
#if defined(__APPLE__)
e->vision_map_ready = ds4_gpu_set_model_map_range(
e->vision_map_ready = ds4_gpu_set_aux_model_map_range(
e->vision_model.map,
e->vision_model.size,
e->vision_model.tensor_data_pos,
e->vision_model.size - e->vision_model.tensor_data_pos,
e->vision_model.max_tensor_bytes) != 0;
e->vision_model.size - e->vision_model.tensor_data_pos) != 0;
#else
e->vision_map_ready = ds4_gpu_set_aux_model_map_range(
e->vision_model.map,
e->vision_model.size,
e->vision_model.tensor_data_pos,
e->vision_model.size - e->vision_model.tensor_data_pos) != 0;
#endif
if (e->vision_map_ready) {
e->vision_map_generation = ds4_gpu_model_views_generation();
}
if (!e->vision_map_ready) {
fprintf(stderr,
"ds4: %s failed to map the GLM-5.3 vision encoder\n",
ds4_backend_name(e->backend));
"ds4: %s failed to map the %s vision encoder\n",
ds4_backend_name(e->backend),
e->vision_kind == DS4_VISION_DEEPSEEK4 ?
"DeepSeek" : "GLM-5.3");
ds4_engine_close(e);
*out = NULL;
return 1;
Expand Down Expand Up @@ -63884,21 +63892,25 @@ static int ds4_engine_vision_encode_image(
if (!e->metal_ready) {
e->metal_ready = ds4_gpu_init() != 0;
}
if (e->metal_ready && !e->vision_map_ready) {
if (e->metal_ready &&
(!e->vision_map_ready ||
e->vision_map_generation != ds4_gpu_model_views_generation())) {
#if defined(__APPLE__)
e->vision_map_ready = ds4_gpu_set_model_map_range(
e->vision_map_ready = ds4_gpu_set_aux_model_map_range(
e->vision_model.map,
e->vision_model.size,
e->vision_model.tensor_data_pos,
e->vision_model.size - e->vision_model.tensor_data_pos,
e->vision_model.max_tensor_bytes) != 0;
e->vision_model.size - e->vision_model.tensor_data_pos) != 0;
#else
e->vision_map_ready = ds4_gpu_set_aux_model_map_range(
e->vision_model.map,
e->vision_model.size,
e->vision_model.tensor_data_pos,
e->vision_model.size - e->vision_model.tensor_data_pos) != 0;
#endif
if (e->vision_map_ready) {
e->vision_map_generation = ds4_gpu_model_views_generation();
}
}
if (!e->metal_ready || !e->vision_map_ready) {
if (error && error_cap) {
Expand Down
6 changes: 6 additions & 0 deletions ds4_cuda.cu
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3794,6 +3794,12 @@ extern "C" int ds4_gpu_set_model_map_range(const void *model_map, uint64_t model
return 1;
}

extern "C" uint64_t ds4_gpu_model_views_generation(void) {
/* The CUDA backend never clears registered model views, so the
* generation is constant. */
return 0;
}

extern "C" int ds4_gpu_set_aux_model_map_range(
const void *model_map,
uint64_t model_size,
Expand Down
4 changes: 4 additions & 0 deletions ds4_gpu.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,10 @@ int ds4_gpu_set_aux_model_map_range(const void *model_map,
uint64_t map_offset,
uint64_t map_size);
int ds4_gpu_set_model_map_spans(const void *model_map, uint64_t model_size, const uint64_t *offsets, const uint64_t *sizes, uint32_t count, uint64_t max_tensor_bytes);
/* Current invalidation generation of the mapped model views. Incremented
* whenever registered views are cleared; callers must re-check before
* relying on a previously registered auxiliary/vision view. */
uint64_t ds4_gpu_model_views_generation(void);
int ds4_gpu_cache_model_range(const void *model_map, uint64_t model_size, uint64_t offset, uint64_t bytes, const char *label);
int ds4_gpu_cache_q8_f16_range(const void *model_map, uint64_t model_size, uint64_t offset, uint64_t bytes, uint64_t in_dim, uint64_t out_dim, const char *label);
int ds4_gpu_q8_cache_suppressed(void);
Expand Down
101 changes: 95 additions & 6 deletions ds4_metal.m
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -366,6 +366,7 @@
static id<MTLBuffer> g_attn_out_group_ids_buffer;
static int g_model_fd = -1;
static const void *g_model_map_ptr;
static const void *g_last_aux_map;
static uint64_t g_model_map_size;
static uint64_t g_model_mapped_offset;
static uint64_t g_model_mapped_size;
Expand Down Expand Up @@ -607,6 +608,10 @@ static void ds4_gpu_print_device_summary(void) {

static ds4_gpu_model_view g_model_views[DS4_METAL_MAX_MODEL_VIEWS];
static uint32_t g_model_view_count;
/* Bumped whenever previously registered model views are invalidated, so
* callers holding stale view registrations (e.g. the vision sidecar map)
* can detect that their views no longer exist. */
static uint64_t g_model_views_generation;

enum {
DS4_METAL_STREAM_EXPERT_CACHE_MAX_LAYER = 80,
Expand Down Expand Up @@ -1690,6 +1695,32 @@ static void ds4_gpu_model_views_clear(void) {
g_model_views[i].bytes = 0;
}
g_model_view_count = 0;
g_model_views_generation++;
}

/* Drop only the views belonging to one GGUF mapping (identified by map
* pointer and size) and keep views of other mappings alive (e.g. the
* vision sidecar map). Bumps the generation when anything was removed. */
static void ds4_gpu_model_views_clear_for_map(const void *model_map,
uint64_t model_size) {
uint32_t out = 0;
for (uint32_t i = 0; i < g_model_view_count; i++) {
if (g_model_views[i].model_map == model_map &&
g_model_views[i].model_size == model_size) {
g_model_views[i].buffer = nil;
g_model_views[i].model_map = NULL;
g_model_views[i].model_size = 0;
g_model_views[i].model_offset = 0;
g_model_views[i].bytes = 0;
continue;
}
g_model_views[out] = g_model_views[i];
out++;
}
if (out != g_model_view_count) {
g_model_view_count = out;
g_model_views_generation++;
}
}

static void ds4_gpu_model_residency_clear(void) {
Expand Down Expand Up @@ -11289,6 +11320,55 @@ int ds4_gpu_embed_tokens_hc_tensor(
return 1;
}

static int ds4_gpu_model_views_cover_range(
const void *model_map,
uint64_t model_size,
uint64_t offset,
uint64_t size);

uint64_t ds4_gpu_model_views_generation(void) {
return g_model_views_generation;
}

int ds4_gpu_set_aux_model_map_range(
const void *model_map,
uint64_t model_size,
uint64_t map_offset,
uint64_t map_size) {
/* Register a secondary GGUF mapping (e.g. the vision encoder sidecar)
* WITHOUT replacing the primary model map globals: only append the
* views for this map. */
if (!g_initialized && !ds4_gpu_init()) return 0;
if (!model_map || model_size == 0) return 0;
if (map_offset > model_size || map_size == 0 ||
map_size > model_size - map_offset) return 0;

if (ds4_gpu_model_views_cover_range(model_map,
model_size,
map_offset,
map_size)) {
return 1;
}

@autoreleasepool {
uint64_t mapped_total = 0;
if (!ds4_gpu_add_model_view_range(model_map,
model_size,
map_offset,
map_size,
0,
false,
&mapped_total)) {
return 0;
}
if (!ds4_gpu_finish_model_views(0.0, mapped_total, map_offset)) {
return 0;
}
g_last_aux_map = model_map;
return 1;
}
}

int ds4_gpu_set_model_map_range(const void *model_map, uint64_t model_size, uint64_t map_offset, uint64_t map_size, uint64_t max_tensor_bytes) {
if (!g_initialized && !ds4_gpu_init()) return 0;
if (!model_map || model_size == 0) return 0;
Expand Down Expand Up @@ -11363,16 +11443,18 @@ int ds4_gpu_set_model_map_spans(
const double t0 = ds4_gpu_now_ms();
max_tensor_bytes = ds4_gpu_effective_model_max_tensor_bytes(model_size, max_tensor_bytes);

/* Drop only this mapping's views; keep aux/sidecar views (e.g. the
* vision encoder) of other mappings alive. */
ds4_gpu_model_views_clear_for_map(model_map, model_size);
ds4_gpu_model_residency_clear();
ds4_gpu_model_views_clear();

uint64_t mapped_total = 0;
uint64_t first_offset = UINT64_MAX;
for (uint32_t i = 0; i < count; i++) {
if (offsets[i] > model_size || sizes[i] == 0 || sizes[i] > model_size - offsets[i]) {
fprintf(stderr, "ds4: Metal model span %u is outside the GGUF mapping\n", i);
ds4_gpu_model_views_clear_for_map(model_map, model_size);
ds4_gpu_model_residency_clear();
ds4_gpu_model_views_clear();
return 0;
}
if (offsets[i] < first_offset) first_offset = offsets[i];
Expand All @@ -11385,14 +11467,14 @@ int ds4_gpu_set_model_map_spans(
effective_max,
true,
&mapped_total)) {
ds4_gpu_model_views_clear_for_map(model_map, model_size);
ds4_gpu_model_residency_clear();
ds4_gpu_model_views_clear();
return 0;
}
}
if (!ds4_gpu_finish_model_views(t0, mapped_total, first_offset)) {
ds4_gpu_model_views_clear_for_map(model_map, model_size);
ds4_gpu_model_residency_clear();
ds4_gpu_model_views_clear();
return 0;
}
g_model_map_ptr = model_map;
Expand Down Expand Up @@ -11492,9 +11574,16 @@ int ds4_gpu_set_model_fd_for_map(int fd, const void *model_map) {
}

fprintf(stderr,
"ds4: Metal model range %.2f..%.2f GiB is not covered by mapped model views\n",
"ds4: Metal model range %.2f..%.2f GiB is not covered by mapped model views "
"(map=%p primary=%d aux=%d off=0x%llx len=0x%llx views=%u)\n",
ds4_gpu_gib(offset),
ds4_gpu_gib(end));
ds4_gpu_gib(end),
model_map,
model_map == g_model_map_ptr,
model_map == g_last_aux_map,
(unsigned long long)offset,
(unsigned long long)len,
g_model_view_count);
return nil;
}

Expand Down
6 changes: 6 additions & 0 deletions rocm/ds4_rocm_runtime.cuh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6215,6 +6215,12 @@ extern "C" int ds4_gpu_set_model_map_range(const void *model_map, uint64_t model
return 1;
}

extern "C" uint64_t ds4_gpu_model_views_generation(void) {
/* The ROCm backend never clears registered model views, so the
* generation is constant. */
return 0;
}

extern "C" int ds4_gpu_set_aux_model_map_range(
const void *model_map,
uint64_t model_size,
Expand Down