diff --git a/ds4.c b/ds4.c index b54075539e..71bd237f43 100644 --- a/ds4.c +++ b/ds4.c @@ -38394,6 +38394,10 @@ struct ds4_engine { bool mtp_ready; bool vision_ready; bool vision_map_ready; + /* Generation of the Metal model-view set at the time vision_map_ready + * was last established; a mismatch means the views were cleared and the + * sidecar map must be re-registered. */ + uint64_t vision_map_generation; bool share_session_prefill_workspace; #ifndef DS4_NO_GPU bool shared_prefill_workspace_ready; @@ -63410,12 +63414,11 @@ static int ds4_engine_open_internal(ds4_engine **out, } if (e->vision_ready) { #if defined(__APPLE__) - e->vision_map_ready = ds4_gpu_set_model_map_range( + e->vision_map_ready = ds4_gpu_set_aux_model_map_range( e->vision_model.map, e->vision_model.size, e->vision_model.tensor_data_pos, - e->vision_model.size - e->vision_model.tensor_data_pos, - e->vision_model.max_tensor_bytes) != 0; + e->vision_model.size - e->vision_model.tensor_data_pos) != 0; #else e->vision_map_ready = ds4_gpu_set_aux_model_map_range( e->vision_model.map, @@ -63423,10 +63426,15 @@ static int ds4_engine_open_internal(ds4_engine **out, e->vision_model.tensor_data_pos, e->vision_model.size - e->vision_model.tensor_data_pos) != 0; #endif + if (e->vision_map_ready) { + e->vision_map_generation = ds4_gpu_model_views_generation(); + } if (!e->vision_map_ready) { fprintf(stderr, - "ds4: %s failed to map the GLM-5.3 vision encoder\n", - ds4_backend_name(e->backend)); + "ds4: %s failed to map the %s vision encoder\n", + ds4_backend_name(e->backend), + e->vision_kind == DS4_VISION_DEEPSEEK4 ? + "DeepSeek" : "GLM-5.3"); ds4_engine_close(e); *out = NULL; return 1; @@ -63884,14 +63892,15 @@ static int ds4_engine_vision_encode_image( if (!e->metal_ready) { e->metal_ready = ds4_gpu_init() != 0; } - if (e->metal_ready && !e->vision_map_ready) { + if (e->metal_ready && + (!e->vision_map_ready || + e->vision_map_generation != ds4_gpu_model_views_generation())) { #if defined(__APPLE__) - e->vision_map_ready = ds4_gpu_set_model_map_range( + e->vision_map_ready = ds4_gpu_set_aux_model_map_range( e->vision_model.map, e->vision_model.size, e->vision_model.tensor_data_pos, - e->vision_model.size - e->vision_model.tensor_data_pos, - e->vision_model.max_tensor_bytes) != 0; + e->vision_model.size - e->vision_model.tensor_data_pos) != 0; #else e->vision_map_ready = ds4_gpu_set_aux_model_map_range( e->vision_model.map, @@ -63899,6 +63908,9 @@ static int ds4_engine_vision_encode_image( e->vision_model.tensor_data_pos, e->vision_model.size - e->vision_model.tensor_data_pos) != 0; #endif + if (e->vision_map_ready) { + e->vision_map_generation = ds4_gpu_model_views_generation(); + } } if (!e->metal_ready || !e->vision_map_ready) { if (error && error_cap) { diff --git a/ds4_cuda.cu b/ds4_cuda.cu index fd36d7014e..cac8cf3de3 100644 --- a/ds4_cuda.cu +++ b/ds4_cuda.cu @@ -3794,6 +3794,12 @@ extern "C" int ds4_gpu_set_model_map_range(const void *model_map, uint64_t model return 1; } +extern "C" uint64_t ds4_gpu_model_views_generation(void) { + /* The CUDA backend never clears registered model views, so the + * generation is constant. */ + return 0; +} + extern "C" int ds4_gpu_set_aux_model_map_range( const void *model_map, uint64_t model_size, diff --git a/ds4_gpu.h b/ds4_gpu.h index 21d0160191..59c13a4588 100644 --- a/ds4_gpu.h +++ b/ds4_gpu.h @@ -124,6 +124,10 @@ int ds4_gpu_set_aux_model_map_range(const void *model_map, uint64_t map_offset, uint64_t map_size); int ds4_gpu_set_model_map_spans(const void *model_map, uint64_t model_size, const uint64_t *offsets, const uint64_t *sizes, uint32_t count, uint64_t max_tensor_bytes); +/* Current invalidation generation of the mapped model views. Incremented + * whenever registered views are cleared; callers must re-check before + * relying on a previously registered auxiliary/vision view. */ +uint64_t ds4_gpu_model_views_generation(void); int ds4_gpu_cache_model_range(const void *model_map, uint64_t model_size, uint64_t offset, uint64_t bytes, const char *label); int ds4_gpu_cache_q8_f16_range(const void *model_map, uint64_t model_size, uint64_t offset, uint64_t bytes, uint64_t in_dim, uint64_t out_dim, const char *label); int ds4_gpu_q8_cache_suppressed(void); diff --git a/ds4_metal.m b/ds4_metal.m index 3363d7df5e..49691cea44 100644 --- a/ds4_metal.m +++ b/ds4_metal.m @@ -366,6 +366,7 @@ static id g_attn_out_group_ids_buffer; static int g_model_fd = -1; static const void *g_model_map_ptr; +static const void *g_last_aux_map; static uint64_t g_model_map_size; static uint64_t g_model_mapped_offset; static uint64_t g_model_mapped_size; @@ -607,6 +608,10 @@ static void ds4_gpu_print_device_summary(void) { static ds4_gpu_model_view g_model_views[DS4_METAL_MAX_MODEL_VIEWS]; static uint32_t g_model_view_count; +/* Bumped whenever previously registered model views are invalidated, so + * callers holding stale view registrations (e.g. the vision sidecar map) + * can detect that their views no longer exist. */ +static uint64_t g_model_views_generation; enum { DS4_METAL_STREAM_EXPERT_CACHE_MAX_LAYER = 80, @@ -1690,6 +1695,32 @@ static void ds4_gpu_model_views_clear(void) { g_model_views[i].bytes = 0; } g_model_view_count = 0; + g_model_views_generation++; +} + +/* Drop only the views belonging to one GGUF mapping (identified by map + * pointer and size) and keep views of other mappings alive (e.g. the + * vision sidecar map). Bumps the generation when anything was removed. */ +static void ds4_gpu_model_views_clear_for_map(const void *model_map, + uint64_t model_size) { + uint32_t out = 0; + for (uint32_t i = 0; i < g_model_view_count; i++) { + if (g_model_views[i].model_map == model_map && + g_model_views[i].model_size == model_size) { + g_model_views[i].buffer = nil; + g_model_views[i].model_map = NULL; + g_model_views[i].model_size = 0; + g_model_views[i].model_offset = 0; + g_model_views[i].bytes = 0; + continue; + } + g_model_views[out] = g_model_views[i]; + out++; + } + if (out != g_model_view_count) { + g_model_view_count = out; + g_model_views_generation++; + } } static void ds4_gpu_model_residency_clear(void) { @@ -11289,6 +11320,55 @@ int ds4_gpu_embed_tokens_hc_tensor( return 1; } +static int ds4_gpu_model_views_cover_range( + const void *model_map, + uint64_t model_size, + uint64_t offset, + uint64_t size); + +uint64_t ds4_gpu_model_views_generation(void) { + return g_model_views_generation; +} + +int ds4_gpu_set_aux_model_map_range( + const void *model_map, + uint64_t model_size, + uint64_t map_offset, + uint64_t map_size) { + /* Register a secondary GGUF mapping (e.g. the vision encoder sidecar) + * WITHOUT replacing the primary model map globals: only append the + * views for this map. */ + if (!g_initialized && !ds4_gpu_init()) return 0; + if (!model_map || model_size == 0) return 0; + if (map_offset > model_size || map_size == 0 || + map_size > model_size - map_offset) return 0; + + if (ds4_gpu_model_views_cover_range(model_map, + model_size, + map_offset, + map_size)) { + return 1; + } + + @autoreleasepool { + uint64_t mapped_total = 0; + if (!ds4_gpu_add_model_view_range(model_map, + model_size, + map_offset, + map_size, + 0, + false, + &mapped_total)) { + return 0; + } + if (!ds4_gpu_finish_model_views(0.0, mapped_total, map_offset)) { + return 0; + } + g_last_aux_map = model_map; + return 1; + } +} + int ds4_gpu_set_model_map_range(const void *model_map, uint64_t model_size, uint64_t map_offset, uint64_t map_size, uint64_t max_tensor_bytes) { if (!g_initialized && !ds4_gpu_init()) return 0; if (!model_map || model_size == 0) return 0; @@ -11363,16 +11443,18 @@ int ds4_gpu_set_model_map_spans( const double t0 = ds4_gpu_now_ms(); max_tensor_bytes = ds4_gpu_effective_model_max_tensor_bytes(model_size, max_tensor_bytes); + /* Drop only this mapping's views; keep aux/sidecar views (e.g. the + * vision encoder) of other mappings alive. */ + ds4_gpu_model_views_clear_for_map(model_map, model_size); ds4_gpu_model_residency_clear(); - ds4_gpu_model_views_clear(); uint64_t mapped_total = 0; uint64_t first_offset = UINT64_MAX; for (uint32_t i = 0; i < count; i++) { if (offsets[i] > model_size || sizes[i] == 0 || sizes[i] > model_size - offsets[i]) { fprintf(stderr, "ds4: Metal model span %u is outside the GGUF mapping\n", i); + ds4_gpu_model_views_clear_for_map(model_map, model_size); ds4_gpu_model_residency_clear(); - ds4_gpu_model_views_clear(); return 0; } if (offsets[i] < first_offset) first_offset = offsets[i]; @@ -11385,14 +11467,14 @@ int ds4_gpu_set_model_map_spans( effective_max, true, &mapped_total)) { + ds4_gpu_model_views_clear_for_map(model_map, model_size); ds4_gpu_model_residency_clear(); - ds4_gpu_model_views_clear(); return 0; } } if (!ds4_gpu_finish_model_views(t0, mapped_total, first_offset)) { + ds4_gpu_model_views_clear_for_map(model_map, model_size); ds4_gpu_model_residency_clear(); - ds4_gpu_model_views_clear(); return 0; } g_model_map_ptr = model_map; @@ -11492,9 +11574,16 @@ int ds4_gpu_set_model_fd_for_map(int fd, const void *model_map) { } fprintf(stderr, - "ds4: Metal model range %.2f..%.2f GiB is not covered by mapped model views\n", + "ds4: Metal model range %.2f..%.2f GiB is not covered by mapped model views " + "(map=%p primary=%d aux=%d off=0x%llx len=0x%llx views=%u)\n", ds4_gpu_gib(offset), - ds4_gpu_gib(end)); + ds4_gpu_gib(end), + model_map, + model_map == g_model_map_ptr, + model_map == g_last_aux_map, + (unsigned long long)offset, + (unsigned long long)len, + g_model_view_count); return nil; } diff --git a/rocm/ds4_rocm_runtime.cuh b/rocm/ds4_rocm_runtime.cuh index 00d4424877..6c22bf415d 100644 --- a/rocm/ds4_rocm_runtime.cuh +++ b/rocm/ds4_rocm_runtime.cuh @@ -6215,6 +6215,12 @@ extern "C" int ds4_gpu_set_model_map_range(const void *model_map, uint64_t model return 1; } +extern "C" uint64_t ds4_gpu_model_views_generation(void) { + /* The ROCm backend never clears registered model views, so the + * generation is constant. */ + return 0; +} + extern "C" int ds4_gpu_set_aux_model_map_range( const void *model_map, uint64_t model_size,