fix(server): skip memory guard when the model footprint is indeterminate

CMakeLists.txt

@@ -1744,6 +1744,7 @@ add_executable(audiocpp_server
     app/server/base64.cpp
     app/server/config.cpp
     app/server/http.cpp
+    app/server/model_memory.cpp
     app/server/multipart.cpp
     app/server/runtime.cpp
     app/server/ui_assets.cpp
@@ -2667,6 +2668,7 @@ if (ENGINE_BUILD_TESTS)
     add_executable(server_config_test
         tests/unittests/test_server_config.cpp
         app/server/config.cpp
+        app/server/model_memory.cpp
         app/cli/args.cpp
         app/cli/request.cpp
     )

app/server/README.md

@@ -99,7 +99,7 @@ Set top-level `"max_loaded_models"` to bound how many models are resident in mem
 
 Set top-level `"idle_unload_ms"` to have the server unload every resident model after it has gone that long without any model load/run. The next request reloads lazily; a model mid-inference is never unloaded. This complements `max_loaded_models`: that bounds peak residency, this frees memory during quiet periods. Defaults to `0` (disabled). The `--idle-unload-ms <ms>` command-line flag overrides the config value.
 
-Set top-level `"min_free_memory_mb"` to refuse a model load when the host or the GPU backend does not have that much free memory left after the estimated footprint of the new model. The estimate covers only what the loader will actually read: a single-file model's weights, the one GGUF a model directory selects (`model.gguf` or the sole `*.gguf`), or a full safetensors/HF checkpoint tree, plus any session auxiliary files. A directory holding several GGUFs with no `model.gguf` is ambiguous, so the guard makes no estimate there and the loader's own error surfaces instead. The estimate is scaled by a runtime overhead factor plus a fixed floor. When the check fails, the request returns HTTP 503 with `insufficient_memory`; the client may retry later. Defaults to `0`, which disables the guard entirely so existing deployments are unaffected; set a positive value to opt in. The `--min-free-memory-mb <mb>` command-line flag overrides the config value.
+Set top-level `"min_free_memory_mb"` to refuse a model load when the host or the GPU backend does not have that much free memory left after the estimated footprint of the new model. The estimate covers only what the loader will actually read: a single-file model's weights, the one GGUF a model directory selects (`model.gguf` or the sole `*.gguf`), or a full safetensors/HF checkpoint tree, plus any session auxiliary files. A directory holding several GGUFs with no `model.gguf` is ambiguous, so the guard makes no estimate there: the loader's own error surfaces for spec-driven loads, and family-specific layouts the estimator cannot resolve load unguarded (the server logs that the guard skipped the model). The estimate is scaled by a runtime overhead factor plus a fixed floor. When the check fails, the request returns HTTP 503 with `insufficient_memory`; the client may retry later. Defaults to `0`, which disables the guard entirely so existing deployments are unaffected; set a positive value to opt in. The `--min-free-memory-mb <mb>` command-line flag overrides the config value.
 
 Set per-model `"default_request_options"` to apply request-option defaults to every request for that model. Values supplied by the actual request body override these defaults.
 

app/server/main.cpp

@@ -76,11 +76,13 @@ void print_help() {
         << "                                   busy this long; default 300000, 0 disables\n"
         << "  --max-loaded-models <n>          keep at most n models resident in memory, unloading\n"
         << "                                   the least recently used idle model first; 1 enforces\n"
-        << "                                   a single loaded model, default 0 (no limit)\n"        << "  --idle-unload-ms <ms>            unload all resident models after this many ms without\n"
+        << "                                   a single loaded model, default 0 (no limit)\n"
+        << "  --idle-unload-ms <ms>            unload all resident models after this many ms without\n"
         << "                                   any model load/run; default 0 (disabled), next request\n"
-        << "                                   reloads lazily\n"        << "  --min-free-memory-mb <mb>        refuse a model load unless host and GPU each keep at\n"
-        << "                                   least this many MiB free after the load; default 512,\n"
-        << "                                   0 disables the extra headroom\n"
+        << "                                   reloads lazily\n"
+        << "  --min-free-memory-mb <mb>        refuse a model load unless host and GPU each keep at\n"
+        << "                                   least this many MiB free after the load; default 0\n"
+        << "                                   (guard disabled)\n"
         << "  --voice-dir <directory>          override the shared reference voice library directory\n"
         << "  --cors-origins \"*\"              experimental; disabled by default. Allows browser\n"
         << "                                   requests from any origin for trusted local demos only\n"

app/server/model_memory.cpp

@@ -0,0 +1,104 @@
+#include "model_memory.h"
+
+#include "engine/framework/assets/tensor_source.h"
+
+#include <filesystem>
+#include <functional>
+#include <string>
+#include <system_error>
+#include <vector>
+
+namespace minitts::server {
+
+std::optional<size_t> estimate_model_memory_bytes(const ServerModelConfig & model) {
+    size_t weights = 0;
+    std::error_code ec;
+    // Checkpoint trees (safetensors / HF-style directories) are summed recursively
+    // with hard limits so a pathological tree cannot stall the load path or blow the
+    // counter; anything beyond the limits contributes 0, while the fixed floor
+    // below keeps the estimate conservative even for a directory that reads as
+    // empty.
+    constexpr size_t kMaxDepth = 3;
+    constexpr size_t kMaxFiles = 10000;
+    size_t visited_files = 0;
+    const auto add_file = [&](const std::filesystem::path & path) {
+        if (!std::filesystem::is_regular_file(path, ec)) {
+            return;
+        }
+        // A file that disappears or becomes unreadable mid-scan contributes 0
+        // (file_size reports (uintmax_t)-1 with ec set) rather than wrapping the
+        // counter.
+        const auto size = std::filesystem::file_size(path, ec);
+        if (ec) {
+            return;
+        }
+        weights += static_cast<size_t>(size);
+        ++visited_files;
+    };
+    const std::function<void(const std::filesystem::path &, size_t)> add_tree =
+        [&](const std::filesystem::path & path, size_t depth) {
+            if (std::filesystem::is_regular_file(path, ec)) {
+                add_file(path);
+            } else if (std::filesystem::is_directory(path, ec) && depth < kMaxDepth) {
+                std::filesystem::directory_iterator it(path, ec), end;
+                for (; it != end && visited_files < kMaxFiles; it.increment(ec)) {
+                    add_tree(it->path(), depth + 1);
+                }
+            }
+        };
+    // Estimate only what the loader will actually read from model.path, so the
+    // guard neither overestimates nor masks the loader's own error:
+    //  - a single-file model contributes that file;
+    //  - a model directory contributes the one GGUF it selects (model.gguf, or the
+    //    sole *.gguf) -- a package holding several variants is loaded from just one;
+    //  - a directory with no GGUF is a safetensors/HF checkpoint whose whole tree
+    //    loads, so it is summed;
+    //  - a directory with several GGUFs and no model.gguf is ambiguous: the loader
+    //    rejects the spec-driven case with its own "contains N GGUF files" error
+    //    and loads family-specific layouts instead, so the footprint is
+    //    indeterminate and the caller skips the guard rather than answer 503.
+    if (std::filesystem::is_regular_file(model.path, ec)) {
+        add_file(model.path);
+    } else if (std::filesystem::is_directory(model.path, ec)) {
+        // One listing of the directory, mirroring the loader's own selection:
+        // model.gguf wins, the sole *.gguf is used alone, and several GGUFs
+        // without model.gguf are ambiguous.
+        const auto ggufs = engine::assets::directory_gguf_files(model.path);
+        std::optional<std::filesystem::path> selected;
+        for (const auto & gguf : ggufs) {
+            if (gguf.filename() == "model.gguf") {
+                selected = gguf;
+                break;
+            }
+        }
+        if (!selected.has_value() && ggufs.size() == 1) {
+            selected = ggufs.front();
+        }
+        if (selected.has_value()) {
+            add_file(*selected);
+        } else if (!ggufs.empty()) {
+            return std::nullopt;
+        } else {
+            add_tree(model.path, 0);
+        }
+    }
+    // Relative auxiliary paths resolve against the model directory when model.path
+    // is a directory, and against the model file's parent when it is a file.
+    const std::filesystem::path aux_base =
+        std::filesystem::is_directory(model.path, ec) ? model.path : model.path.parent_path();
+    for (const auto & [key, value] : model.session_options) {
+        (void)key;
+        std::filesystem::path aux(value);
+        if (aux.is_relative()) {
+            aux = aux_base / aux;
+        }
+        add_tree(aux, 0);
+    }
+    // Weights plus a runtime factor for Metal/GPU buffers, activation graphs and
+    // KV state, plus a fixed floor for per-model bookkeeping.
+    constexpr double kRuntimeOverheadFactor = 1.5;
+    constexpr size_t kFixedOverhead = 128ull * 1024 * 1024;
+    return static_cast<size_t>(static_cast<double>(weights) * kRuntimeOverheadFactor) + kFixedOverhead;
+}
+
+}  // namespace minitts::server

app/server/model_memory.h

@@ -0,0 +1,20 @@
+#pragma once
+
+#include "config.h"
+
+#include <cstddef>
+#include <optional>
+
+namespace minitts::server {
+
+// Estimated resident bytes a model will occupy once loaded (weights plus a
+// runtime overhead factor for GPU buffers / compute graphs). Returns nullopt
+// when the footprint is indeterminate: a model directory holding several GGUFs
+// and no model.gguf. The guard must not refuse such a load with a 503 -- the
+// loader either rejects the ambiguous directory itself (the spec-driven path
+// fails with its "contains N GGUF files" error) or accepts it under a
+// family-specific layout the estimator cannot resolve, so the guard skips
+// rather than guess.
+std::optional<size_t> estimate_model_memory_bytes(const ServerModelConfig & model);
+
+}  // namespace minitts::server

app/server/runtime.cpp

@@ -1,6 +1,7 @@
 #include "runtime.h"
 
 #include "base64.h"
+#include "model_memory.h"
 #include "multipart.h"
 #include "ui_assets.h"
 
@@ -9,7 +10,6 @@
 #include "../streaming/streaming.h"
 
 #include "engine/framework/core/host_memory.h"
-#include "engine/framework/assets/tensor_source.h"
 #include "engine/framework/debug/trace.h"
 #include "engine/framework/io/json.h"
 #include "engine/framework/model_spec/metadata.h"
@@ -2840,85 +2840,29 @@ std::string format_bytes(size_t bytes) {
 }
 }  // namespace
 
-size_t ServerState::estimate_model_memory_bytes(const ServerModelConfig & model) const {
-    size_t weights = 0;
-    std::error_code ec;
-    // Checkpoint trees (safetensors / HF-style directories) are summed recursively
-    // with hard limits so a pathological tree cannot stall the load path or blow the
-    // counter; anything beyond the limits contributes 0 (the fixed floor below still
-    // applies, so the estimate never reads as completely empty).
-    constexpr size_t kMaxDepth = 3;
-    constexpr size_t kMaxFiles = 10000;
-    size_t visited_files = 0;
-    const auto add_file = [&](const std::filesystem::path & path) {
-        if (std::filesystem::is_regular_file(path, ec)) {
-            weights += static_cast<size_t>(std::filesystem::file_size(path, ec));
-            ++visited_files;
-        }
-    };
-    const std::function<void(const std::filesystem::path &, size_t)> add_tree =
-        [&](const std::filesystem::path & path, size_t depth) {
-            if (std::filesystem::is_regular_file(path, ec)) {
-                add_file(path);
-            } else if (std::filesystem::is_directory(path, ec) && depth < kMaxDepth) {
-                std::filesystem::directory_iterator it(path, ec), end;
-                for (; it != end && visited_files < kMaxFiles; it.increment(ec)) {
-                    add_tree(it->path(), depth + 1);
-                }
-            }
-        };
-    // Estimate only what the loader will actually read from model.path, so the
-    // guard neither overestimates nor masks the loader's own error:
-    //  - a single-file model contributes that file;
-    //  - a model directory contributes the one GGUF it selects (model.gguf, or the
-    //    sole *.gguf) -- a package holding several variants is loaded from just one;
-    //  - a directory with no GGUF is a safetensors/HF checkpoint whose whole tree
-    //    loads, so it is summed;
-    //  - a directory with several GGUFs and no model.gguf is ambiguous: the loader
-    //    rejects it with "contains N GGUF files", so we estimate nothing rather than
-    //    answer 503 and hide that real error.
-    if (std::filesystem::is_regular_file(model.path, ec)) {
-        add_file(model.path);
-    } else if (std::filesystem::is_directory(model.path, ec)) {
-        if (const auto selected = engine::assets::find_directory_gguf(model.path)) {
-            add_file(*selected);
-        } else if (engine::assets::directory_gguf_files(model.path).empty()) {
-            add_tree(model.path, 0);
-        }
-    }
-    // Relative auxiliary paths resolve against the model directory when model.path
-    // is a directory, and against the model file's parent when it is a file.
-    const std::filesystem::path aux_base =
-        std::filesystem::is_directory(model.path, ec) ? model.path : model.path.parent_path();
-    for (const auto & [key, value] : model.session_options) {
-        (void)key;
-        std::filesystem::path aux(value);
-        if (aux.is_relative()) {
-            aux = aux_base / aux;
-        }
-        add_tree(aux, 0);
-    }
-    // Weights plus a runtime factor for Metal/GPU buffers, activation graphs and
-    // KV state, plus a fixed floor for per-model bookkeeping.
-    constexpr double kRuntimeOverheadFactor = 1.5;
-    constexpr size_t kFixedOverhead = 128ull * 1024 * 1024;
-    return static_cast<size_t>(static_cast<double>(weights) * kRuntimeOverheadFactor) + kFixedOverhead;
-}
-
 void ServerState::ensure_model_fits_memory(const ServerModelConfig & model) {
     // The guard is opt-in: 0 disables it entirely so existing deployments see no
     // behavior change. This also keeps lazy loads unserialized (see the call site)
     // when neither this guard nor max_loaded_models is active.
     if (config_.min_free_memory_mb <= 0) {
         return;
     }
-    const size_t estimate = estimate_model_memory_bytes(model);
+    const auto estimate = estimate_model_memory_bytes(model);
+    if (!estimate.has_value()) {
+        // Indeterminate footprint (an ambiguous multi-GGUF model directory): the
+        // loader rejects the spec-driven case with its own error and loads
+        // family-specific layouts, so the guard has no basis to refuse and must
+        // not mask the real outcome with a 503. Say so instead of skipping silently.
+        std::cerr << "[server] memory guard skipped for model '" << model.id
+                  << "': indeterminate footprint (ambiguous model directory)\n";
+        return;
+    }
     const size_t headroom = static_cast<size_t>(config_.min_free_memory_mb) * 1024ull * 1024ull;
 
     const size_t host_available = engine::core::available_host_memory_bytes();
-    if (host_available > 0 && estimate + headroom > host_available) {
+    if (host_available > 0 && *estimate + headroom > host_available) {
         throw InsufficientMemoryError(
-            "cannot load model '" + model.id + "': estimated " + format_bytes(estimate) +
+            "cannot load model '" + model.id + "': estimated " + format_bytes(*estimate) +
             " + " + std::to_string(config_.min_free_memory_mb) + " MiB headroom exceeds available host memory (" +
             format_bytes(host_available) + ")");
     }
@@ -2931,9 +2875,9 @@ void ServerState::ensure_model_fits_memory(const ServerModelConfig & model) {
     const engine::core::BackendMemorySnapshot device =
         engine::core::query_backend_memory(engine::core::BackendConfig{
             config_.backend, config_.device, config_.threads});
-    if (device.available && estimate + headroom > static_cast<size_t>(device.free_bytes)) {
+    if (device.available && *estimate + headroom > static_cast<size_t>(device.free_bytes)) {
         throw InsufficientMemoryError(
-            "cannot load model '" + model.id + "': estimated " + format_bytes(estimate) +
+            "cannot load model '" + model.id + "': estimated " + format_bytes(*estimate) +
             " + " + std::to_string(config_.min_free_memory_mb) + " MiB headroom exceeds available " +
             backend_name(config_.backend) + " memory (" +
             format_bytes(static_cast<size_t>(device.free_bytes)) + ")");

app/server/runtime.h

@@ -19,6 +19,7 @@
 #include <functional>
 #include <memory>
 #include <mutex>
+#include <optional>
 #include <shared_mutex>
 #include <thread>
 #include <unordered_map>
@@ -108,9 +109,6 @@ class ServerState final : public IHttpHandler {
     // `loading` fits within the limit. A model mid-inference is never a victim;
     // when nothing can be evicted this throws ServerBusyError (-> HTTP 503).
     void evict_for_model_limit(const LoadedModel & loading);
-    // Estimated resident bytes this model will occupy once loaded (weights plus
-    // a runtime overhead factor for GPU buffers / compute graphs).
-    size_t estimate_model_memory_bytes(const ServerModelConfig & model) const;
     // Refuse the load with InsufficientMemoryError (-> HTTP 503) when the
     // estimated footprint plus configured headroom does not fit the free host
     // memory and (for GPU backends) the backend device memory.