diff --git a/CMakeLists.txt b/CMakeLists.txt index 7b847ce0..0422ca3c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1744,6 +1744,7 @@ add_executable(audiocpp_server app/server/base64.cpp app/server/config.cpp app/server/http.cpp + app/server/model_memory.cpp app/server/multipart.cpp app/server/runtime.cpp app/server/ui_assets.cpp @@ -2667,6 +2668,7 @@ if (ENGINE_BUILD_TESTS) add_executable(server_config_test tests/unittests/test_server_config.cpp app/server/config.cpp + app/server/model_memory.cpp app/cli/args.cpp app/cli/request.cpp ) diff --git a/app/server/README.md b/app/server/README.md index 30bcd131..5e6acd0e 100644 --- a/app/server/README.md +++ b/app/server/README.md @@ -99,7 +99,7 @@ Set top-level `"max_loaded_models"` to bound how many models are resident in mem Set top-level `"idle_unload_ms"` to have the server unload every resident model after it has gone that long without any model load/run. The next request reloads lazily; a model mid-inference is never unloaded. This complements `max_loaded_models`: that bounds peak residency, this frees memory during quiet periods. Defaults to `0` (disabled). The `--idle-unload-ms ` command-line flag overrides the config value. -Set top-level `"min_free_memory_mb"` to refuse a model load when the host or the GPU backend does not have that much free memory left after the estimated footprint of the new model. The estimate covers only what the loader will actually read: a single-file model's weights, the one GGUF a model directory selects (`model.gguf` or the sole `*.gguf`), or a full safetensors/HF checkpoint tree, plus any session auxiliary files. A directory holding several GGUFs with no `model.gguf` is ambiguous, so the guard makes no estimate there and the loader's own error surfaces instead. The estimate is scaled by a runtime overhead factor plus a fixed floor. When the check fails, the request returns HTTP 503 with `insufficient_memory`; the client may retry later. Defaults to `0`, which disables the guard entirely so existing deployments are unaffected; set a positive value to opt in. The `--min-free-memory-mb ` command-line flag overrides the config value. +Set top-level `"min_free_memory_mb"` to refuse a model load when the host or the GPU backend does not have that much free memory left after the estimated footprint of the new model. The estimate covers only what the loader will actually read: a single-file model's weights, the one GGUF a model directory selects (`model.gguf` or the sole `*.gguf`), or a full safetensors/HF checkpoint tree, plus any session auxiliary files. A directory holding several GGUFs with no `model.gguf` is ambiguous, so the guard makes no estimate there: the loader's own error surfaces for spec-driven loads, and family-specific layouts the estimator cannot resolve load unguarded (the server logs that the guard skipped the model). The estimate is scaled by a runtime overhead factor plus a fixed floor. When the check fails, the request returns HTTP 503 with `insufficient_memory`; the client may retry later. Defaults to `0`, which disables the guard entirely so existing deployments are unaffected; set a positive value to opt in. The `--min-free-memory-mb ` command-line flag overrides the config value. Set per-model `"default_request_options"` to apply request-option defaults to every request for that model. Values supplied by the actual request body override these defaults. diff --git a/app/server/main.cpp b/app/server/main.cpp index 89002559..b1d5e49a 100644 --- a/app/server/main.cpp +++ b/app/server/main.cpp @@ -76,11 +76,13 @@ void print_help() { << " busy this long; default 300000, 0 disables\n" << " --max-loaded-models keep at most n models resident in memory, unloading\n" << " the least recently used idle model first; 1 enforces\n" - << " a single loaded model, default 0 (no limit)\n" << " --idle-unload-ms unload all resident models after this many ms without\n" + << " a single loaded model, default 0 (no limit)\n" + << " --idle-unload-ms unload all resident models after this many ms without\n" << " any model load/run; default 0 (disabled), next request\n" - << " reloads lazily\n" << " --min-free-memory-mb refuse a model load unless host and GPU each keep at\n" - << " least this many MiB free after the load; default 512,\n" - << " 0 disables the extra headroom\n" + << " reloads lazily\n" + << " --min-free-memory-mb refuse a model load unless host and GPU each keep at\n" + << " least this many MiB free after the load; default 0\n" + << " (guard disabled)\n" << " --voice-dir override the shared reference voice library directory\n" << " --cors-origins \"*\" experimental; disabled by default. Allows browser\n" << " requests from any origin for trusted local demos only\n" diff --git a/app/server/model_memory.cpp b/app/server/model_memory.cpp new file mode 100644 index 00000000..252ab990 --- /dev/null +++ b/app/server/model_memory.cpp @@ -0,0 +1,104 @@ +#include "model_memory.h" + +#include "engine/framework/assets/tensor_source.h" + +#include +#include +#include +#include +#include + +namespace minitts::server { + +std::optional estimate_model_memory_bytes(const ServerModelConfig & model) { + size_t weights = 0; + std::error_code ec; + // Checkpoint trees (safetensors / HF-style directories) are summed recursively + // with hard limits so a pathological tree cannot stall the load path or blow the + // counter; anything beyond the limits contributes 0, while the fixed floor + // below keeps the estimate conservative even for a directory that reads as + // empty. + constexpr size_t kMaxDepth = 3; + constexpr size_t kMaxFiles = 10000; + size_t visited_files = 0; + const auto add_file = [&](const std::filesystem::path & path) { + if (!std::filesystem::is_regular_file(path, ec)) { + return; + } + // A file that disappears or becomes unreadable mid-scan contributes 0 + // (file_size reports (uintmax_t)-1 with ec set) rather than wrapping the + // counter. + const auto size = std::filesystem::file_size(path, ec); + if (ec) { + return; + } + weights += static_cast(size); + ++visited_files; + }; + const std::function add_tree = + [&](const std::filesystem::path & path, size_t depth) { + if (std::filesystem::is_regular_file(path, ec)) { + add_file(path); + } else if (std::filesystem::is_directory(path, ec) && depth < kMaxDepth) { + std::filesystem::directory_iterator it(path, ec), end; + for (; it != end && visited_files < kMaxFiles; it.increment(ec)) { + add_tree(it->path(), depth + 1); + } + } + }; + // Estimate only what the loader will actually read from model.path, so the + // guard neither overestimates nor masks the loader's own error: + // - a single-file model contributes that file; + // - a model directory contributes the one GGUF it selects (model.gguf, or the + // sole *.gguf) -- a package holding several variants is loaded from just one; + // - a directory with no GGUF is a safetensors/HF checkpoint whose whole tree + // loads, so it is summed; + // - a directory with several GGUFs and no model.gguf is ambiguous: the loader + // rejects the spec-driven case with its own "contains N GGUF files" error + // and loads family-specific layouts instead, so the footprint is + // indeterminate and the caller skips the guard rather than answer 503. + if (std::filesystem::is_regular_file(model.path, ec)) { + add_file(model.path); + } else if (std::filesystem::is_directory(model.path, ec)) { + // One listing of the directory, mirroring the loader's own selection: + // model.gguf wins, the sole *.gguf is used alone, and several GGUFs + // without model.gguf are ambiguous. + const auto ggufs = engine::assets::directory_gguf_files(model.path); + std::optional selected; + for (const auto & gguf : ggufs) { + if (gguf.filename() == "model.gguf") { + selected = gguf; + break; + } + } + if (!selected.has_value() && ggufs.size() == 1) { + selected = ggufs.front(); + } + if (selected.has_value()) { + add_file(*selected); + } else if (!ggufs.empty()) { + return std::nullopt; + } else { + add_tree(model.path, 0); + } + } + // Relative auxiliary paths resolve against the model directory when model.path + // is a directory, and against the model file's parent when it is a file. + const std::filesystem::path aux_base = + std::filesystem::is_directory(model.path, ec) ? model.path : model.path.parent_path(); + for (const auto & [key, value] : model.session_options) { + (void)key; + std::filesystem::path aux(value); + if (aux.is_relative()) { + aux = aux_base / aux; + } + add_tree(aux, 0); + } + // Weights plus a runtime factor for Metal/GPU buffers, activation graphs and + // KV state, plus a fixed floor for per-model bookkeeping. + constexpr double kRuntimeOverheadFactor = 1.5; + constexpr size_t kFixedOverhead = 128ull * 1024 * 1024; + return static_cast(static_cast(weights) * kRuntimeOverheadFactor) + kFixedOverhead; +} + +} // namespace minitts::server diff --git a/app/server/model_memory.h b/app/server/model_memory.h new file mode 100644 index 00000000..8d0e7792 --- /dev/null +++ b/app/server/model_memory.h @@ -0,0 +1,20 @@ +#pragma once + +#include "config.h" + +#include +#include + +namespace minitts::server { + +// Estimated resident bytes a model will occupy once loaded (weights plus a +// runtime overhead factor for GPU buffers / compute graphs). Returns nullopt +// when the footprint is indeterminate: a model directory holding several GGUFs +// and no model.gguf. The guard must not refuse such a load with a 503 -- the +// loader either rejects the ambiguous directory itself (the spec-driven path +// fails with its "contains N GGUF files" error) or accepts it under a +// family-specific layout the estimator cannot resolve, so the guard skips +// rather than guess. +std::optional estimate_model_memory_bytes(const ServerModelConfig & model); + +} // namespace minitts::server diff --git a/app/server/runtime.cpp b/app/server/runtime.cpp index 141b3506..0ce8e0e0 100644 --- a/app/server/runtime.cpp +++ b/app/server/runtime.cpp @@ -1,6 +1,7 @@ #include "runtime.h" #include "base64.h" +#include "model_memory.h" #include "multipart.h" #include "ui_assets.h" @@ -9,7 +10,6 @@ #include "../streaming/streaming.h" #include "engine/framework/core/host_memory.h" -#include "engine/framework/assets/tensor_source.h" #include "engine/framework/debug/trace.h" #include "engine/framework/io/json.h" #include "engine/framework/model_spec/metadata.h" @@ -2840,71 +2840,6 @@ std::string format_bytes(size_t bytes) { } } // namespace -size_t ServerState::estimate_model_memory_bytes(const ServerModelConfig & model) const { - size_t weights = 0; - std::error_code ec; - // Checkpoint trees (safetensors / HF-style directories) are summed recursively - // with hard limits so a pathological tree cannot stall the load path or blow the - // counter; anything beyond the limits contributes 0 (the fixed floor below still - // applies, so the estimate never reads as completely empty). - constexpr size_t kMaxDepth = 3; - constexpr size_t kMaxFiles = 10000; - size_t visited_files = 0; - const auto add_file = [&](const std::filesystem::path & path) { - if (std::filesystem::is_regular_file(path, ec)) { - weights += static_cast(std::filesystem::file_size(path, ec)); - ++visited_files; - } - }; - const std::function add_tree = - [&](const std::filesystem::path & path, size_t depth) { - if (std::filesystem::is_regular_file(path, ec)) { - add_file(path); - } else if (std::filesystem::is_directory(path, ec) && depth < kMaxDepth) { - std::filesystem::directory_iterator it(path, ec), end; - for (; it != end && visited_files < kMaxFiles; it.increment(ec)) { - add_tree(it->path(), depth + 1); - } - } - }; - // Estimate only what the loader will actually read from model.path, so the - // guard neither overestimates nor masks the loader's own error: - // - a single-file model contributes that file; - // - a model directory contributes the one GGUF it selects (model.gguf, or the - // sole *.gguf) -- a package holding several variants is loaded from just one; - // - a directory with no GGUF is a safetensors/HF checkpoint whose whole tree - // loads, so it is summed; - // - a directory with several GGUFs and no model.gguf is ambiguous: the loader - // rejects it with "contains N GGUF files", so we estimate nothing rather than - // answer 503 and hide that real error. - if (std::filesystem::is_regular_file(model.path, ec)) { - add_file(model.path); - } else if (std::filesystem::is_directory(model.path, ec)) { - if (const auto selected = engine::assets::find_directory_gguf(model.path)) { - add_file(*selected); - } else if (engine::assets::directory_gguf_files(model.path).empty()) { - add_tree(model.path, 0); - } - } - // Relative auxiliary paths resolve against the model directory when model.path - // is a directory, and against the model file's parent when it is a file. - const std::filesystem::path aux_base = - std::filesystem::is_directory(model.path, ec) ? model.path : model.path.parent_path(); - for (const auto & [key, value] : model.session_options) { - (void)key; - std::filesystem::path aux(value); - if (aux.is_relative()) { - aux = aux_base / aux; - } - add_tree(aux, 0); - } - // Weights plus a runtime factor for Metal/GPU buffers, activation graphs and - // KV state, plus a fixed floor for per-model bookkeeping. - constexpr double kRuntimeOverheadFactor = 1.5; - constexpr size_t kFixedOverhead = 128ull * 1024 * 1024; - return static_cast(static_cast(weights) * kRuntimeOverheadFactor) + kFixedOverhead; -} - void ServerState::ensure_model_fits_memory(const ServerModelConfig & model) { // The guard is opt-in: 0 disables it entirely so existing deployments see no // behavior change. This also keeps lazy loads unserialized (see the call site) @@ -2912,13 +2847,22 @@ void ServerState::ensure_model_fits_memory(const ServerModelConfig & model) { if (config_.min_free_memory_mb <= 0) { return; } - const size_t estimate = estimate_model_memory_bytes(model); + const auto estimate = estimate_model_memory_bytes(model); + if (!estimate.has_value()) { + // Indeterminate footprint (an ambiguous multi-GGUF model directory): the + // loader rejects the spec-driven case with its own error and loads + // family-specific layouts, so the guard has no basis to refuse and must + // not mask the real outcome with a 503. Say so instead of skipping silently. + std::cerr << "[server] memory guard skipped for model '" << model.id + << "': indeterminate footprint (ambiguous model directory)\n"; + return; + } const size_t headroom = static_cast(config_.min_free_memory_mb) * 1024ull * 1024ull; const size_t host_available = engine::core::available_host_memory_bytes(); - if (host_available > 0 && estimate + headroom > host_available) { + if (host_available > 0 && *estimate + headroom > host_available) { throw InsufficientMemoryError( - "cannot load model '" + model.id + "': estimated " + format_bytes(estimate) + + "cannot load model '" + model.id + "': estimated " + format_bytes(*estimate) + " + " + std::to_string(config_.min_free_memory_mb) + " MiB headroom exceeds available host memory (" + format_bytes(host_available) + ")"); } @@ -2931,9 +2875,9 @@ void ServerState::ensure_model_fits_memory(const ServerModelConfig & model) { const engine::core::BackendMemorySnapshot device = engine::core::query_backend_memory(engine::core::BackendConfig{ config_.backend, config_.device, config_.threads}); - if (device.available && estimate + headroom > static_cast(device.free_bytes)) { + if (device.available && *estimate + headroom > static_cast(device.free_bytes)) { throw InsufficientMemoryError( - "cannot load model '" + model.id + "': estimated " + format_bytes(estimate) + + "cannot load model '" + model.id + "': estimated " + format_bytes(*estimate) + " + " + std::to_string(config_.min_free_memory_mb) + " MiB headroom exceeds available " + backend_name(config_.backend) + " memory (" + format_bytes(static_cast(device.free_bytes)) + ")"); diff --git a/app/server/runtime.h b/app/server/runtime.h index bb00c2b7..d6899c98 100644 --- a/app/server/runtime.h +++ b/app/server/runtime.h @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -108,9 +109,6 @@ class ServerState final : public IHttpHandler { // `loading` fits within the limit. A model mid-inference is never a victim; // when nothing can be evicted this throws ServerBusyError (-> HTTP 503). void evict_for_model_limit(const LoadedModel & loading); - // Estimated resident bytes this model will occupy once loaded (weights plus - // a runtime overhead factor for GPU buffers / compute graphs). - size_t estimate_model_memory_bytes(const ServerModelConfig & model) const; // Refuse the load with InsufficientMemoryError (-> HTTP 503) when the // estimated footprint plus configured headroom does not fit the free host // memory and (for GPU backends) the backend device memory. diff --git a/tests/unittests/test_server_config.cpp b/tests/unittests/test_server_config.cpp index 6352041a..254f3a93 100644 --- a/tests/unittests/test_server_config.cpp +++ b/tests/unittests/test_server_config.cpp @@ -1,5 +1,6 @@ #include "busy_guard.h" #include "config.h" +#include "model_memory.h" #include "engine/framework/io/json.h" @@ -503,6 +504,107 @@ void test_model_run_overrun_predicate() { require(!model_run_has_overrun(1000, 10'000'000, -1), "a non-positive timeout disables the guard"); } +// Mirror of the estimator's formula: weights * 1.5 plus the fixed floor, so the +// tests assert absolute values rather than "greater than". +size_t expected_estimate(size_t weights) { + constexpr double kRuntimeOverheadFactor = 1.5; + constexpr size_t kFixedOverhead = 128ull * 1024 * 1024; + return static_cast(static_cast(weights) * kRuntimeOverheadFactor) + kFixedOverhead; +} + +void write_file(const std::filesystem::path & path, size_t bytes) { + std::ofstream out(path, std::ios::binary | std::ios::trunc); + out << std::string(bytes, 'x'); + if (!out) { + throw std::runtime_error("failed to write test file: " + path.string()); + } +} + +void test_model_memory_estimator() { + using minitts::server::estimate_model_memory_bytes; + using minitts::server::ServerModelConfig; + const auto root = make_temp_root(); + + // A single-file model contributes that file. + { + const auto path = root / "single.gguf"; + write_file(path, 1000); + ServerModelConfig model; + model.path = path; + const auto estimate = estimate_model_memory_bytes(model); + require(estimate.has_value(), "single-file model has a determinate footprint"); + require(*estimate == expected_estimate(1000), "single-file estimate sums the file"); + } + + // A directory with exactly one GGUF contributes that file. + { + const auto dir = root / "sole"; + std::filesystem::create_directories(dir); + write_file(dir / "variant.gguf", 2000); + ServerModelConfig model; + model.path = dir; + const auto estimate = estimate_model_memory_bytes(model); + require(estimate.has_value(), "sole-GGUF directory has a determinate footprint"); + require(*estimate == expected_estimate(2000), "the sole GGUF is selected"); + } + + // model.gguf disambiguates a multi-GGUF directory, ignoring other variants. + { + const auto dir = root / "named"; + std::filesystem::create_directories(dir); + write_file(dir / "a.gguf", 3000); + write_file(dir / "model.gguf", 4000); + ServerModelConfig model; + model.path = dir; + const auto estimate = estimate_model_memory_bytes(model); + require(estimate.has_value(), "model.gguf makes the directory determinate"); + require(*estimate == expected_estimate(4000), "model.gguf wins over the other variants"); + } + + // Several GGUFs and no model.gguf: the loader rejects the spec-driven case + // with its own error, so the footprint is indeterminate and the guard skips. + { + const auto dir = root / "ambiguous"; + std::filesystem::create_directories(dir); + write_file(dir / "a.gguf", 100); + write_file(dir / "b.gguf", 100); + ServerModelConfig model; + model.path = dir; + require( + !estimate_model_memory_bytes(model).has_value(), + "an ambiguous multi-GGUF directory has an indeterminate footprint"); + } + + // A directory with no GGUF is a safetensors/HF checkpoint: the tree is summed. + { + const auto dir = root / "tree"; + std::filesystem::create_directories(dir / "sub"); + write_file(dir / "model.safetensors", 5000); + write_file(dir / "sub" / "chunk.bin", 6000); + ServerModelConfig model; + model.path = dir; + const auto estimate = estimate_model_memory_bytes(model); + require(estimate.has_value(), "a no-GGUF directory has a determinate footprint"); + require(*estimate == expected_estimate(11000), "a checkpoint tree is summed recursively"); + } + + // Relative auxiliary session files resolve against the model directory. + { + // Not named "aux": that is a reserved DOS device name and cannot be + // created on Windows. + const auto dir = root / "sidecar"; + std::filesystem::create_directories(dir); + write_file(dir / "model.gguf", 7000); + write_file(dir / "head.bin", 8000); + ServerModelConfig model; + model.path = dir; + model.session_options["aux_path"] = "head.bin"; + const auto estimate = estimate_model_memory_bytes(model); + require(estimate.has_value(), "an aux-resolved directory has a determinate footprint"); + require(*estimate == expected_estimate(15000), "a relative aux path resolves against the model directory"); + } +} + } // namespace int main() { @@ -529,6 +631,7 @@ int main() { test_empty_models_require_ui_management(); test_request_timeout_is_clamped_to_policy(); test_model_run_overrun_predicate(); + test_model_memory_estimator(); } catch (const std::exception & error) { std::cerr << error.what() << '\n'; return 1;