From 1f70f9ae4af16612344901de84a5b454b5bcf9ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E5=BA=86=E4=B8=B0?= Date: Tue, 25 Aug 2026 14:43:53 +0800 Subject: [PATCH 1/6] feat(server): add --idle-unload-ms to auto-unload idle models The 5-minute idle unload used to live in an external Python monitor inside the audio-server wrapper, which polled the server log mtime and called /v1/tasks/unload_all_models. Move it into audiocpp_server itself: - new ServerConfig field idle_unload_ms (default 0 = disabled), parsed from server.json and overridable via --idle-unload-ms - a background thread unloads every resident non-busy model once the server has been idle that long without a model load/run; the next request reloads lazily This drops the log-mtime heuristic (which required --log trace spam) and removes the need for the external Python wrapper. --- app/server/config.cpp | 4 ++++ app/server/config.h | 5 ++++ app/server/main.cpp | 12 ++++++++-- app/server/runtime.cpp | 52 ++++++++++++++++++++++++++++++++++++++++++ app/server/runtime.h | 9 ++++++++ 5 files changed, 80 insertions(+), 2 deletions(-) diff --git a/app/server/config.cpp b/app/server/config.cpp index af219842a..ba52fcd62 100644 --- a/app/server/config.cpp +++ b/app/server/config.cpp @@ -235,6 +235,7 @@ ServerConfig load_server_config(const std::filesystem::path & path) { } config.busy_timeout_ms = engine::io::json::optional_i32(root, "busy_timeout_ms", config.busy_timeout_ms); config.max_loaded_models = engine::io::json::optional_i32(root, "max_loaded_models", config.max_loaded_models); + config.idle_unload_ms = engine::io::json::optional_i32(root, "idle_unload_ms", config.idle_unload_ms); if (const auto * value = root.find("live_ingest")) { config.live_ingest = parse_live_ingest_limits(*value, config.live_ingest, "server live_ingest"); } @@ -256,6 +257,9 @@ ServerConfig load_server_config(const std::filesystem::path & path) { if (config.max_loaded_models < 0) { throw std::runtime_error("server max_loaded_models must be >= 0 (0 disables the limit)"); } + if (config.idle_unload_ms < 0) { + throw std::runtime_error("server idle_unload_ms must be >= 0 (0 disables idle unload)"); + } if (config.threads <= 0) { throw std::runtime_error("server threads must be positive"); } diff --git a/app/server/config.h b/app/server/config.h index 3aee4ead4..ca0a51541 100644 --- a/app/server/config.h +++ b/app/server/config.h @@ -90,6 +90,11 @@ struct ServerConfig { // limit and keeps the original behavior: once loaded, a model stays in memory // until it is unloaded explicitly or the server exits. int max_loaded_models = 0; + // Unload every resident model once the server has been idle this long without + // any model load/run (steady-clock ms). 0 disables idle unload. Complements + // max_loaded_models: that bounds peak residency, this frees memory during + // quiet periods. The next request reloads lazily. + int idle_unload_ms = 0; // Fleet-wide bounds for incrementally delivered request bodies. The defaults are // in LiveIngestLimits; a model entry may override any subset of them. LiveIngestLimits live_ingest; diff --git a/app/server/main.cpp b/app/server/main.cpp index d3e6fdd9f..11335111d 100644 --- a/app/server/main.cpp +++ b/app/server/main.cpp @@ -62,7 +62,7 @@ void print_help() { std::cout << "audiocpp_server [--config ] [--ui] [--host ] [--port ] [--backend ]\n" << " [--device ] [--list-devices] [--threads ] [--busy-timeout-ms ]\n" - << " [--max-loaded-models ]\n" + << " [--max-loaded-models ] [--idle-unload-ms ]\n" << " [--model-spec-override ] [--voice-dir ]\n" << " [--log] [--log-file ]\n" << " [--cors-origins ]\n" @@ -76,7 +76,9 @@ void print_help() { << " busy this long; default 300000, 0 disables\n" << " --max-loaded-models keep at most n models resident in memory, unloading\n" << " the least recently used idle model first; 1 enforces\n" - << " a single loaded model, default 0 (no limit)\n" + << " a single loaded model, default 0 (no limit)\n" << " --idle-unload-ms unload all resident models after this many ms without\n" + << " any model load/run; default 0 (disabled), next request\n" + << " reloads lazily\n" << " --voice-dir override the shared reference voice library directory\n" << " --cors-origins \"*\" experimental; disabled by default. Allows browser\n" << " requests from any origin for trusted local demos only\n" @@ -188,6 +190,9 @@ int main(int argc, char ** argv) { if (const auto max_loaded_models = arg_value(argc, argv, "--max-loaded-models")) { config.max_loaded_models = std::stoi(*max_loaded_models); } + if (const auto idle_unload_ms = arg_value(argc, argv, "--idle-unload-ms")) { + config.idle_unload_ms = std::stoi(*idle_unload_ms); + } if (const auto model_spec = arg_value(argc, argv, "--model-spec-override")) { config.model_spec_override = std::filesystem::path(*model_spec); } @@ -206,6 +211,9 @@ int main(int argc, char ** argv) { if (config.max_loaded_models < 0) { throw std::runtime_error("--max-loaded-models must be >= 0 (0 disables the limit)"); } + if (config.idle_unload_ms < 0) { + throw std::runtime_error("--idle-unload-ms must be >= 0 (0 disables idle unload)"); + } const auto ui_resource_anchor = executable_directory(argc > 0 ? argv[0] : nullptr); minitts::server::ServerState state( diff --git a/app/server/runtime.cpp b/app/server/runtime.cpp index 9ec08011a..2e895140d 100644 --- a/app/server/runtime.cpp +++ b/app/server/runtime.cpp @@ -29,6 +29,7 @@ #include #include #include +#include #include #include @@ -968,9 +969,16 @@ ServerState::ServerState( } #endif load_models(); + if (config_.idle_unload_ms > 0) { + idle_unload_thread_ = std::thread(&ServerState::idle_unload_loop, this); + } } ServerState::~ServerState() { + idle_unload_shutdown_.store(true, std::memory_order_relaxed); + if (idle_unload_thread_.joinable()) { + idle_unload_thread_.join(); + } if (!upload_root_.empty()) { std::error_code ec; std::filesystem::remove_all(upload_root_, ec); @@ -1626,6 +1634,7 @@ void ServerState::evict_for_model_limit(const LoadedModel & loading) { } void ServerState::ensure_model_loaded_locked(LoadedModel & model) { + last_activity_ms_.store(steady_now_ms(), std::memory_order_relaxed); model.last_used_ms.store(steady_now_ms(), std::memory_order_relaxed); if (model.session != nullptr) { return; @@ -2747,6 +2756,49 @@ void ServerState::LoadedModel::unload() { loaded.store(false); } +void ServerState::idle_unload_loop() { + const auto interval_ms = std::max(1000, config_.idle_unload_ms / 10); + while (!idle_unload_shutdown_.load(std::memory_order_relaxed)) { + std::this_thread::sleep_for(std::chrono::milliseconds(interval_ms)); + if (idle_unload_shutdown_.load(std::memory_order_relaxed)) { + break; + } + const auto idle_ms = steady_now_ms() - last_activity_ms_.load(std::memory_order_relaxed); + if (idle_ms >= config_.idle_unload_ms) { + unload_idle_models(); + } + } +} + +void ServerState::unload_idle_models() { + std::vector resident; + { + std::lock_guard state_lock(models_mutex_); + for (const auto & model : models_) { + if (model->session != nullptr) { + resident.push_back(model.get()); + } + } + } + int unloaded = 0; + for (LoadedModel * model : resident) { + // Never unload a model mid-inference; a busy model keeps its slot and the + // next idle pass retries it. + const auto lock = model->busy.try_acquire(); + if (!lock.has_value()) { + continue; + } + model->unload(); + ++unloaded; + } + if (unloaded > 0) { + const auto idle_ms = steady_now_ms() - last_activity_ms_.load(std::memory_order_relaxed); + std::cerr << "[server] idle " << idle_ms << " ms: unloaded " << unloaded << " model(s)\n"; + // Restart the idle clock so we do not spin on unload attempts every interval. + last_activity_ms_.store(steady_now_ms(), std::memory_order_relaxed); + } +} + HttpResponse ServerState::handle_unload_models(const std::string & body_text) { const auto body = engine::io::json::parse(body_text); const auto * ids = body.find("model_ids"); diff --git a/app/server/runtime.h b/app/server/runtime.h index 45af1c555..3cfb6e0e7 100644 --- a/app/server/runtime.h +++ b/app/server/runtime.h @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -168,6 +169,10 @@ class ServerState final : public IHttpHandler { HttpResponse handle_voices(const HttpRequest & request) const; HttpResponse handle_unload_models(const std::string & body_text); HttpResponse handle_unload_all_models(); + // Background loop started when idle_unload_ms > 0: when the server has gone + // that long without a model load/run, unloads every resident (non-busy) model. + void idle_unload_loop(); + void unload_idle_models(); std::string models_json(bool include_session_options = false) const; std::string get_allowed_origin(const HttpRequest & request) const; @@ -189,6 +194,10 @@ class ServerState final : public IHttpHandler { std::unique_ptr model_installer_; #endif std::atomic next_upload_id_{1}; + // Steady-clock ms of the most recent model load/run; drives idle unload. + std::atomic last_activity_ms_{0}; + std::atomic idle_unload_shutdown_{false}; + std::thread idle_unload_thread_; }; } // namespace minitts::server From 454f099315f9df7a4873c3e8745adccd1cdf13be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E5=BA=86=E4=B8=B0?= Date: Tue, 25 Aug 2026 14:52:24 +0800 Subject: [PATCH 2/6] feat(server): refuse model load when host/GPU memory is insufficient Before every lazy model load, estimate the model's resident footprint (weights plus runtime overhead) and compare against free host memory and, for GPU backends, the backend device's free memory. Refuse the load with HTTP 503 insufficient_memory when estimate + configured headroom does not fit, instead of exhausting the machine (the previous failure mode was kIOGPUCommandBufferCallbackErrorOutOfMemory after models accumulated on a 16GB Mac). - add ServerConfig.min_free_memory_mb (default 512 MiB headroom), parsed from server.json and overridable via --min-free-memory-mb - add a macOS implementation of available_host_memory_bytes() using Mach VM stats (free + inactive + purgeable pages); Linux/Windows were already covered - add InsufficientMemoryError, mapped to 503 insufficient_memory --- app/server/busy_guard.h | 8 ++++ app/server/config.cpp | 4 ++ app/server/config.h | 6 +++ app/server/main.cpp | 12 +++++- app/server/runtime.cpp | 62 ++++++++++++++++++++++++++++++ app/server/runtime.h | 7 ++++ src/framework/core/host_memory.cpp | 33 +++++++++++++++- 7 files changed, 128 insertions(+), 4 deletions(-) diff --git a/app/server/busy_guard.h b/app/server/busy_guard.h index 2c130ff6d..80c6e0f6b 100644 --- a/app/server/busy_guard.h +++ b/app/server/busy_guard.h @@ -18,6 +18,14 @@ class ServerBusyError : public std::runtime_error { using std::runtime_error::runtime_error; }; +// The host/GPU does not currently have enough free memory to load a model. +// Mapped to HTTP 503: a transient server condition the caller may retry after +// other models have been evicted or the system has freed memory. +class InsufficientMemoryError : public std::runtime_error { +public: + explicit InsufficientMemoryError(const std::string & message) : std::runtime_error(message) {} +}; + inline std::int64_t steady_now_ms() { return std::chrono::duration_cast( std::chrono::steady_clock::now().time_since_epoch()) diff --git a/app/server/config.cpp b/app/server/config.cpp index ba52fcd62..ed98865c5 100644 --- a/app/server/config.cpp +++ b/app/server/config.cpp @@ -236,6 +236,7 @@ ServerConfig load_server_config(const std::filesystem::path & path) { config.busy_timeout_ms = engine::io::json::optional_i32(root, "busy_timeout_ms", config.busy_timeout_ms); config.max_loaded_models = engine::io::json::optional_i32(root, "max_loaded_models", config.max_loaded_models); config.idle_unload_ms = engine::io::json::optional_i32(root, "idle_unload_ms", config.idle_unload_ms); + config.min_free_memory_mb = engine::io::json::optional_i32(root, "min_free_memory_mb", config.min_free_memory_mb); if (const auto * value = root.find("live_ingest")) { config.live_ingest = parse_live_ingest_limits(*value, config.live_ingest, "server live_ingest"); } @@ -260,6 +261,9 @@ ServerConfig load_server_config(const std::filesystem::path & path) { if (config.idle_unload_ms < 0) { throw std::runtime_error("server idle_unload_ms must be >= 0 (0 disables idle unload)"); } + if (config.min_free_memory_mb < 0) { + throw std::runtime_error("server min_free_memory_mb must be >= 0 (0 disables the headroom)"); + } if (config.threads <= 0) { throw std::runtime_error("server threads must be positive"); } diff --git a/app/server/config.h b/app/server/config.h index ca0a51541..9c4ac00e7 100644 --- a/app/server/config.h +++ b/app/server/config.h @@ -95,6 +95,12 @@ struct ServerConfig { // max_loaded_models: that bounds peak residency, this frees memory during // quiet periods. The next request reloads lazily. int idle_unload_ms = 0; + // Minimum free memory (host and GPU, each) the server must retain after + // loading a model, in MiB. Before every lazy load the server estimates the + // model's resident footprint from its weights and refuses to load when + // estimate + this headroom would not fit. 0 disables the extra headroom + // (the estimate must still fit in what is free). + int min_free_memory_mb = 512; // Fleet-wide bounds for incrementally delivered request bodies. The defaults are // in LiveIngestLimits; a model entry may override any subset of them. LiveIngestLimits live_ingest; diff --git a/app/server/main.cpp b/app/server/main.cpp index 11335111d..46eb85d98 100644 --- a/app/server/main.cpp +++ b/app/server/main.cpp @@ -62,7 +62,7 @@ void print_help() { std::cout << "audiocpp_server [--config ] [--ui] [--host ] [--port ] [--backend ]\n" << " [--device ] [--list-devices] [--threads ] [--busy-timeout-ms ]\n" - << " [--max-loaded-models ] [--idle-unload-ms ]\n" + << " [--max-loaded-models ] [--idle-unload-ms ] [--min-free-memory-mb ]\n" << " [--model-spec-override ] [--voice-dir ]\n" << " [--log] [--log-file ]\n" << " [--cors-origins ]\n" @@ -78,7 +78,9 @@ void print_help() { << " the least recently used idle model first; 1 enforces\n" << " a single loaded model, default 0 (no limit)\n" << " --idle-unload-ms unload all resident models after this many ms without\n" << " any model load/run; default 0 (disabled), next request\n" - << " reloads lazily\n" + << " reloads lazily\n" << " --min-free-memory-mb refuse a model load unless host and GPU each keep at\n" + << " least this many MiB free after the load; default 1024,\n" + << " 0 disables the extra headroom\n" << " --voice-dir override the shared reference voice library directory\n" << " --cors-origins \"*\" experimental; disabled by default. Allows browser\n" << " requests from any origin for trusted local demos only\n" @@ -193,6 +195,9 @@ int main(int argc, char ** argv) { if (const auto idle_unload_ms = arg_value(argc, argv, "--idle-unload-ms")) { config.idle_unload_ms = std::stoi(*idle_unload_ms); } + if (const auto min_free_memory_mb = arg_value(argc, argv, "--min-free-memory-mb")) { + config.min_free_memory_mb = std::stoi(*min_free_memory_mb); + } if (const auto model_spec = arg_value(argc, argv, "--model-spec-override")) { config.model_spec_override = std::filesystem::path(*model_spec); } @@ -214,6 +219,9 @@ int main(int argc, char ** argv) { if (config.idle_unload_ms < 0) { throw std::runtime_error("--idle-unload-ms must be >= 0 (0 disables idle unload)"); } + if (config.min_free_memory_mb < 0) { + throw std::runtime_error("--min-free-memory-mb must be >= 0 (0 disables the headroom)"); + } const auto ui_resource_anchor = executable_directory(argc > 0 ? argv[0] : nullptr); minitts::server::ServerState state( diff --git a/app/server/runtime.cpp b/app/server/runtime.cpp index 2e895140d..50536a038 100644 --- a/app/server/runtime.cpp +++ b/app/server/runtime.cpp @@ -8,6 +8,7 @@ #include "../streaming/pcm_source.h" #include "../streaming/streaming.h" +#include "engine/framework/core/host_memory.h" #include "engine/framework/debug/trace.h" #include "engine/framework/io/json.h" #include "engine/framework/model_spec/metadata.h" @@ -22,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -1107,6 +1109,8 @@ HttpResponse ServerState::handle(const HttpRequest & request) { // there. Checked before ServerBusyError only because both are // runtime_error; the two conditions are disjoint. response = error_response(400, ex.what(), "invalid_request_error"); + } catch (const InsufficientMemoryError & ex) { + response = error_response(503, ex.what(), "insufficient_memory"); } catch (const ServerBusyError & ex) { // Non-streaming requests surface the busy state as 503 before any response is // sent. (Streaming requests acquire the lock inside the stream body, after @@ -1644,6 +1648,7 @@ void ServerState::ensure_model_loaded_locked(LoadedModel & model) { load_lock = std::unique_lock(model_load_mutex_); evict_for_model_limit(model); } + ensure_model_fits_memory(model.config); auto registry = engine::runtime::make_default_registry(); engine::runtime::ModelLoadRequest load_request; @@ -2799,6 +2804,63 @@ void ServerState::unload_idle_models() { } } +namespace { +std::string format_bytes(size_t bytes) { + const double gib = static_cast(bytes) / (1024.0 * 1024.0 * 1024.0); + std::ostringstream out; + out << std::fixed << std::setprecision(2) << gib << " GiB"; + return out.str(); +} +} // namespace + +size_t ServerState::estimate_model_memory_bytes(const ServerModelConfig & model) const { + size_t weights = 0; + const auto add_file = [&weights](const std::filesystem::path & path) { + std::error_code ec; + if (std::filesystem::is_regular_file(path, ec)) { + weights += static_cast(std::filesystem::file_size(path, ec)); + } + }; + add_file(model.path); + for (const auto & [key, value] : model.session_options) { + (void)key; + std::filesystem::path aux(value); + if (aux.is_relative()) { + aux = model.path.parent_path() / aux; + } + add_file(aux); + } + // Weights plus a runtime factor for Metal/GPU buffers, activation graphs and + // KV state, plus a fixed floor for per-model bookkeeping. + constexpr double kRuntimeOverheadFactor = 1.5; + constexpr size_t kFixedOverhead = 128ull * 1024 * 1024; + return static_cast(static_cast(weights) * kRuntimeOverheadFactor) + kFixedOverhead; +} + +void ServerState::ensure_model_fits_memory(const ServerModelConfig & model) { + const size_t estimate = estimate_model_memory_bytes(model); + const size_t headroom = static_cast(config_.min_free_memory_mb) * 1024ull * 1024ull; + + const size_t host_available = engine::core::available_host_memory_bytes(); + if (host_available > 0 && estimate + headroom > host_available) { + throw InsufficientMemoryError( + "cannot load model '" + model.id + "': estimated " + format_bytes(estimate) + + " + " + std::to_string(config_.min_free_memory_mb) + " MiB headroom exceeds available host memory (" + + format_bytes(host_available) + ")"); + } + + const engine::core::BackendMemorySnapshot device = + engine::core::query_backend_memory(engine::core::BackendConfig{ + config_.backend, config_.device, config_.threads}); + if (device.available && estimate + headroom > static_cast(device.free_bytes)) { + throw InsufficientMemoryError( + "cannot load model '" + model.id + "': estimated " + format_bytes(estimate) + + " + " + std::to_string(config_.min_free_memory_mb) + " MiB headroom exceeds available " + + backend_name(config_.backend) + " memory (" + + format_bytes(static_cast(device.free_bytes)) + ")"); + } +} + HttpResponse ServerState::handle_unload_models(const std::string & body_text) { const auto body = engine::io::json::parse(body_text); const auto * ids = body.find("model_ids"); diff --git a/app/server/runtime.h b/app/server/runtime.h index 3cfb6e0e7..2f42e97f9 100644 --- a/app/server/runtime.h +++ b/app/server/runtime.h @@ -108,6 +108,13 @@ class ServerState final : public IHttpHandler { // `loading` fits within the limit. A model mid-inference is never a victim; // when nothing can be evicted this throws ServerBusyError (-> HTTP 503). void evict_for_model_limit(const LoadedModel & loading); + // Estimated resident bytes this model will occupy once loaded (weights plus + // a runtime overhead factor for GPU buffers / compute graphs). + size_t estimate_model_memory_bytes(const ServerModelConfig & model) const; + // Refuse the load with InsufficientMemoryError (-> HTTP 503) when the + // estimated footprint plus configured headroom does not fit the free host + // memory and (for GPU backends) the backend device memory. + void ensure_model_fits_memory(const ServerModelConfig & model); LoadedModel & require_model(const engine::io::json::Value & body); const LoadedModel::RuntimeVoicePreset * select_voice_preset( const LoadedModel & model, diff --git a/src/framework/core/host_memory.cpp b/src/framework/core/host_memory.cpp index 377857c74..d72552962 100644 --- a/src/framework/core/host_memory.cpp +++ b/src/framework/core/host_memory.cpp @@ -5,7 +5,10 @@ #elif defined(__linux__) #include #include -#elif defined(__unix__) || defined(__APPLE__) +#elif defined(__APPLE__) +#include +#include +#elif defined(__unix__) #include #endif @@ -36,6 +39,28 @@ size_t mem_available_bytes() { return bytes; } +} // namespace +#elif defined(__APPLE__) +namespace { + +// Mach VM stats: pages that a fresh allocation can claim without swapping. +// free_count is the idle pool; inactive_count is reclaimable file cache / +// anonymous memory; purgeable_count can be dropped on demand. Compressed +// pages are deliberately excluded -- dropping them would force decompression +// and they are already backing live data. +size_t mem_available_bytes() { + mach_msg_type_number_t count = HOST_VM_INFO64_COUNT; + vm_statistics64_data_t vm_stat{}; + if (host_statistics64(mach_host_self(), HOST_VM_INFO64, reinterpret_cast(&vm_stat), &count) != KERN_SUCCESS) { + return 0; + } + const uint64_t page = static_cast(vm_page_size); + return static_cast( + (static_cast(vm_stat.free_count) + + static_cast(vm_stat.inactive_count) + + static_cast(vm_stat.purgeable_count)) * page); +} + } // namespace #endif @@ -50,6 +75,10 @@ size_t available_host_memory_bytes() { if (const size_t bytes = mem_available_bytes(); bytes > 0) { return bytes; } +#elif defined(__APPLE__) + if (const size_t bytes = mem_available_bytes(); bytes > 0) { + return bytes; + } #elif defined(_SC_AVPHYS_PAGES) && defined(_SC_PAGE_SIZE) const long pages = sysconf(_SC_AVPHYS_PAGES); const long page_size = sysconf(_SC_PAGE_SIZE); @@ -57,7 +86,7 @@ size_t available_host_memory_bytes() { return static_cast(pages) * static_cast(page_size); } #endif - // macOS has no _SC_AVPHYS_PAGES, and any query can fail: report unknown. + // A query can fail on any platform: report unknown (0). return 0; } From d345331e7b27f1c4c3306f78a757c414b6047e17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E5=BA=86=E4=B8=B0?= Date: Tue, 25 Aug 2026 15:02:18 +0800 Subject: [PATCH 3/6] fix(server): address code review for idle unload and memory guard Independent review found no criticals; fix the actionable findings: - --min-free-memory-mb help text now matches the actual 512 MiB default - estimate_model_memory_bytes() sums directory-style model trees (with depth/file limits) instead of counting only regular files, so directory models are no longer estimated as 0 - the model load path is serialized even when max_loaded_models is 0, so concurrent lazy loads cannot both pass the memory pre-check - expose engine::core::ensure_backends_loaded() and call it before the GPU memory query so the very first load actually runs the device check - the idle-unload thread now wakes on shutdown in <=250ms slices instead of waiting out a full poll interval - document idle_unload_ms / min_free_memory_mb in app/server/README.md and example.json - add server_config_test coverage for the new fields (defaults, overrides, negative rejection) --- app/server/README.md | 4 ++ app/server/example.json | 3 ++ app/server/main.cpp | 2 +- app/server/runtime.cpp | 64 ++++++++++++++++++++----- include/engine/framework/core/backend.h | 5 ++ src/framework/core/backend.cpp | 4 +- src/framework/core/host_memory.cpp | 2 +- tests/unittests/test_server_config.cpp | 64 +++++++++++++++++++++++++ 8 files changed, 131 insertions(+), 17 deletions(-) diff --git a/app/server/README.md b/app/server/README.md index 0361a7e7d..caa0845f1 100644 --- a/app/server/README.md +++ b/app/server/README.md @@ -97,6 +97,10 @@ Set top-level `"lazy_load": true` to register all configured model ids at startu Set top-level `"max_loaded_models"` to bound how many models are resident in memory at once. When a request needs a model that is not loaded and the limit is already reached, the server first unloads the least recently used idle model (freeing VRAM on GPU backends) and reloads it on its own next request. `1` enforces a single loaded model at a time, which is the practical choice when each model alone nearly fills the device. Higher values keep that many most recently used models warm. The default `0` disables the limit. A model that is mid-inference is never unloaded; if the limit is reached and every loaded model is busy, the request fails with `503` so the client can retry. With more non-lazy models configured than the limit allows, startup loads the first `max_loaded_models` of them and defers the rest to their first request. The equivalent command-line option is `--max-loaded-models `. +Set top-level `"idle_unload_ms"` to have the server unload every resident model after it has gone that long without any model load/run. The next request reloads lazily; a model mid-inference is never unloaded. This complements `max_loaded_models`: that bounds peak residency, this frees memory during quiet periods. Defaults to `0` (disabled). The `--idle-unload-ms ` command-line flag overrides the config value. + +Set top-level `"min_free_memory_mb"` to refuse a model load when the host or the GPU backend does not have that much free memory left after the estimated footprint of the new model. The estimate is the model's weights (including session auxiliary files; directory trees are summed with depth/file limits) times a runtime overhead factor plus a fixed floor. When the check fails, the request returns HTTP 503 with `insufficient_memory`; the client may retry later. Defaults to `512`; `0` disables the extra headroom (the estimate must still fit in what is free). The `--min-free-memory-mb ` command-line flag overrides the config value. + Set per-model `"default_request_options"` to apply request-option defaults to every request for that model. Values supplied by the actual request body override these defaults. Set top-level `"max_request_body_bytes"` to bound the largest HTTP request body buffered in host RAM before routing. This protects endpoints that accept JSON or audio uploads from unbounded `Content-Length` claims. The default is `2147483648` bytes (2 GiB). Raise or lower it to match the largest upload your deployment intends to accept. Values above `2^53 - 1` are rejected because this config parser stores JSON numbers as doubles. diff --git a/app/server/example.json b/app/server/example.json index 043c30edf..7b10327f0 100644 --- a/app/server/example.json +++ b/app/server/example.json @@ -5,6 +5,9 @@ "device": 0, "threads": 1, "lazy_load": true, + "max_loaded_models": 0, + "idle_unload_ms": 0, + "min_free_memory_mb": 512, "models": [ { "id": "pocket-tts", diff --git a/app/server/main.cpp b/app/server/main.cpp index 46eb85d98..100ae2453 100644 --- a/app/server/main.cpp +++ b/app/server/main.cpp @@ -79,7 +79,7 @@ void print_help() { << " a single loaded model, default 0 (no limit)\n" << " --idle-unload-ms unload all resident models after this many ms without\n" << " any model load/run; default 0 (disabled), next request\n" << " reloads lazily\n" << " --min-free-memory-mb refuse a model load unless host and GPU each keep at\n" - << " least this many MiB free after the load; default 1024,\n" + << " least this many MiB free after the load; default 512,\n" << " 0 disables the extra headroom\n" << " --voice-dir override the shared reference voice library directory\n" << " --cors-origins \"*\" experimental; disabled by default. Allows browser\n" diff --git a/app/server/runtime.cpp b/app/server/runtime.cpp index 50536a038..42e5505cd 100644 --- a/app/server/runtime.cpp +++ b/app/server/runtime.cpp @@ -1643,10 +1643,16 @@ void ServerState::ensure_model_loaded_locked(LoadedModel & model) { if (model.session != nullptr) { return; } - std::unique_lock load_lock; + // Serialize the whole "evict -> memory check -> load" sequence even when + // max_loaded_models is 0 (no eviction): two concurrent lazy loads could + // otherwise both pass the memory pre-check before either allocates, and the + // guard would be meaningless. A model mid-inference is never a victim. + std::unique_lock load_lock(model_load_mutex_); if (config_.max_loaded_models > 0) { - load_lock = std::unique_lock(model_load_mutex_); evict_for_model_limit(model); + // Note: if ensure_model_fits_memory() below then refuses the load, the + // evicted model is already unloaded (freed memory is never restored). + // That is acceptable: the 503 tells the client to retry later. } ensure_model_fits_memory(model.config); auto registry = engine::runtime::make_default_registry(); @@ -2763,14 +2769,22 @@ void ServerState::LoadedModel::unload() { void ServerState::idle_unload_loop() { const auto interval_ms = std::max(1000, config_.idle_unload_ms / 10); + auto deadline = steady_now_ms() + interval_ms; while (!idle_unload_shutdown_.load(std::memory_order_relaxed)) { - std::this_thread::sleep_for(std::chrono::milliseconds(interval_ms)); - if (idle_unload_shutdown_.load(std::memory_order_relaxed)) { - break; - } - const auto idle_ms = steady_now_ms() - last_activity_ms_.load(std::memory_order_relaxed); - if (idle_ms >= config_.idle_unload_ms) { - unload_idle_models(); + // Sleep in small slices so SIGTERM (destructor join) never waits out a + // full poll interval; the deadline keeps the idle check cadence intact. + const auto now = steady_now_ms(); + if (now >= deadline) { + if (idle_unload_shutdown_.load(std::memory_order_relaxed)) { + break; + } + const auto idle_ms = now - last_activity_ms_.load(std::memory_order_relaxed); + if (idle_ms >= config_.idle_unload_ms) { + unload_idle_models(); + } + deadline = steady_now_ms() + interval_ms; + } else { + std::this_thread::sleep_for(std::chrono::milliseconds(250)); } } } @@ -2815,20 +2829,39 @@ std::string format_bytes(size_t bytes) { size_t ServerState::estimate_model_memory_bytes(const ServerModelConfig & model) const { size_t weights = 0; - const auto add_file = [&weights](const std::filesystem::path & path) { - std::error_code ec; + std::error_code ec; + // Directories (model_spec / HF-style checkpoints) are summed recursively with + // hard limits so a pathological tree cannot stall the load path or blow the + // counter; anything beyond the limits contributes 0 (the fixed floor below + // still applies, so the estimate never reads as completely empty). + constexpr size_t kMaxDepth = 3; + constexpr size_t kMaxFiles = 10000; + size_t visited_files = 0; + const auto add_file = [&](const std::filesystem::path & path) { if (std::filesystem::is_regular_file(path, ec)) { weights += static_cast(std::filesystem::file_size(path, ec)); + ++visited_files; } }; - add_file(model.path); + const std::function add_tree = + [&](const std::filesystem::path & path, size_t depth) { + if (std::filesystem::is_regular_file(path, ec)) { + add_file(path); + } else if (std::filesystem::is_directory(path, ec) && depth < kMaxDepth) { + std::filesystem::directory_iterator it(path, ec), end; + for (; it != end && visited_files < kMaxFiles; it.increment(ec)) { + add_tree(it->path(), depth + 1); + } + } + }; + add_tree(model.path, 0); for (const auto & [key, value] : model.session_options) { (void)key; std::filesystem::path aux(value); if (aux.is_relative()) { aux = model.path.parent_path() / aux; } - add_file(aux); + add_tree(aux, 0); } // Weights plus a runtime factor for Metal/GPU buffers, activation graphs and // KV state, plus a fixed floor for per-model bookkeeping. @@ -2849,6 +2882,11 @@ void ServerState::ensure_model_fits_memory(const ServerModelConfig & model) { format_bytes(host_available) + ")"); } + // ggml backend registries may not be loaded yet on the very first request + // (init_backend happens inside a session load, after this pre-check), which + // would make query_backend_memory() report "unknown" and silently skip the + // GPU check. Loading them here is idempotent and cheap once done. + engine::core::ensure_backends_loaded(); const engine::core::BackendMemorySnapshot device = engine::core::query_backend_memory(engine::core::BackendConfig{ config_.backend, config_.device, config_.threads}); diff --git a/include/engine/framework/core/backend.h b/include/engine/framework/core/backend.h index d56637e87..5c2cb9914 100644 --- a/include/engine/framework/core/backend.h +++ b/include/engine/framework/core/backend.h @@ -32,6 +32,11 @@ struct BackendDeviceInfo { std::vector list_backend_devices(); void print_backend_devices(std::ostream & out); +// Load every registered ggml backend registry (idempotent). Needed before +// query_backend_memory() can report GPU memory on a process that has not yet +// initialized a backend. +void ensure_backends_loaded(); + struct BackendMemorySnapshot { bool available = false; int64_t total_bytes = 0; diff --git a/src/framework/core/backend.cpp b/src/framework/core/backend.cpp index afb5489ed..902b73120 100644 --- a/src/framework/core/backend.cpp +++ b/src/framework/core/backend.cpp @@ -10,14 +10,14 @@ namespace engine::core { -namespace { - void ensure_backends_loaded() { if (ggml_backend_reg_count() == 0) { ggml_backend_load_all(); } } +namespace { + // A backend is identified by the name of the ggml registry that owns it. The device type // (GPU/IGPU/ACCEL) deliberately plays no part in that: Metal reports GPU rather than ACCEL, // Vulkan reports IGPU on integrated GPUs, and those values are free to change upstream. diff --git a/src/framework/core/host_memory.cpp b/src/framework/core/host_memory.cpp index d72552962..dd8694cf1 100644 --- a/src/framework/core/host_memory.cpp +++ b/src/framework/core/host_memory.cpp @@ -6,8 +6,8 @@ #include #include #elif defined(__APPLE__) +#include #include -#include #elif defined(__unix__) #include #endif diff --git a/tests/unittests/test_server_config.cpp b/tests/unittests/test_server_config.cpp index 8c830802c..654f0ad29 100644 --- a/tests/unittests/test_server_config.cpp +++ b/tests/unittests/test_server_config.cpp @@ -432,6 +432,66 @@ void test_request_timeout_is_clamped_to_policy() { "unbounded on both sides stays unbounded"); } +void test_idle_unload_ms_defaults_and_overrides() { + const auto root = make_temp_root(); + + const auto default_path = write_config( + root, "idle_unload_default.json", std::string("{") + kMinimalModel + "}"); + require( + minitts::server::load_server_config(default_path).idle_unload_ms == 0, + "idle_unload_ms defaults to 0 (disabled) when omitted"); + + const auto set_path = write_config( + root, "idle_unload_set.json", std::string(R"JSON({"idle_unload_ms": 300000,)JSON") + kMinimalModel + "}"); + require( + minitts::server::load_server_config(set_path).idle_unload_ms == 300000, + "idle_unload_ms is read from the config"); +} + +void test_negative_idle_unload_ms_is_rejected() { + const auto root = make_temp_root(); + const auto config_path = write_config( + root, "idle_unload_negative.json", std::string(R"JSON({"idle_unload_ms": -1,)JSON") + kMinimalModel + "}"); + + bool rejected = false; + try { + (void) minitts::server::load_server_config(config_path); + } catch (const std::runtime_error & error) { + rejected = std::string(error.what()).find("idle_unload_ms") != std::string::npos; + } + require(rejected, "negative idle_unload_ms is rejected"); +} + +void test_min_free_memory_mb_defaults_and_overrides() { + const auto root = make_temp_root(); + + const auto default_path = write_config( + root, "min_free_default.json", std::string("{") + kMinimalModel + "}"); + require( + minitts::server::load_server_config(default_path).min_free_memory_mb == 512, + "min_free_memory_mb defaults to 512 MiB when omitted"); + + const auto set_path = write_config( + root, "min_free_set.json", std::string(R"JSON({"min_free_memory_mb": 0,)JSON") + kMinimalModel + "}"); + require( + minitts::server::load_server_config(set_path).min_free_memory_mb == 0, + "min_free_memory_mb accepts 0 to disable the headroom"); +} + +void test_negative_min_free_memory_mb_is_rejected() { + const auto root = make_temp_root(); + const auto config_path = write_config( + root, "min_free_negative.json", std::string(R"JSON({"min_free_memory_mb": -1,)JSON") + kMinimalModel + "}"); + + bool rejected = false; + try { + (void) minitts::server::load_server_config(config_path); + } catch (const std::runtime_error & error) { + rejected = std::string(error.what()).find("min_free_memory_mb") != std::string::npos; + } + require(rejected, "negative min_free_memory_mb is rejected"); +} + void test_model_run_overrun_predicate() { using minitts::server::model_run_has_overrun; @@ -459,6 +519,10 @@ int main() { test_negative_busy_timeout_is_rejected(); test_max_loaded_models_defaults_and_overrides(); test_negative_max_loaded_models_is_rejected(); + test_idle_unload_ms_defaults_and_overrides(); + test_negative_idle_unload_ms_is_rejected(); + test_min_free_memory_mb_defaults_and_overrides(); + test_negative_min_free_memory_mb_is_rejected(); test_per_model_busy_timeout(); test_negative_per_model_busy_timeout_is_rejected(); test_ui_configuration(); From 5a0c1c44c16297cd6def376199ee45ecbcb08ce5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E5=BA=86=E4=B8=B0?= Date: Tue, 25 Aug 2026 15:11:16 +0800 Subject: [PATCH 4/6] ci(server): add cross-platform build+test matrix for memory guard Run audiocpp_server build + server_config_test on ubuntu/windows/macos to prove the idle-unload and pre-load memory-check changes compile and behave on all three desktop platforms. GPU backends are off here (the changes query memory through the backend-agnostic ggml_backend_dev_memory; existing workflows already cover CUDA/Vulkan/Metal builds). --- .github/workflows/server-memory-guard.yml | 80 +++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 .github/workflows/server-memory-guard.yml diff --git a/.github/workflows/server-memory-guard.yml b/.github/workflows/server-memory-guard.yml new file mode 100644 index 000000000..3b16bd8ad --- /dev/null +++ b/.github/workflows/server-memory-guard.yml @@ -0,0 +1,80 @@ +name: server-memory-guard cross-platform + +# Verifies the server memory-management changes (--idle-unload-ms and +# --min-free-memory-mb) compile and the config unit tests pass on all three +# desktop platforms. GPU backends are disabled here: the changes themselves are +# backend-agnostic (they query memory through ggml_backend_dev_memory), and the +# existing linux/mac/windows workflows already cover CUDA/Vulkan/Metal builds. + +on: + workflow_dispatch: + pull_request: + paths: + - "app/server/**" + - "src/framework/core/host_memory.cpp" + - "include/engine/framework/core/backend.h" + - "src/framework/core/backend.cpp" + - "tests/unittests/test_server_config.cpp" + - ".github/workflows/server-memory-guard.yml" + +jobs: + build-test: + name: ${{ matrix.os }} build+test + runs-on: ${{ matrix.os }} + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, windows-2022, macos-14] + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Configure (Linux/macOS) + if: runner.os != 'Windows' + run: | + cmake -S . -B build \ + -DCMAKE_BUILD_TYPE=Release \ + -DENGINE_BUILD_TESTS=ON \ + -DENGINE_ENABLE_OPENMP=OFF \ + -DENGINE_ENABLE_CUDA=OFF \ + -DENGINE_ENABLE_VULKAN=OFF \ + -DENGINE_ENABLE_METAL=OFF + + - name: Configure (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: | + cmake -S . -B build ` + -DCMAKE_BUILD_TYPE=Release ` + -DENGINE_BUILD_TESTS=ON ` + -DENGINE_ENABLE_OPENMP=OFF ` + -DENGINE_ENABLE_CUDA=OFF ` + -DENGINE_ENABLE_VULKAN=OFF ` + -DENGINE_ENABLE_METAL=OFF + + - name: Build (Linux/macOS) + if: runner.os != 'Windows' + run: | + cmake --build build --target audiocpp_server server_config_test --parallel "$(nproc)" + + - name: Build (macOS nproc fallback) + if: runner.os == 'macOS' + run: | + cmake --build build --target audiocpp_server server_config_test --parallel "$(sysctl -n hw.logicalcpu)" + + - name: Build (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: | + cmake --build build --config Release --target audiocpp_server server_config_test --parallel + + - name: Run server_config_test (Linux/macOS) + if: runner.os != 'Windows' + run: ./build/bin/server_config_test + + - name: Run server_config_test (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: .\build\bin\Release\server_config_test.exe From 8d9d677d0c25d68341c5a7a880af0d91696d3057 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E5=BA=86=E4=B8=B0?= Date: Tue, 25 Aug 2026 15:44:51 +0800 Subject: [PATCH 5/6] ci(server): fix macOS build step to not run the nproc variant The Linux build step used nproc and its condition also matched macOS, so macOS ran both build steps and the nproc one stalled. Scope the nproc step to Linux and let macOS use sysctl -n hw.logicalcpu. --- .github/workflows/server-memory-guard.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/server-memory-guard.yml b/.github/workflows/server-memory-guard.yml index 3b16bd8ad..57f5829f2 100644 --- a/.github/workflows/server-memory-guard.yml +++ b/.github/workflows/server-memory-guard.yml @@ -54,12 +54,12 @@ jobs: -DENGINE_ENABLE_VULKAN=OFF ` -DENGINE_ENABLE_METAL=OFF - - name: Build (Linux/macOS) - if: runner.os != 'Windows' + - name: Build (Linux) + if: runner.os == 'Linux' run: | cmake --build build --target audiocpp_server server_config_test --parallel "$(nproc)" - - name: Build (macOS nproc fallback) + - name: Build (macOS) if: runner.os == 'macOS' run: | cmake --build build --target audiocpp_server server_config_test --parallel "$(sysctl -n hw.logicalcpu)" From 0082244d2b6b0719cdf1c3bfe50eeb8c77896264 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E5=BA=86=E4=B8=B0?= Date: Wed, 26 Aug 2026 10:20:55 +0800 Subject: [PATCH 6/6] fix(server): address maintainer review on memory guard Address 0xShug0's review on #306: 1. Make the memory guard opt-in: min_free_memory_mb now defaults to 0 and 0 disables ensure_model_fits_memory entirely, so existing configs that never opted in see no load-behavior change. 2. Stop over-summing directory models and stop masking the real loader error. estimate_model_memory_bytes now estimates only what the loader will read: a single file, the one GGUF a directory selects (find_directory_gguf), or a full safetensors/HF tree. A directory with several GGUFs and no model.gguf is ambiguous, so the guard estimates nothing and the loader's own "contains N GGUF files" error surfaces instead of a misleading 503. 3. Resolve relative session-option paths against the model directory when model.path is a directory (parent_path only when it is a file). 4. Measure idle time from request completion, not start: run_model and run_streaming_model_impl now stamp last_activity_ms_ on completion, so a long inference is not unloaded the moment it returns. 5. Only serialize lazy loads through model_load_mutex_ when a guard needs it (max_loaded_models > 0 or min_free_memory_mb > 0); with both off, unrelated first-load requests keep their original concurrency. Verified on macOS (Apple M4): audiocpp_server + server_config_test build and pass; startup smoke shows a single-GGUF dir estimates just the selected file (10.83 GiB -> 503 when guard on), and an ambiguous multi-GGUF dir surfaces the real "contains 2 GGUF files" loader error with the guard both on and off. --- app/server/README.md | 2 +- app/server/config.cpp | 2 +- app/server/config.h | 7 +-- app/server/example.json | 2 +- app/server/main.cpp | 2 +- app/server/runtime.cpp | 63 +++++++++++++++++++++----- app/server/runtime.h | 11 +++-- tests/unittests/test_server_config.cpp | 10 ++-- 8 files changed, 72 insertions(+), 27 deletions(-) diff --git a/app/server/README.md b/app/server/README.md index caa0845f1..30bcd1319 100644 --- a/app/server/README.md +++ b/app/server/README.md @@ -99,7 +99,7 @@ Set top-level `"max_loaded_models"` to bound how many models are resident in mem Set top-level `"idle_unload_ms"` to have the server unload every resident model after it has gone that long without any model load/run. The next request reloads lazily; a model mid-inference is never unloaded. This complements `max_loaded_models`: that bounds peak residency, this frees memory during quiet periods. Defaults to `0` (disabled). The `--idle-unload-ms ` command-line flag overrides the config value. -Set top-level `"min_free_memory_mb"` to refuse a model load when the host or the GPU backend does not have that much free memory left after the estimated footprint of the new model. The estimate is the model's weights (including session auxiliary files; directory trees are summed with depth/file limits) times a runtime overhead factor plus a fixed floor. When the check fails, the request returns HTTP 503 with `insufficient_memory`; the client may retry later. Defaults to `512`; `0` disables the extra headroom (the estimate must still fit in what is free). The `--min-free-memory-mb ` command-line flag overrides the config value. +Set top-level `"min_free_memory_mb"` to refuse a model load when the host or the GPU backend does not have that much free memory left after the estimated footprint of the new model. The estimate covers only what the loader will actually read: a single-file model's weights, the one GGUF a model directory selects (`model.gguf` or the sole `*.gguf`), or a full safetensors/HF checkpoint tree, plus any session auxiliary files. A directory holding several GGUFs with no `model.gguf` is ambiguous, so the guard makes no estimate there and the loader's own error surfaces instead. The estimate is scaled by a runtime overhead factor plus a fixed floor. When the check fails, the request returns HTTP 503 with `insufficient_memory`; the client may retry later. Defaults to `0`, which disables the guard entirely so existing deployments are unaffected; set a positive value to opt in. The `--min-free-memory-mb ` command-line flag overrides the config value. Set per-model `"default_request_options"` to apply request-option defaults to every request for that model. Values supplied by the actual request body override these defaults. diff --git a/app/server/config.cpp b/app/server/config.cpp index ed98865c5..1199dd15d 100644 --- a/app/server/config.cpp +++ b/app/server/config.cpp @@ -262,7 +262,7 @@ ServerConfig load_server_config(const std::filesystem::path & path) { throw std::runtime_error("server idle_unload_ms must be >= 0 (0 disables idle unload)"); } if (config.min_free_memory_mb < 0) { - throw std::runtime_error("server min_free_memory_mb must be >= 0 (0 disables the headroom)"); + throw std::runtime_error("server min_free_memory_mb must be >= 0 (0 disables the memory guard)"); } if (config.threads <= 0) { throw std::runtime_error("server threads must be positive"); diff --git a/app/server/config.h b/app/server/config.h index 9c4ac00e7..43f83f020 100644 --- a/app/server/config.h +++ b/app/server/config.h @@ -98,9 +98,10 @@ struct ServerConfig { // Minimum free memory (host and GPU, each) the server must retain after // loading a model, in MiB. Before every lazy load the server estimates the // model's resident footprint from its weights and refuses to load when - // estimate + this headroom would not fit. 0 disables the extra headroom - // (the estimate must still fit in what is free). - int min_free_memory_mb = 512; + // estimate + this headroom would not fit. 0 disables the memory guard + // entirely (the default), so existing deployments see no behavior change + // unless they opt in. + int min_free_memory_mb = 0; // Fleet-wide bounds for incrementally delivered request bodies. The defaults are // in LiveIngestLimits; a model entry may override any subset of them. LiveIngestLimits live_ingest; diff --git a/app/server/example.json b/app/server/example.json index 7b10327f0..116901419 100644 --- a/app/server/example.json +++ b/app/server/example.json @@ -7,7 +7,7 @@ "lazy_load": true, "max_loaded_models": 0, "idle_unload_ms": 0, - "min_free_memory_mb": 512, + "min_free_memory_mb": 0, "models": [ { "id": "pocket-tts", diff --git a/app/server/main.cpp b/app/server/main.cpp index 100ae2453..890025591 100644 --- a/app/server/main.cpp +++ b/app/server/main.cpp @@ -220,7 +220,7 @@ int main(int argc, char ** argv) { throw std::runtime_error("--idle-unload-ms must be >= 0 (0 disables idle unload)"); } if (config.min_free_memory_mb < 0) { - throw std::runtime_error("--min-free-memory-mb must be >= 0 (0 disables the headroom)"); + throw std::runtime_error("--min-free-memory-mb must be >= 0 (0 disables the memory guard)"); } const auto ui_resource_anchor = executable_directory(argc > 0 ? argv[0] : nullptr); diff --git a/app/server/runtime.cpp b/app/server/runtime.cpp index 42e5505cd..141b35067 100644 --- a/app/server/runtime.cpp +++ b/app/server/runtime.cpp @@ -9,6 +9,7 @@ #include "../streaming/streaming.h" #include "engine/framework/core/host_memory.h" +#include "engine/framework/assets/tensor_source.h" #include "engine/framework/debug/trace.h" #include "engine/framework/io/json.h" #include "engine/framework/model_spec/metadata.h" @@ -1643,11 +1644,16 @@ void ServerState::ensure_model_loaded_locked(LoadedModel & model) { if (model.session != nullptr) { return; } - // Serialize the whole "evict -> memory check -> load" sequence even when - // max_loaded_models is 0 (no eviction): two concurrent lazy loads could - // otherwise both pass the memory pre-check before either allocates, and the - // guard would be meaningless. A model mid-inference is never a victim. - std::unique_lock load_lock(model_load_mutex_); + // Serialize the whole "evict -> memory check -> load" sequence only when a guard + // actually needs it: eviction (max_loaded_models > 0) or the memory pre-check + // (min_free_memory_mb > 0). With both off there is nothing for concurrent loads + // to race over, so unrelated first-load requests keep their original concurrency. + const bool serialize_load = + config_.max_loaded_models > 0 || config_.min_free_memory_mb > 0; + std::unique_lock load_lock(model_load_mutex_, std::defer_lock); + if (serialize_load) { + load_lock.lock(); + } if (config_.max_loaded_models > 0) { evict_for_model_limit(model); // Note: if ensure_model_fits_memory() below then refuses the load, the @@ -1927,6 +1933,10 @@ ServerState::TimedTaskResult ServerState::run_model( const auto started = Clock::now(); model.session->prepare(engine::runtime::build_preparation_request(request)); auto result = model.offline->run(request); + // Mark activity at completion too: idle unload must measure from when the + // request finished, not when it started, or a long inference would look idle + // (and be unloaded) the moment it returns. + last_activity_ms_.store(steady_now_ms(), std::memory_order_relaxed); return TimedTaskResult{std::move(result), elapsed_ms(started), std::nullopt}; } @@ -1964,6 +1974,9 @@ ServerState::TimedTaskResult ServerState::run_streaming_model_impl( if (!timed_result.ttft_ms.has_value() && task_result_has_output(timed_result.result)) { timed_result.ttft_ms = timed_result.wall_ms; } + // Mark activity at completion too (see run_model): idle unload measures from + // request finish, so a long stream is not unloaded the moment it ends. + last_activity_ms_.store(steady_now_ms(), std::memory_order_relaxed); return timed_result; } @@ -2830,10 +2843,10 @@ std::string format_bytes(size_t bytes) { size_t ServerState::estimate_model_memory_bytes(const ServerModelConfig & model) const { size_t weights = 0; std::error_code ec; - // Directories (model_spec / HF-style checkpoints) are summed recursively with - // hard limits so a pathological tree cannot stall the load path or blow the - // counter; anything beyond the limits contributes 0 (the fixed floor below - // still applies, so the estimate never reads as completely empty). + // Checkpoint trees (safetensors / HF-style directories) are summed recursively + // with hard limits so a pathological tree cannot stall the load path or blow the + // counter; anything beyond the limits contributes 0 (the fixed floor below still + // applies, so the estimate never reads as completely empty). constexpr size_t kMaxDepth = 3; constexpr size_t kMaxFiles = 10000; size_t visited_files = 0; @@ -2854,12 +2867,34 @@ size_t ServerState::estimate_model_memory_bytes(const ServerModelConfig & model) } } }; - add_tree(model.path, 0); + // Estimate only what the loader will actually read from model.path, so the + // guard neither overestimates nor masks the loader's own error: + // - a single-file model contributes that file; + // - a model directory contributes the one GGUF it selects (model.gguf, or the + // sole *.gguf) -- a package holding several variants is loaded from just one; + // - a directory with no GGUF is a safetensors/HF checkpoint whose whole tree + // loads, so it is summed; + // - a directory with several GGUFs and no model.gguf is ambiguous: the loader + // rejects it with "contains N GGUF files", so we estimate nothing rather than + // answer 503 and hide that real error. + if (std::filesystem::is_regular_file(model.path, ec)) { + add_file(model.path); + } else if (std::filesystem::is_directory(model.path, ec)) { + if (const auto selected = engine::assets::find_directory_gguf(model.path)) { + add_file(*selected); + } else if (engine::assets::directory_gguf_files(model.path).empty()) { + add_tree(model.path, 0); + } + } + // Relative auxiliary paths resolve against the model directory when model.path + // is a directory, and against the model file's parent when it is a file. + const std::filesystem::path aux_base = + std::filesystem::is_directory(model.path, ec) ? model.path : model.path.parent_path(); for (const auto & [key, value] : model.session_options) { (void)key; std::filesystem::path aux(value); if (aux.is_relative()) { - aux = model.path.parent_path() / aux; + aux = aux_base / aux; } add_tree(aux, 0); } @@ -2871,6 +2906,12 @@ size_t ServerState::estimate_model_memory_bytes(const ServerModelConfig & model) } void ServerState::ensure_model_fits_memory(const ServerModelConfig & model) { + // The guard is opt-in: 0 disables it entirely so existing deployments see no + // behavior change. This also keeps lazy loads unserialized (see the call site) + // when neither this guard nor max_loaded_models is active. + if (config_.min_free_memory_mb <= 0) { + return; + } const size_t estimate = estimate_model_memory_bytes(model); const size_t headroom = static_cast(config_.min_free_memory_mb) * 1024ull * 1024ull; diff --git a/app/server/runtime.h b/app/server/runtime.h index 2f42e97f9..bb00c2b7b 100644 --- a/app/server/runtime.h +++ b/app/server/runtime.h @@ -188,9 +188,10 @@ class ServerState final : public IHttpHandler { std::vector> models_; std::unordered_map model_index_; mutable std::mutex models_mutex_; - // Serializes framework loads while max_loaded_models is active, so two - // concurrent lazy loads cannot both pass the eviction check and overshoot - // the limit. Not taken when the limit is 0: loads stay concurrent there. + // Serializes framework loads while max_loaded_models or the memory guard + // (min_free_memory_mb) is active, so two concurrent lazy loads cannot both + // pass the eviction/memory check and overshoot. Not taken when both are off: + // unrelated first loads stay concurrent there. std::mutex model_load_mutex_; std::filesystem::path upload_root_; std::filesystem::path repository_root_; @@ -201,7 +202,9 @@ class ServerState final : public IHttpHandler { std::unique_ptr model_installer_; #endif std::atomic next_upload_id_{1}; - // Steady-clock ms of the most recent model load/run; drives idle unload. + // Steady-clock ms of the most recent model load/run completion; drives idle + // unload. Updated at run start and again at completion so a long inference + // does not read as idle the moment it finishes. std::atomic last_activity_ms_{0}; std::atomic idle_unload_shutdown_{false}; std::thread idle_unload_thread_; diff --git a/tests/unittests/test_server_config.cpp b/tests/unittests/test_server_config.cpp index 654f0ad29..6352041ae 100644 --- a/tests/unittests/test_server_config.cpp +++ b/tests/unittests/test_server_config.cpp @@ -468,14 +468,14 @@ void test_min_free_memory_mb_defaults_and_overrides() { const auto default_path = write_config( root, "min_free_default.json", std::string("{") + kMinimalModel + "}"); require( - minitts::server::load_server_config(default_path).min_free_memory_mb == 512, - "min_free_memory_mb defaults to 512 MiB when omitted"); + minitts::server::load_server_config(default_path).min_free_memory_mb == 0, + "min_free_memory_mb defaults to 0 (guard disabled) when omitted"); const auto set_path = write_config( - root, "min_free_set.json", std::string(R"JSON({"min_free_memory_mb": 0,)JSON") + kMinimalModel + "}"); + root, "min_free_set.json", std::string(R"JSON({"min_free_memory_mb": 256,)JSON") + kMinimalModel + "}"); require( - minitts::server::load_server_config(set_path).min_free_memory_mb == 0, - "min_free_memory_mb accepts 0 to disable the headroom"); + minitts::server::load_server_config(set_path).min_free_memory_mb == 256, + "min_free_memory_mb is read from the config to opt into the guard"); } void test_negative_min_free_memory_mb_is_rejected() {