From 1f70f9ae4af16612344901de84a5b454b5bcf9ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E5=BA=86=E4=B8=B0?= Date: Tue, 25 Aug 2026 14:43:53 +0800 Subject: [PATCH 1/5] feat(server): add --idle-unload-ms to auto-unload idle models The 5-minute idle unload used to live in an external Python monitor inside the audio-server wrapper, which polled the server log mtime and called /v1/tasks/unload_all_models. Move it into audiocpp_server itself: - new ServerConfig field idle_unload_ms (default 0 = disabled), parsed from server.json and overridable via --idle-unload-ms - a background thread unloads every resident non-busy model once the server has been idle that long without a model load/run; the next request reloads lazily This drops the log-mtime heuristic (which required --log trace spam) and removes the need for the external Python wrapper. --- app/server/config.cpp | 4 ++++ app/server/config.h | 5 ++++ app/server/main.cpp | 12 ++++++++-- app/server/runtime.cpp | 52 ++++++++++++++++++++++++++++++++++++++++++ app/server/runtime.h | 9 ++++++++ 5 files changed, 80 insertions(+), 2 deletions(-) diff --git a/app/server/config.cpp b/app/server/config.cpp index af219842a..ba52fcd62 100644 --- a/app/server/config.cpp +++ b/app/server/config.cpp @@ -235,6 +235,7 @@ ServerConfig load_server_config(const std::filesystem::path & path) { } config.busy_timeout_ms = engine::io::json::optional_i32(root, "busy_timeout_ms", config.busy_timeout_ms); config.max_loaded_models = engine::io::json::optional_i32(root, "max_loaded_models", config.max_loaded_models); + config.idle_unload_ms = engine::io::json::optional_i32(root, "idle_unload_ms", config.idle_unload_ms); if (const auto * value = root.find("live_ingest")) { config.live_ingest = parse_live_ingest_limits(*value, config.live_ingest, "server live_ingest"); } @@ -256,6 +257,9 @@ ServerConfig load_server_config(const std::filesystem::path & path) { if (config.max_loaded_models < 0) { throw std::runtime_error("server max_loaded_models must be >= 0 (0 disables the limit)"); } + if (config.idle_unload_ms < 0) { + throw std::runtime_error("server idle_unload_ms must be >= 0 (0 disables idle unload)"); + } if (config.threads <= 0) { throw std::runtime_error("server threads must be positive"); } diff --git a/app/server/config.h b/app/server/config.h index 3aee4ead4..ca0a51541 100644 --- a/app/server/config.h +++ b/app/server/config.h @@ -90,6 +90,11 @@ struct ServerConfig { // limit and keeps the original behavior: once loaded, a model stays in memory // until it is unloaded explicitly or the server exits. int max_loaded_models = 0; + // Unload every resident model once the server has been idle this long without + // any model load/run (steady-clock ms). 0 disables idle unload. Complements + // max_loaded_models: that bounds peak residency, this frees memory during + // quiet periods. The next request reloads lazily. + int idle_unload_ms = 0; // Fleet-wide bounds for incrementally delivered request bodies. The defaults are // in LiveIngestLimits; a model entry may override any subset of them. LiveIngestLimits live_ingest; diff --git a/app/server/main.cpp b/app/server/main.cpp index d3e6fdd9f..11335111d 100644 --- a/app/server/main.cpp +++ b/app/server/main.cpp @@ -62,7 +62,7 @@ void print_help() { std::cout << "audiocpp_server [--config ] [--ui] [--host ] [--port ] [--backend ]\n" << " [--device ] [--list-devices] [--threads ] [--busy-timeout-ms ]\n" - << " [--max-loaded-models ]\n" + << " [--max-loaded-models ] [--idle-unload-ms ]\n" << " [--model-spec-override ] [--voice-dir ]\n" << " [--log] [--log-file ]\n" << " [--cors-origins ]\n" @@ -76,7 +76,9 @@ void print_help() { << " busy this long; default 300000, 0 disables\n" << " --max-loaded-models keep at most n models resident in memory, unloading\n" << " the least recently used idle model first; 1 enforces\n" - << " a single loaded model, default 0 (no limit)\n" + << " a single loaded model, default 0 (no limit)\n" << " --idle-unload-ms unload all resident models after this many ms without\n" + << " any model load/run; default 0 (disabled), next request\n" + << " reloads lazily\n" << " --voice-dir override the shared reference voice library directory\n" << " --cors-origins \"*\" experimental; disabled by default. Allows browser\n" << " requests from any origin for trusted local demos only\n" @@ -188,6 +190,9 @@ int main(int argc, char ** argv) { if (const auto max_loaded_models = arg_value(argc, argv, "--max-loaded-models")) { config.max_loaded_models = std::stoi(*max_loaded_models); } + if (const auto idle_unload_ms = arg_value(argc, argv, "--idle-unload-ms")) { + config.idle_unload_ms = std::stoi(*idle_unload_ms); + } if (const auto model_spec = arg_value(argc, argv, "--model-spec-override")) { config.model_spec_override = std::filesystem::path(*model_spec); } @@ -206,6 +211,9 @@ int main(int argc, char ** argv) { if (config.max_loaded_models < 0) { throw std::runtime_error("--max-loaded-models must be >= 0 (0 disables the limit)"); } + if (config.idle_unload_ms < 0) { + throw std::runtime_error("--idle-unload-ms must be >= 0 (0 disables idle unload)"); + } const auto ui_resource_anchor = executable_directory(argc > 0 ? argv[0] : nullptr); minitts::server::ServerState state( diff --git a/app/server/runtime.cpp b/app/server/runtime.cpp index 9ec08011a..2e895140d 100644 --- a/app/server/runtime.cpp +++ b/app/server/runtime.cpp @@ -29,6 +29,7 @@ #include #include #include +#include #include #include @@ -968,9 +969,16 @@ ServerState::ServerState( } #endif load_models(); + if (config_.idle_unload_ms > 0) { + idle_unload_thread_ = std::thread(&ServerState::idle_unload_loop, this); + } } ServerState::~ServerState() { + idle_unload_shutdown_.store(true, std::memory_order_relaxed); + if (idle_unload_thread_.joinable()) { + idle_unload_thread_.join(); + } if (!upload_root_.empty()) { std::error_code ec; std::filesystem::remove_all(upload_root_, ec); @@ -1626,6 +1634,7 @@ void ServerState::evict_for_model_limit(const LoadedModel & loading) { } void ServerState::ensure_model_loaded_locked(LoadedModel & model) { + last_activity_ms_.store(steady_now_ms(), std::memory_order_relaxed); model.last_used_ms.store(steady_now_ms(), std::memory_order_relaxed); if (model.session != nullptr) { return; @@ -2747,6 +2756,49 @@ void ServerState::LoadedModel::unload() { loaded.store(false); } +void ServerState::idle_unload_loop() { + const auto interval_ms = std::max(1000, config_.idle_unload_ms / 10); + while (!idle_unload_shutdown_.load(std::memory_order_relaxed)) { + std::this_thread::sleep_for(std::chrono::milliseconds(interval_ms)); + if (idle_unload_shutdown_.load(std::memory_order_relaxed)) { + break; + } + const auto idle_ms = steady_now_ms() - last_activity_ms_.load(std::memory_order_relaxed); + if (idle_ms >= config_.idle_unload_ms) { + unload_idle_models(); + } + } +} + +void ServerState::unload_idle_models() { + std::vector resident; + { + std::lock_guard state_lock(models_mutex_); + for (const auto & model : models_) { + if (model->session != nullptr) { + resident.push_back(model.get()); + } + } + } + int unloaded = 0; + for (LoadedModel * model : resident) { + // Never unload a model mid-inference; a busy model keeps its slot and the + // next idle pass retries it. + const auto lock = model->busy.try_acquire(); + if (!lock.has_value()) { + continue; + } + model->unload(); + ++unloaded; + } + if (unloaded > 0) { + const auto idle_ms = steady_now_ms() - last_activity_ms_.load(std::memory_order_relaxed); + std::cerr << "[server] idle " << idle_ms << " ms: unloaded " << unloaded << " model(s)\n"; + // Restart the idle clock so we do not spin on unload attempts every interval. + last_activity_ms_.store(steady_now_ms(), std::memory_order_relaxed); + } +} + HttpResponse ServerState::handle_unload_models(const std::string & body_text) { const auto body = engine::io::json::parse(body_text); const auto * ids = body.find("model_ids"); diff --git a/app/server/runtime.h b/app/server/runtime.h index 45af1c555..3cfb6e0e7 100644 --- a/app/server/runtime.h +++ b/app/server/runtime.h @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -168,6 +169,10 @@ class ServerState final : public IHttpHandler { HttpResponse handle_voices(const HttpRequest & request) const; HttpResponse handle_unload_models(const std::string & body_text); HttpResponse handle_unload_all_models(); + // Background loop started when idle_unload_ms > 0: when the server has gone + // that long without a model load/run, unloads every resident (non-busy) model. + void idle_unload_loop(); + void unload_idle_models(); std::string models_json(bool include_session_options = false) const; std::string get_allowed_origin(const HttpRequest & request) const; @@ -189,6 +194,10 @@ class ServerState final : public IHttpHandler { std::unique_ptr model_installer_; #endif std::atomic next_upload_id_{1}; + // Steady-clock ms of the most recent model load/run; drives idle unload. + std::atomic last_activity_ms_{0}; + std::atomic idle_unload_shutdown_{false}; + std::thread idle_unload_thread_; }; } // namespace minitts::server From 454f099315f9df7a4873c3e8745adccd1cdf13be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E5=BA=86=E4=B8=B0?= Date: Tue, 25 Aug 2026 14:52:24 +0800 Subject: [PATCH 2/5] feat(server): refuse model load when host/GPU memory is insufficient Before every lazy model load, estimate the model's resident footprint (weights plus runtime overhead) and compare against free host memory and, for GPU backends, the backend device's free memory. Refuse the load with HTTP 503 insufficient_memory when estimate + configured headroom does not fit, instead of exhausting the machine (the previous failure mode was kIOGPUCommandBufferCallbackErrorOutOfMemory after models accumulated on a 16GB Mac). - add ServerConfig.min_free_memory_mb (default 512 MiB headroom), parsed from server.json and overridable via --min-free-memory-mb - add a macOS implementation of available_host_memory_bytes() using Mach VM stats (free + inactive + purgeable pages); Linux/Windows were already covered - add InsufficientMemoryError, mapped to 503 insufficient_memory --- app/server/busy_guard.h | 8 ++++ app/server/config.cpp | 4 ++ app/server/config.h | 6 +++ app/server/main.cpp | 12 +++++- app/server/runtime.cpp | 62 ++++++++++++++++++++++++++++++ app/server/runtime.h | 7 ++++ src/framework/core/host_memory.cpp | 33 +++++++++++++++- 7 files changed, 128 insertions(+), 4 deletions(-) diff --git a/app/server/busy_guard.h b/app/server/busy_guard.h index 2c130ff6d..80c6e0f6b 100644 --- a/app/server/busy_guard.h +++ b/app/server/busy_guard.h @@ -18,6 +18,14 @@ class ServerBusyError : public std::runtime_error { using std::runtime_error::runtime_error; }; +// The host/GPU does not currently have enough free memory to load a model. +// Mapped to HTTP 503: a transient server condition the caller may retry after +// other models have been evicted or the system has freed memory. +class InsufficientMemoryError : public std::runtime_error { +public: + explicit InsufficientMemoryError(const std::string & message) : std::runtime_error(message) {} +}; + inline std::int64_t steady_now_ms() { return std::chrono::duration_cast( std::chrono::steady_clock::now().time_since_epoch()) diff --git a/app/server/config.cpp b/app/server/config.cpp index ba52fcd62..ed98865c5 100644 --- a/app/server/config.cpp +++ b/app/server/config.cpp @@ -236,6 +236,7 @@ ServerConfig load_server_config(const std::filesystem::path & path) { config.busy_timeout_ms = engine::io::json::optional_i32(root, "busy_timeout_ms", config.busy_timeout_ms); config.max_loaded_models = engine::io::json::optional_i32(root, "max_loaded_models", config.max_loaded_models); config.idle_unload_ms = engine::io::json::optional_i32(root, "idle_unload_ms", config.idle_unload_ms); + config.min_free_memory_mb = engine::io::json::optional_i32(root, "min_free_memory_mb", config.min_free_memory_mb); if (const auto * value = root.find("live_ingest")) { config.live_ingest = parse_live_ingest_limits(*value, config.live_ingest, "server live_ingest"); } @@ -260,6 +261,9 @@ ServerConfig load_server_config(const std::filesystem::path & path) { if (config.idle_unload_ms < 0) { throw std::runtime_error("server idle_unload_ms must be >= 0 (0 disables idle unload)"); } + if (config.min_free_memory_mb < 0) { + throw std::runtime_error("server min_free_memory_mb must be >= 0 (0 disables the headroom)"); + } if (config.threads <= 0) { throw std::runtime_error("server threads must be positive"); } diff --git a/app/server/config.h b/app/server/config.h index ca0a51541..9c4ac00e7 100644 --- a/app/server/config.h +++ b/app/server/config.h @@ -95,6 +95,12 @@ struct ServerConfig { // max_loaded_models: that bounds peak residency, this frees memory during // quiet periods. The next request reloads lazily. int idle_unload_ms = 0; + // Minimum free memory (host and GPU, each) the server must retain after + // loading a model, in MiB. Before every lazy load the server estimates the + // model's resident footprint from its weights and refuses to load when + // estimate + this headroom would not fit. 0 disables the extra headroom + // (the estimate must still fit in what is free). + int min_free_memory_mb = 512; // Fleet-wide bounds for incrementally delivered request bodies. The defaults are // in LiveIngestLimits; a model entry may override any subset of them. LiveIngestLimits live_ingest; diff --git a/app/server/main.cpp b/app/server/main.cpp index 11335111d..46eb85d98 100644 --- a/app/server/main.cpp +++ b/app/server/main.cpp @@ -62,7 +62,7 @@ void print_help() { std::cout << "audiocpp_server [--config ] [--ui] [--host ] [--port ] [--backend ]\n" << " [--device ] [--list-devices] [--threads ] [--busy-timeout-ms ]\n" - << " [--max-loaded-models ] [--idle-unload-ms ]\n" + << " [--max-loaded-models ] [--idle-unload-ms ] [--min-free-memory-mb ]\n" << " [--model-spec-override ] [--voice-dir ]\n" << " [--log] [--log-file ]\n" << " [--cors-origins ]\n" @@ -78,7 +78,9 @@ void print_help() { << " the least recently used idle model first; 1 enforces\n" << " a single loaded model, default 0 (no limit)\n" << " --idle-unload-ms unload all resident models after this many ms without\n" << " any model load/run; default 0 (disabled), next request\n" - << " reloads lazily\n" + << " reloads lazily\n" << " --min-free-memory-mb refuse a model load unless host and GPU each keep at\n" + << " least this many MiB free after the load; default 1024,\n" + << " 0 disables the extra headroom\n" << " --voice-dir override the shared reference voice library directory\n" << " --cors-origins \"*\" experimental; disabled by default. Allows browser\n" << " requests from any origin for trusted local demos only\n" @@ -193,6 +195,9 @@ int main(int argc, char ** argv) { if (const auto idle_unload_ms = arg_value(argc, argv, "--idle-unload-ms")) { config.idle_unload_ms = std::stoi(*idle_unload_ms); } + if (const auto min_free_memory_mb = arg_value(argc, argv, "--min-free-memory-mb")) { + config.min_free_memory_mb = std::stoi(*min_free_memory_mb); + } if (const auto model_spec = arg_value(argc, argv, "--model-spec-override")) { config.model_spec_override = std::filesystem::path(*model_spec); } @@ -214,6 +219,9 @@ int main(int argc, char ** argv) { if (config.idle_unload_ms < 0) { throw std::runtime_error("--idle-unload-ms must be >= 0 (0 disables idle unload)"); } + if (config.min_free_memory_mb < 0) { + throw std::runtime_error("--min-free-memory-mb must be >= 0 (0 disables the headroom)"); + } const auto ui_resource_anchor = executable_directory(argc > 0 ? argv[0] : nullptr); minitts::server::ServerState state( diff --git a/app/server/runtime.cpp b/app/server/runtime.cpp index 2e895140d..50536a038 100644 --- a/app/server/runtime.cpp +++ b/app/server/runtime.cpp @@ -8,6 +8,7 @@ #include "../streaming/pcm_source.h" #include "../streaming/streaming.h" +#include "engine/framework/core/host_memory.h" #include "engine/framework/debug/trace.h" #include "engine/framework/io/json.h" #include "engine/framework/model_spec/metadata.h" @@ -22,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -1107,6 +1109,8 @@ HttpResponse ServerState::handle(const HttpRequest & request) { // there. Checked before ServerBusyError only because both are // runtime_error; the two conditions are disjoint. response = error_response(400, ex.what(), "invalid_request_error"); + } catch (const InsufficientMemoryError & ex) { + response = error_response(503, ex.what(), "insufficient_memory"); } catch (const ServerBusyError & ex) { // Non-streaming requests surface the busy state as 503 before any response is // sent. (Streaming requests acquire the lock inside the stream body, after @@ -1644,6 +1648,7 @@ void ServerState::ensure_model_loaded_locked(LoadedModel & model) { load_lock = std::unique_lock(model_load_mutex_); evict_for_model_limit(model); } + ensure_model_fits_memory(model.config); auto registry = engine::runtime::make_default_registry(); engine::runtime::ModelLoadRequest load_request; @@ -2799,6 +2804,63 @@ void ServerState::unload_idle_models() { } } +namespace { +std::string format_bytes(size_t bytes) { + const double gib = static_cast(bytes) / (1024.0 * 1024.0 * 1024.0); + std::ostringstream out; + out << std::fixed << std::setprecision(2) << gib << " GiB"; + return out.str(); +} +} // namespace + +size_t ServerState::estimate_model_memory_bytes(const ServerModelConfig & model) const { + size_t weights = 0; + const auto add_file = [&weights](const std::filesystem::path & path) { + std::error_code ec; + if (std::filesystem::is_regular_file(path, ec)) { + weights += static_cast(std::filesystem::file_size(path, ec)); + } + }; + add_file(model.path); + for (const auto & [key, value] : model.session_options) { + (void)key; + std::filesystem::path aux(value); + if (aux.is_relative()) { + aux = model.path.parent_path() / aux; + } + add_file(aux); + } + // Weights plus a runtime factor for Metal/GPU buffers, activation graphs and + // KV state, plus a fixed floor for per-model bookkeeping. + constexpr double kRuntimeOverheadFactor = 1.5; + constexpr size_t kFixedOverhead = 128ull * 1024 * 1024; + return static_cast(static_cast(weights) * kRuntimeOverheadFactor) + kFixedOverhead; +} + +void ServerState::ensure_model_fits_memory(const ServerModelConfig & model) { + const size_t estimate = estimate_model_memory_bytes(model); + const size_t headroom = static_cast(config_.min_free_memory_mb) * 1024ull * 1024ull; + + const size_t host_available = engine::core::available_host_memory_bytes(); + if (host_available > 0 && estimate + headroom > host_available) { + throw InsufficientMemoryError( + "cannot load model '" + model.id + "': estimated " + format_bytes(estimate) + + " + " + std::to_string(config_.min_free_memory_mb) + " MiB headroom exceeds available host memory (" + + format_bytes(host_available) + ")"); + } + + const engine::core::BackendMemorySnapshot device = + engine::core::query_backend_memory(engine::core::BackendConfig{ + config_.backend, config_.device, config_.threads}); + if (device.available && estimate + headroom > static_cast(device.free_bytes)) { + throw InsufficientMemoryError( + "cannot load model '" + model.id + "': estimated " + format_bytes(estimate) + + " + " + std::to_string(config_.min_free_memory_mb) + " MiB headroom exceeds available " + + backend_name(config_.backend) + " memory (" + + format_bytes(static_cast(device.free_bytes)) + ")"); + } +} + HttpResponse ServerState::handle_unload_models(const std::string & body_text) { const auto body = engine::io::json::parse(body_text); const auto * ids = body.find("model_ids"); diff --git a/app/server/runtime.h b/app/server/runtime.h index 3cfb6e0e7..2f42e97f9 100644 --- a/app/server/runtime.h +++ b/app/server/runtime.h @@ -108,6 +108,13 @@ class ServerState final : public IHttpHandler { // `loading` fits within the limit. A model mid-inference is never a victim; // when nothing can be evicted this throws ServerBusyError (-> HTTP 503). void evict_for_model_limit(const LoadedModel & loading); + // Estimated resident bytes this model will occupy once loaded (weights plus + // a runtime overhead factor for GPU buffers / compute graphs). + size_t estimate_model_memory_bytes(const ServerModelConfig & model) const; + // Refuse the load with InsufficientMemoryError (-> HTTP 503) when the + // estimated footprint plus configured headroom does not fit the free host + // memory and (for GPU backends) the backend device memory. + void ensure_model_fits_memory(const ServerModelConfig & model); LoadedModel & require_model(const engine::io::json::Value & body); const LoadedModel::RuntimeVoicePreset * select_voice_preset( const LoadedModel & model, diff --git a/src/framework/core/host_memory.cpp b/src/framework/core/host_memory.cpp index 377857c74..d72552962 100644 --- a/src/framework/core/host_memory.cpp +++ b/src/framework/core/host_memory.cpp @@ -5,7 +5,10 @@ #elif defined(__linux__) #include #include -#elif defined(__unix__) || defined(__APPLE__) +#elif defined(__APPLE__) +#include +#include +#elif defined(__unix__) #include #endif @@ -36,6 +39,28 @@ size_t mem_available_bytes() { return bytes; } +} // namespace +#elif defined(__APPLE__) +namespace { + +// Mach VM stats: pages that a fresh allocation can claim without swapping. +// free_count is the idle pool; inactive_count is reclaimable file cache / +// anonymous memory; purgeable_count can be dropped on demand. Compressed +// pages are deliberately excluded -- dropping them would force decompression +// and they are already backing live data. +size_t mem_available_bytes() { + mach_msg_type_number_t count = HOST_VM_INFO64_COUNT; + vm_statistics64_data_t vm_stat{}; + if (host_statistics64(mach_host_self(), HOST_VM_INFO64, reinterpret_cast(&vm_stat), &count) != KERN_SUCCESS) { + return 0; + } + const uint64_t page = static_cast(vm_page_size); + return static_cast( + (static_cast(vm_stat.free_count) + + static_cast(vm_stat.inactive_count) + + static_cast(vm_stat.purgeable_count)) * page); +} + } // namespace #endif @@ -50,6 +75,10 @@ size_t available_host_memory_bytes() { if (const size_t bytes = mem_available_bytes(); bytes > 0) { return bytes; } +#elif defined(__APPLE__) + if (const size_t bytes = mem_available_bytes(); bytes > 0) { + return bytes; + } #elif defined(_SC_AVPHYS_PAGES) && defined(_SC_PAGE_SIZE) const long pages = sysconf(_SC_AVPHYS_PAGES); const long page_size = sysconf(_SC_PAGE_SIZE); @@ -57,7 +86,7 @@ size_t available_host_memory_bytes() { return static_cast(pages) * static_cast(page_size); } #endif - // macOS has no _SC_AVPHYS_PAGES, and any query can fail: report unknown. + // A query can fail on any platform: report unknown (0). return 0; } From d345331e7b27f1c4c3306f78a757c414b6047e17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E5=BA=86=E4=B8=B0?= Date: Tue, 25 Aug 2026 15:02:18 +0800 Subject: [PATCH 3/5] fix(server): address code review for idle unload and memory guard Independent review found no criticals; fix the actionable findings: - --min-free-memory-mb help text now matches the actual 512 MiB default - estimate_model_memory_bytes() sums directory-style model trees (with depth/file limits) instead of counting only regular files, so directory models are no longer estimated as 0 - the model load path is serialized even when max_loaded_models is 0, so concurrent lazy loads cannot both pass the memory pre-check - expose engine::core::ensure_backends_loaded() and call it before the GPU memory query so the very first load actually runs the device check - the idle-unload thread now wakes on shutdown in <=250ms slices instead of waiting out a full poll interval - document idle_unload_ms / min_free_memory_mb in app/server/README.md and example.json - add server_config_test coverage for the new fields (defaults, overrides, negative rejection) --- app/server/README.md | 4 ++ app/server/example.json | 3 ++ app/server/main.cpp | 2 +- app/server/runtime.cpp | 64 ++++++++++++++++++++----- include/engine/framework/core/backend.h | 5 ++ src/framework/core/backend.cpp | 4 +- src/framework/core/host_memory.cpp | 2 +- tests/unittests/test_server_config.cpp | 64 +++++++++++++++++++++++++ 8 files changed, 131 insertions(+), 17 deletions(-) diff --git a/app/server/README.md b/app/server/README.md index 0361a7e7d..caa0845f1 100644 --- a/app/server/README.md +++ b/app/server/README.md @@ -97,6 +97,10 @@ Set top-level `"lazy_load": true` to register all configured model ids at startu Set top-level `"max_loaded_models"` to bound how many models are resident in memory at once. When a request needs a model that is not loaded and the limit is already reached, the server first unloads the least recently used idle model (freeing VRAM on GPU backends) and reloads it on its own next request. `1` enforces a single loaded model at a time, which is the practical choice when each model alone nearly fills the device. Higher values keep that many most recently used models warm. The default `0` disables the limit. A model that is mid-inference is never unloaded; if the limit is reached and every loaded model is busy, the request fails with `503` so the client can retry. With more non-lazy models configured than the limit allows, startup loads the first `max_loaded_models` of them and defers the rest to their first request. The equivalent command-line option is `--max-loaded-models `. +Set top-level `"idle_unload_ms"` to have the server unload every resident model after it has gone that long without any model load/run. The next request reloads lazily; a model mid-inference is never unloaded. This complements `max_loaded_models`: that bounds peak residency, this frees memory during quiet periods. Defaults to `0` (disabled). The `--idle-unload-ms ` command-line flag overrides the config value. + +Set top-level `"min_free_memory_mb"` to refuse a model load when the host or the GPU backend does not have that much free memory left after the estimated footprint of the new model. The estimate is the model's weights (including session auxiliary files; directory trees are summed with depth/file limits) times a runtime overhead factor plus a fixed floor. When the check fails, the request returns HTTP 503 with `insufficient_memory`; the client may retry later. Defaults to `512`; `0` disables the extra headroom (the estimate must still fit in what is free). The `--min-free-memory-mb ` command-line flag overrides the config value. + Set per-model `"default_request_options"` to apply request-option defaults to every request for that model. Values supplied by the actual request body override these defaults. Set top-level `"max_request_body_bytes"` to bound the largest HTTP request body buffered in host RAM before routing. This protects endpoints that accept JSON or audio uploads from unbounded `Content-Length` claims. The default is `2147483648` bytes (2 GiB). Raise or lower it to match the largest upload your deployment intends to accept. Values above `2^53 - 1` are rejected because this config parser stores JSON numbers as doubles. diff --git a/app/server/example.json b/app/server/example.json index 043c30edf..7b10327f0 100644 --- a/app/server/example.json +++ b/app/server/example.json @@ -5,6 +5,9 @@ "device": 0, "threads": 1, "lazy_load": true, + "max_loaded_models": 0, + "idle_unload_ms": 0, + "min_free_memory_mb": 512, "models": [ { "id": "pocket-tts", diff --git a/app/server/main.cpp b/app/server/main.cpp index 46eb85d98..100ae2453 100644 --- a/app/server/main.cpp +++ b/app/server/main.cpp @@ -79,7 +79,7 @@ void print_help() { << " a single loaded model, default 0 (no limit)\n" << " --idle-unload-ms unload all resident models after this many ms without\n" << " any model load/run; default 0 (disabled), next request\n" << " reloads lazily\n" << " --min-free-memory-mb refuse a model load unless host and GPU each keep at\n" - << " least this many MiB free after the load; default 1024,\n" + << " least this many MiB free after the load; default 512,\n" << " 0 disables the extra headroom\n" << " --voice-dir override the shared reference voice library directory\n" << " --cors-origins \"*\" experimental; disabled by default. Allows browser\n" diff --git a/app/server/runtime.cpp b/app/server/runtime.cpp index 50536a038..42e5505cd 100644 --- a/app/server/runtime.cpp +++ b/app/server/runtime.cpp @@ -1643,10 +1643,16 @@ void ServerState::ensure_model_loaded_locked(LoadedModel & model) { if (model.session != nullptr) { return; } - std::unique_lock load_lock; + // Serialize the whole "evict -> memory check -> load" sequence even when + // max_loaded_models is 0 (no eviction): two concurrent lazy loads could + // otherwise both pass the memory pre-check before either allocates, and the + // guard would be meaningless. A model mid-inference is never a victim. + std::unique_lock load_lock(model_load_mutex_); if (config_.max_loaded_models > 0) { - load_lock = std::unique_lock(model_load_mutex_); evict_for_model_limit(model); + // Note: if ensure_model_fits_memory() below then refuses the load, the + // evicted model is already unloaded (freed memory is never restored). + // That is acceptable: the 503 tells the client to retry later. } ensure_model_fits_memory(model.config); auto registry = engine::runtime::make_default_registry(); @@ -2763,14 +2769,22 @@ void ServerState::LoadedModel::unload() { void ServerState::idle_unload_loop() { const auto interval_ms = std::max(1000, config_.idle_unload_ms / 10); + auto deadline = steady_now_ms() + interval_ms; while (!idle_unload_shutdown_.load(std::memory_order_relaxed)) { - std::this_thread::sleep_for(std::chrono::milliseconds(interval_ms)); - if (idle_unload_shutdown_.load(std::memory_order_relaxed)) { - break; - } - const auto idle_ms = steady_now_ms() - last_activity_ms_.load(std::memory_order_relaxed); - if (idle_ms >= config_.idle_unload_ms) { - unload_idle_models(); + // Sleep in small slices so SIGTERM (destructor join) never waits out a + // full poll interval; the deadline keeps the idle check cadence intact. + const auto now = steady_now_ms(); + if (now >= deadline) { + if (idle_unload_shutdown_.load(std::memory_order_relaxed)) { + break; + } + const auto idle_ms = now - last_activity_ms_.load(std::memory_order_relaxed); + if (idle_ms >= config_.idle_unload_ms) { + unload_idle_models(); + } + deadline = steady_now_ms() + interval_ms; + } else { + std::this_thread::sleep_for(std::chrono::milliseconds(250)); } } } @@ -2815,20 +2829,39 @@ std::string format_bytes(size_t bytes) { size_t ServerState::estimate_model_memory_bytes(const ServerModelConfig & model) const { size_t weights = 0; - const auto add_file = [&weights](const std::filesystem::path & path) { - std::error_code ec; + std::error_code ec; + // Directories (model_spec / HF-style checkpoints) are summed recursively with + // hard limits so a pathological tree cannot stall the load path or blow the + // counter; anything beyond the limits contributes 0 (the fixed floor below + // still applies, so the estimate never reads as completely empty). + constexpr size_t kMaxDepth = 3; + constexpr size_t kMaxFiles = 10000; + size_t visited_files = 0; + const auto add_file = [&](const std::filesystem::path & path) { if (std::filesystem::is_regular_file(path, ec)) { weights += static_cast(std::filesystem::file_size(path, ec)); + ++visited_files; } }; - add_file(model.path); + const std::function add_tree = + [&](const std::filesystem::path & path, size_t depth) { + if (std::filesystem::is_regular_file(path, ec)) { + add_file(path); + } else if (std::filesystem::is_directory(path, ec) && depth < kMaxDepth) { + std::filesystem::directory_iterator it(path, ec), end; + for (; it != end && visited_files < kMaxFiles; it.increment(ec)) { + add_tree(it->path(), depth + 1); + } + } + }; + add_tree(model.path, 0); for (const auto & [key, value] : model.session_options) { (void)key; std::filesystem::path aux(value); if (aux.is_relative()) { aux = model.path.parent_path() / aux; } - add_file(aux); + add_tree(aux, 0); } // Weights plus a runtime factor for Metal/GPU buffers, activation graphs and // KV state, plus a fixed floor for per-model bookkeeping. @@ -2849,6 +2882,11 @@ void ServerState::ensure_model_fits_memory(const ServerModelConfig & model) { format_bytes(host_available) + ")"); } + // ggml backend registries may not be loaded yet on the very first request + // (init_backend happens inside a session load, after this pre-check), which + // would make query_backend_memory() report "unknown" and silently skip the + // GPU check. Loading them here is idempotent and cheap once done. + engine::core::ensure_backends_loaded(); const engine::core::BackendMemorySnapshot device = engine::core::query_backend_memory(engine::core::BackendConfig{ config_.backend, config_.device, config_.threads}); diff --git a/include/engine/framework/core/backend.h b/include/engine/framework/core/backend.h index d56637e87..5c2cb9914 100644 --- a/include/engine/framework/core/backend.h +++ b/include/engine/framework/core/backend.h @@ -32,6 +32,11 @@ struct BackendDeviceInfo { std::vector list_backend_devices(); void print_backend_devices(std::ostream & out); +// Load every registered ggml backend registry (idempotent). Needed before +// query_backend_memory() can report GPU memory on a process that has not yet +// initialized a backend. +void ensure_backends_loaded(); + struct BackendMemorySnapshot { bool available = false; int64_t total_bytes = 0; diff --git a/src/framework/core/backend.cpp b/src/framework/core/backend.cpp index afb5489ed..902b73120 100644 --- a/src/framework/core/backend.cpp +++ b/src/framework/core/backend.cpp @@ -10,14 +10,14 @@ namespace engine::core { -namespace { - void ensure_backends_loaded() { if (ggml_backend_reg_count() == 0) { ggml_backend_load_all(); } } +namespace { + // A backend is identified by the name of the ggml registry that owns it. The device type // (GPU/IGPU/ACCEL) deliberately plays no part in that: Metal reports GPU rather than ACCEL, // Vulkan reports IGPU on integrated GPUs, and those values are free to change upstream. diff --git a/src/framework/core/host_memory.cpp b/src/framework/core/host_memory.cpp index d72552962..dd8694cf1 100644 --- a/src/framework/core/host_memory.cpp +++ b/src/framework/core/host_memory.cpp @@ -6,8 +6,8 @@ #include #include #elif defined(__APPLE__) +#include #include -#include #elif defined(__unix__) #include #endif diff --git a/tests/unittests/test_server_config.cpp b/tests/unittests/test_server_config.cpp index 8c830802c..654f0ad29 100644 --- a/tests/unittests/test_server_config.cpp +++ b/tests/unittests/test_server_config.cpp @@ -432,6 +432,66 @@ void test_request_timeout_is_clamped_to_policy() { "unbounded on both sides stays unbounded"); } +void test_idle_unload_ms_defaults_and_overrides() { + const auto root = make_temp_root(); + + const auto default_path = write_config( + root, "idle_unload_default.json", std::string("{") + kMinimalModel + "}"); + require( + minitts::server::load_server_config(default_path).idle_unload_ms == 0, + "idle_unload_ms defaults to 0 (disabled) when omitted"); + + const auto set_path = write_config( + root, "idle_unload_set.json", std::string(R"JSON({"idle_unload_ms": 300000,)JSON") + kMinimalModel + "}"); + require( + minitts::server::load_server_config(set_path).idle_unload_ms == 300000, + "idle_unload_ms is read from the config"); +} + +void test_negative_idle_unload_ms_is_rejected() { + const auto root = make_temp_root(); + const auto config_path = write_config( + root, "idle_unload_negative.json", std::string(R"JSON({"idle_unload_ms": -1,)JSON") + kMinimalModel + "}"); + + bool rejected = false; + try { + (void) minitts::server::load_server_config(config_path); + } catch (const std::runtime_error & error) { + rejected = std::string(error.what()).find("idle_unload_ms") != std::string::npos; + } + require(rejected, "negative idle_unload_ms is rejected"); +} + +void test_min_free_memory_mb_defaults_and_overrides() { + const auto root = make_temp_root(); + + const auto default_path = write_config( + root, "min_free_default.json", std::string("{") + kMinimalModel + "}"); + require( + minitts::server::load_server_config(default_path).min_free_memory_mb == 512, + "min_free_memory_mb defaults to 512 MiB when omitted"); + + const auto set_path = write_config( + root, "min_free_set.json", std::string(R"JSON({"min_free_memory_mb": 0,)JSON") + kMinimalModel + "}"); + require( + minitts::server::load_server_config(set_path).min_free_memory_mb == 0, + "min_free_memory_mb accepts 0 to disable the headroom"); +} + +void test_negative_min_free_memory_mb_is_rejected() { + const auto root = make_temp_root(); + const auto config_path = write_config( + root, "min_free_negative.json", std::string(R"JSON({"min_free_memory_mb": -1,)JSON") + kMinimalModel + "}"); + + bool rejected = false; + try { + (void) minitts::server::load_server_config(config_path); + } catch (const std::runtime_error & error) { + rejected = std::string(error.what()).find("min_free_memory_mb") != std::string::npos; + } + require(rejected, "negative min_free_memory_mb is rejected"); +} + void test_model_run_overrun_predicate() { using minitts::server::model_run_has_overrun; @@ -459,6 +519,10 @@ int main() { test_negative_busy_timeout_is_rejected(); test_max_loaded_models_defaults_and_overrides(); test_negative_max_loaded_models_is_rejected(); + test_idle_unload_ms_defaults_and_overrides(); + test_negative_idle_unload_ms_is_rejected(); + test_min_free_memory_mb_defaults_and_overrides(); + test_negative_min_free_memory_mb_is_rejected(); test_per_model_busy_timeout(); test_negative_per_model_busy_timeout_is_rejected(); test_ui_configuration(); From 5a0c1c44c16297cd6def376199ee45ecbcb08ce5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E5=BA=86=E4=B8=B0?= Date: Tue, 25 Aug 2026 15:11:16 +0800 Subject: [PATCH 4/5] ci(server): add cross-platform build+test matrix for memory guard Run audiocpp_server build + server_config_test on ubuntu/windows/macos to prove the idle-unload and pre-load memory-check changes compile and behave on all three desktop platforms. GPU backends are off here (the changes query memory through the backend-agnostic ggml_backend_dev_memory; existing workflows already cover CUDA/Vulkan/Metal builds). --- .github/workflows/server-memory-guard.yml | 80 +++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 .github/workflows/server-memory-guard.yml diff --git a/.github/workflows/server-memory-guard.yml b/.github/workflows/server-memory-guard.yml new file mode 100644 index 000000000..3b16bd8ad --- /dev/null +++ b/.github/workflows/server-memory-guard.yml @@ -0,0 +1,80 @@ +name: server-memory-guard cross-platform + +# Verifies the server memory-management changes (--idle-unload-ms and +# --min-free-memory-mb) compile and the config unit tests pass on all three +# desktop platforms. GPU backends are disabled here: the changes themselves are +# backend-agnostic (they query memory through ggml_backend_dev_memory), and the +# existing linux/mac/windows workflows already cover CUDA/Vulkan/Metal builds. + +on: + workflow_dispatch: + pull_request: + paths: + - "app/server/**" + - "src/framework/core/host_memory.cpp" + - "include/engine/framework/core/backend.h" + - "src/framework/core/backend.cpp" + - "tests/unittests/test_server_config.cpp" + - ".github/workflows/server-memory-guard.yml" + +jobs: + build-test: + name: ${{ matrix.os }} build+test + runs-on: ${{ matrix.os }} + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, windows-2022, macos-14] + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Configure (Linux/macOS) + if: runner.os != 'Windows' + run: | + cmake -S . -B build \ + -DCMAKE_BUILD_TYPE=Release \ + -DENGINE_BUILD_TESTS=ON \ + -DENGINE_ENABLE_OPENMP=OFF \ + -DENGINE_ENABLE_CUDA=OFF \ + -DENGINE_ENABLE_VULKAN=OFF \ + -DENGINE_ENABLE_METAL=OFF + + - name: Configure (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: | + cmake -S . -B build ` + -DCMAKE_BUILD_TYPE=Release ` + -DENGINE_BUILD_TESTS=ON ` + -DENGINE_ENABLE_OPENMP=OFF ` + -DENGINE_ENABLE_CUDA=OFF ` + -DENGINE_ENABLE_VULKAN=OFF ` + -DENGINE_ENABLE_METAL=OFF + + - name: Build (Linux/macOS) + if: runner.os != 'Windows' + run: | + cmake --build build --target audiocpp_server server_config_test --parallel "$(nproc)" + + - name: Build (macOS nproc fallback) + if: runner.os == 'macOS' + run: | + cmake --build build --target audiocpp_server server_config_test --parallel "$(sysctl -n hw.logicalcpu)" + + - name: Build (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: | + cmake --build build --config Release --target audiocpp_server server_config_test --parallel + + - name: Run server_config_test (Linux/macOS) + if: runner.os != 'Windows' + run: ./build/bin/server_config_test + + - name: Run server_config_test (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: .\build\bin\Release\server_config_test.exe From 8d9d677d0c25d68341c5a7a880af0d91696d3057 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E5=BA=86=E4=B8=B0?= Date: Tue, 25 Aug 2026 15:44:51 +0800 Subject: [PATCH 5/5] ci(server): fix macOS build step to not run the nproc variant The Linux build step used nproc and its condition also matched macOS, so macOS ran both build steps and the nproc one stalled. Scope the nproc step to Linux and let macOS use sysctl -n hw.logicalcpu. --- .github/workflows/server-memory-guard.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/server-memory-guard.yml b/.github/workflows/server-memory-guard.yml index 3b16bd8ad..57f5829f2 100644 --- a/.github/workflows/server-memory-guard.yml +++ b/.github/workflows/server-memory-guard.yml @@ -54,12 +54,12 @@ jobs: -DENGINE_ENABLE_VULKAN=OFF ` -DENGINE_ENABLE_METAL=OFF - - name: Build (Linux/macOS) - if: runner.os != 'Windows' + - name: Build (Linux) + if: runner.os == 'Linux' run: | cmake --build build --target audiocpp_server server_config_test --parallel "$(nproc)" - - name: Build (macOS nproc fallback) + - name: Build (macOS) if: runner.os == 'macOS' run: | cmake --build build --target audiocpp_server server_config_test --parallel "$(sysctl -n hw.logicalcpu)"