diff --git a/app/server/README.md b/app/server/README.md index e9e49d862..790a49c62 100644 --- a/app/server/README.md +++ b/app/server/README.md @@ -397,3 +397,42 @@ curl http://127.0.0.1:8080/v1/tasks/run \ } }' ``` + +### `POST /v1/tasks/unload_models` + +Unload specific models from memory to free resources (e.g. VRAM on GPU backends). Subsequent requests to an unloaded model will trigger a transparent reload. The server waits for any in-flight inference on each target model to complete before unloading it. + +The request body must contain a `model_ids` array of strings. Unknown ids are reported in the response rather than causing an error. Models that are not yet loaded (e.g. lazy-loaded models that have not been requested yet) are skipped silently. + +```bash +curl http://127.0.0.1:8080/v1/tasks/unload_models \ + -H 'Content-Type: application/json' \ + -d '{ + "model_ids": ["pocket-tts", "qwen3-asr"] + }' +``` + +Response: + +```json +{ + "unloaded": ["pocket-tts", "qwen3-asr"], + "not_found": [] +} +``` + +### `POST /v1/tasks/unload_all_models` + +Unload all currently loaded models from memory. No request body is required. As with the selective endpoint, subsequent requests will reload models transparently. + +```bash +curl -X POST http://127.0.0.1:8080/v1/tasks/unload_all_models +``` + +Response: + +```json +{ + "unloaded": ["pocket-tts", "qwen3-asr"] +} +``` diff --git a/app/server/runtime.cpp b/app/server/runtime.cpp index 46ce9755a..3d1250d21 100644 --- a/app/server/runtime.cpp +++ b/app/server/runtime.cpp @@ -686,6 +686,12 @@ HttpResponse ServerState::handle(const HttpRequest & request) { else if (request.method == "POST" && request.path == "/v1/tasks/stream") { response = handle_generic_stream(request.body); } + else if (request.method == "POST" && request.path == "/v1/tasks/unload_models") { + response = handle_unload_models(request.body); + } + else if (request.method == "POST" && request.path == "/v1/tasks/unload_all_models") { + response = handle_unload_all_models(); + } else { response = error_response(404, "unknown endpoint: " + request.path, "not_found"); } @@ -1544,4 +1550,78 @@ std::string ServerState::get_allowed_origin(const HttpRequest & request) const { return ""; } +void ServerState::LoadedModel::unload() { + offline = nullptr; + streaming = nullptr; + session.reset(); + model.reset(); +} + +HttpResponse ServerState::handle_unload_models(const std::string & body_text) { + const auto body = engine::io::json::parse(body_text); + const auto * ids = body.find("model_ids"); + if (ids == nullptr || !ids->is_array()) { + return error_response(400, "request requires a 'model_ids' string array", "invalid_request_error"); + } + + std::vector unloaded; + std::vector not_found; + + for (const auto & id_val : ids->as_array()) { + if (!id_val.is_string()) { + return error_response(400, "each element of 'model_ids' must be a string", "invalid_request_error"); + } + const std::string id = id_val.as_string(); + const auto it = model_index_.find(id); + if (it == model_index_.end()) { + not_found.push_back(id); + continue; + } + LoadedModel & model = *models_.at(it->second); + // Only unload if the model is currently loaded in memory. Acquire the busy + // lock for the duration of the unload so no inference starts mid-operation. + if (model.session != nullptr) { + [[maybe_unused]] BusyGuard::Lock lock = model.busy.acquire(0, model.config.id); + model.unload(); + unloaded.push_back(id); + } + } + + std::ostringstream out; + out << "{\"unloaded\":["; + for (size_t i = 0; i < unloaded.size(); ++i) { + if (i != 0) { out << ","; } + out << json_quote(unloaded[i]); + } + out << "],\"not_found\":"; + out << "["; + for (size_t i = 0; i < not_found.size(); ++i) { + if (i != 0) { out << ","; } + out << json_quote(not_found[i]); + } + out << "]}"; + return json_response(out.str()); +} + +HttpResponse ServerState::handle_unload_all_models() { + std::vector unloaded; + + for (auto & model : models_) { + if (model->session != nullptr) { + [[maybe_unused]] BusyGuard::Lock lock = model->busy.acquire(0, model->config.id); + model->unload(); + unloaded.push_back(model->config.id); + } + } + + std::ostringstream out; + out << "{\"unloaded\":["; + for (size_t i = 0; i < unloaded.size(); ++i) { + if (i != 0) { out << ","; } + out << json_quote(unloaded[i]); + } + out << "]}"; + return json_response(out.str()); +} + } // namespace minitts::server diff --git a/app/server/runtime.h b/app/server/runtime.h index 298dfe369..44b965a0a 100644 --- a/app/server/runtime.h +++ b/app/server/runtime.h @@ -52,6 +52,10 @@ class ServerState final : public IHttpHandler { // Serializes runs on this model and bounds how long a caller waits for its // turn; see BusyGuard. BusyGuard busy; + + // Release the loaded model and session from memory (frees VRAM on GPU backends). + // The next request will trigger a reload via ensure_model_loaded_locked(). + void unload(); }; // Acquire the model's run guard. `request_timeout_ms` is the caller-supplied @@ -125,6 +129,8 @@ class ServerState final : public IHttpHandler { HttpResponse handle_generic_run(const std::string & body_text); HttpResponse handle_generic_stream(const std::string & body_text); HttpResponse handle_voices(const HttpRequest & request) const; + HttpResponse handle_unload_models(const std::string & body_text); + HttpResponse handle_unload_all_models(); std::string models_json() const; std::string get_allowed_origin(const HttpRequest & request) const;