diff --git a/include/engine/community_models/f5_tts/dit_modules.h b/include/engine/community_models/f5_tts/dit_modules.h index 60cbae1f4..63db6946f 100644 --- a/include/engine/community_models/f5_tts/dit_modules.h +++ b/include/engine/community_models/f5_tts/dit_modules.h @@ -33,6 +33,12 @@ std::vector * const_stage_begin(); void const_stage_bind(std::vector * stage, ggml_backend_t backend); void const_stage_upload(std::vector * stage, ggml_backend_t backend); void const_stage_end(std::vector * stage); +// Frees the backend buffers const_stage_bind gave to each staged constant, the +// host copies, and the vector itself. The graph that staged them owns the vector +// from const_stage_begin, so this runs on that graph's destruction: without it +// every CUDA cache eviction leaks the pe table, the masks and the per-block ones +// rows of the evicted graph. +void const_stage_free(std::vector * stage); } // namespace engine::models::f5_tts namespace engine::models::f5_tts { diff --git a/src/community_models/f5_tts/dit_modules.cpp b/src/community_models/f5_tts/dit_modules.cpp index b579a7985..0a9aa534b 100644 --- a/src/community_models/f5_tts/dit_modules.cpp +++ b/src/community_models/f5_tts/dit_modules.cpp @@ -249,12 +249,20 @@ void const_stage_upload(std::vector * stage, ggml_backend_t backend) } } void const_stage_end(std::vector * stage) { - // detach only; the vector is deliberately leaked (a few KB per graph - // build) to avoid ownership hazards across build paths. + // detach only; ownership of the staged constants was handed to the graph that + // built them (const_stage_free runs with that graph's destructor). t_const_stage = nullptr; (void)stage; } +void const_stage_free(std::vector * stage) { + if (stage == nullptr) return; + for (auto & c : *stage) { + if (c.owned_buffer != nullptr) ggml_backend_buffer_free(c.owned_buffer); + } + delete stage; +} + F5DiTGraphBuild build_dit_modules_graph( ggml_context * ggml, diff --git a/src/community_models/f5_tts/runtime.cpp b/src/community_models/f5_tts/runtime.cpp index c914d16a4..cfe6de9fa 100644 --- a/src/community_models/f5_tts/runtime.cpp +++ b/src/community_models/f5_tts/runtime.cpp @@ -328,6 +328,11 @@ std::pair, std::vector> f5_dit_forward_cfg( ggml_tensor * th_t = nullptr; // [256, 1] ggml_tensor * output = nullptr; // [MEL, N] (half 0, conditioned) ggml_tensor * out_u = nullptr; // [MEL, N] (half 1, uncond) + // The staged graph constants this build created (pe table, masks, + // per-block ones rows). On CUDA each carries a private backend buffer + // allocated in const_stage_bind: the eviction path destroys the + // unique_ptr, and nothing else frees them. + std::vector * stage = nullptr; ~CfgGraph() { // Cache eviction (more than two distinct (N, NT) keys in one process, // i.e. every chunk past the second of a chunked request) destroys the @@ -338,6 +343,7 @@ std::pair, std::vector> f5_dit_forward_cfg( if (gallocr != nullptr) ggml_gallocr_free(gallocr); if (io_buffer != nullptr) ggml_backend_buffer_free(io_buffer); if (ctx != nullptr) ggml_free(ctx); + const_stage_free(stage); } }; static auto * cache = new std::map, std::unique_ptr>(); @@ -367,6 +373,7 @@ std::pair, std::vector> f5_dit_forward_cfg( ctx, dit_w, arch, N, NT, model.backend_type); const_stage_end(stage); cfg_staged = stage; + gnew->stage = stage; // Mark I/O explicitly: without ggml_set_input/output the gallocr // treats leaves as scratch and may clobber them across replays. ggml_set_input(io.x.tensor); @@ -539,12 +546,17 @@ std::vector f5_dit_forward( ggml_tensor * tap_time_embed = nullptr; ggml_tensor * tap_block0 = nullptr; ggml_tensor * tap_block21 = nullptr; + // The staged graph constants this build created (pe table, per-block ones + // rows). On CUDA each carries a private backend buffer from + // const_stage_bind, so the vector has to die with the graph. + std::vector * stage = nullptr; ~DiTGraph() { // Leaked by design when cached (freed only on program-graph reset); // destroyed here only when construction throws mid-build. if (gallocr != nullptr) ggml_gallocr_free(gallocr); if (io_buffer != nullptr) ggml_backend_buffer_free(io_buffer); if (ctx != nullptr) ggml_free(ctx); + const_stage_free(stage); } }; @@ -587,6 +599,7 @@ std::vector f5_dit_forward( const_stage_end(stage); // staged constants upload after ggml_backend_alloc_ctx_tensors below staged_module_consts = stage; + gnew->stage = stage; ggml_set_input(io.x.tensor); ggml_set_input(io.cond.tensor); ggml_set_input(io.text_ids.tensor);