From 929f3a2296432f63275e0497f538d4e2374b0421 Mon Sep 17 00:00:00 2001 From: 0xShug0 <231717474+0xShug0@users.noreply.github.com> Date: Tue, 6 Oct 2026 17:44:03 -0400 Subject: [PATCH 1/2] Reduce Qwen3, IndexTTS, and BreezeTTS memory allocations --- docs/models/index_tts.md | 8 +-- docs/models/qwen3.md | 8 +-- .../qwen3_index_breeze_tts_optimizations.md | 43 ++++++++++++++++ include/engine/models/breeze_tts/generator.h | 2 +- include/engine/models/index_tts2/session.h | 4 +- include/engine/models/qwen3_tts/session.h | 8 +-- model_specs/breeze_tts.json | 8 +-- model_specs/index_tts2.json | 4 +- model_specs/qwen3_tts.json | 16 +++--- .../modules/vocoders/bigvgan_vocoder.cpp | 4 +- src/models/breeze_tts/generator.cpp | 8 ++- src/models/breeze_tts/session.cpp | 6 +-- src/models/index_tts2/gpt.cpp | 9 ++-- src/models/index_tts2/s2mel.cpp | 6 +-- src/models/index_tts2/session.cpp | 12 ++--- src/models/qwen3_tts/talker.cpp | 50 ++++++++++++++----- .../qwen3_tts/tokenizer_speech_decoder.cpp | 1 + 17 files changed, 133 insertions(+), 64 deletions(-) create mode 100644 docs/reports/qwen3_index_breeze_tts_optimizations.md diff --git a/docs/models/index_tts.md b/docs/models/index_tts.md index 7e76a6fcf..4b654b87e 100644 --- a/docs/models/index_tts.md +++ b/docs/models/index_tts.md @@ -100,8 +100,8 @@ audiocpp_cli --task tts --family index_tts2 \ | `index_tts2.speaker_cache_slots` | integer slots | `1` | Prepared speaker-reference cache slots; set `0` to disable reuse. | | `index_tts2.emotion_cache_slots` | integer slots | `1` | Prepared emotion-reference cache slots; set `0` to disable reuse. | | `index_tts2.emotion_text_cache_slots` | integer slots | `1` | Emotion-text weight cache slots; set `0` to disable reuse. | -| `index_tts2.gpt_graph_arena_mb` | MB | `2048` | GPT graph arena size. | -| `index_tts2.s2mel_graph_arena_mb` | MB | `2048` | S2Mel graph arena size. | +| `index_tts2.gpt_graph_arena_mb` | MB | `32` | GPT graph arena size. | +| `index_tts2.s2mel_graph_arena_mb` | MB | `32` | S2Mel graph arena size. | | `index_tts2.reference_graph_arena_mb` | MB | `512` | Reference encoder and codec graph arena size. | | `index_tts2.emotion_text_prefill_graph_arena_mb` | MB | `2048` | Emotion-text prefill graph arena size. | | `index_tts2.emotion_text_decode_graph_arena_mb` | MB | `512` | Emotion-text cached-step graph arena size. | @@ -251,8 +251,8 @@ or commercial use. | `index_tts2.speaker_cache_slots` | integer slots | `1` | Prepared speaker-reference cache slots; set `0` to disable reuse. | | `index_tts2.emotion_cache_slots` | integer slots | `1` | Prepared emotion-reference cache slots; set `0` to disable reuse. | | `index_tts2.emotion_text_cache_slots` | integer slots | `1` | Emotion-text weight cache slots; set `0` to disable reuse. | -| `index_tts2.gpt_graph_arena_mb` | MB | `2048` | GPT graph arena size. | -| `index_tts2.s2mel_graph_arena_mb` | MB | `2048` | S2Mel graph arena size. | +| `index_tts2.gpt_graph_arena_mb` | MB | `32` | GPT graph arena size. | +| `index_tts2.s2mel_graph_arena_mb` | MB | `32` | S2Mel graph arena size. | | `index_tts2.reference_graph_arena_mb` | MB | `512` | Reference encoder and codec graph arena size. | | `index_tts2.emotion_text_prefill_graph_arena_mb` | MB | `2048` | Emotion-text prefill graph arena size. | | `index_tts2.emotion_text_decode_graph_arena_mb` | MB | `512` | Emotion-text cached-step graph arena size. | diff --git a/docs/models/qwen3.md b/docs/models/qwen3.md index 74dd04fd5..1ac6eb3b7 100644 --- a/docs/models/qwen3.md +++ b/docs/models/qwen3.md @@ -126,13 +126,13 @@ constant-context sizes are in MiB. | `qwen3_tts.speech_encoder_weight_type` | `native`, `f32`, `f16`, `bf16`, `q8_0` | `native` | Reference speech encoder matmul weight storage. | | `qwen3_tts.speech_decoder_weight_type` | `native`, `f32`, `f16`, `bf16`, `q8_0` | `native` | Speech decoder matmul weight storage. | | `qwen3_tts.conv_weight_type` | `native`, `f32`, `f16` | `f32` | Speech tokenizer and speaker encoder convolution weight storage. | -| `qwen3_tts.talker_graph_arena_mb` | positive integer | `256` | Talker graph arena. | +| `qwen3_tts.talker_graph_arena_mb` | positive integer | `32` | Talker graph arena. | | `qwen3_tts.speech_encoder_graph_arena_mb` | positive integer | `32` | Reference speech encoder graph arena. | | `qwen3_tts.speech_decoder_graph_arena_mb` | positive integer | `32` | Speech decoder graph arena. | | `qwen3_tts.speaker_encoder_graph_arena_mb` | positive integer | `32` | Speaker encoder graph arena. | -| `qwen3_tts.talker_constant_context_mb` | positive integer | `4096` | Talker constant tensor context. | -| `qwen3_tts.code_predictor_constant_context_mb` | positive integer | `1536` | Code predictor constant tensor context. | -| `qwen3_tts.speech_decoder_constant_context_mb` | positive integer | `1536` | Speech decoder constant tensor context. | +| `qwen3_tts.talker_constant_context_mb` | positive integer | `4` | Talker constant tensor context. | +| `qwen3_tts.code_predictor_constant_context_mb` | positive integer | `4` | Code predictor constant tensor context. | +| `qwen3_tts.speech_decoder_constant_context_mb` | positive integer | `4` | Speech decoder constant tensor context. | ## Qwen3 ASR diff --git a/docs/reports/qwen3_index_breeze_tts_optimizations.md b/docs/reports/qwen3_index_breeze_tts_optimizations.md new file mode 100644 index 000000000..2448d0d41 --- /dev/null +++ b/docs/reports/qwen3_index_breeze_tts_optimizations.md @@ -0,0 +1,43 @@ +# Qwen3-TTS, IndexTTS, and BreezeTTS Optimization Results + +RTX 5090, CUDA Debug build, Q8 weights, 8 CPU threads, fixed seed. Each build/model ran sequentially in a fresh server with the same mixed-length request sequence and logging enabled. Peak VRAM was sampled continuously across the sequence (5 ms target interval); warm RTF is the median of four identical long-request repeats. Positive RTF changes mean slower. + +| Model/path | Before peak (MiB) | Current peak (MiB) | Peak change (MiB) | Before warm RTF | Current warm RTF | RTF change | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| Qwen3-TTS Base clone | 6,376 | 5,070 | -1,306 | 0.10238 | 0.10417 | +1.75% | +| Qwen3-TTS CustomVoice | 6,022 | 5,068 | -954 | 0.10207 | 0.10307 | +0.98% | +| Qwen3-TTS VoiceDesign | 6,180 | 5,110 | -1,070 | 0.10334 | 0.10347 | +0.12% | +| IndexTTS2, including emotion request | 6,288 | 6,290 | +2 | 0.14590 | 0.14676 | +0.60% | +| IndexTTS2.5, including emotion request | 5,918 | 5,908 | -10 | 0.08688 | 0.08623 | -0.75% | +| BreezeTTS clone | 6,914 | 6,716 | -198 | 0.22636 | 0.22632 | -0.02% | +| BreezeTTS voice design | 6,696 | 6,702 | +6 | 0.27655 | 0.27631 | -0.09% | + +Index warm RTF above measures ordinary synthesis, not the emotion request. Before the first emotion request, the measured ordinary-only peaks were: + +| Path | Before peak (MiB) | Current peak (MiB) | Peak change (MiB) | +| --- | ---: | ---: | ---: | +| IndexTTS2 ordinary synthesis | 6,288 | 5,556 | -732 | +| IndexTTS2.5 ordinary synthesis | 5,918 | 5,188 | -730 | + +## Qwen3 timing cross-check + +The same saved binaries were retested in reversed order (current, then before), without rebuilding or changing code. Four warmed repeats per path: + +| Path | Before warm RTF | Current warm RTF | RTF change | Initial A/B change | +| --- | ---: | ---: | ---: | ---: | +| Base clone | 0.10309 | 0.10196 | -1.09% | +1.75% | +| CustomVoice | 0.10215 | 0.10232 | +0.17% | +0.98% | +| VoiceDesign | 0.10190 | 0.10295 | +1.03% | +0.12% | + +Measured component medians in the reversed run: + +| Path/component | Before (ms) | Current (ms) | +| --- | ---: | ---: | +| Base cached-step compute | 489.660 | 481.306 | +| CustomVoice cached-step compute | 482.797 | 481.691 | +| VoiceDesign cached-step compute | 478.612 | 478.666 | +| VoiceDesign code-predictor compute (unchanged implementation) | 675.911 | 684.647 | + +The initial slowdown did not reproduce consistently: Base reversed direction, CustomVoice narrowed to 0.17%, and VoiceDesign's changed cached-step compute differed by only 0.054 ms. Its total slowdown was mainly in the unchanged code predictor. These results support run-to-run timing variation rather than a demonstrated cached-step regression; a specific clock/thermal/scheduling cause was not established. They do not prove that every sub-percent effect is noise. + +**Parity-safe in the tested coverage:** all 82 mixed-length A/B output WAV pairs and all 18 reversed-order Qwen3 pairs were byte-identical. This is before/current C++ parity, not a new Python parity claim. diff --git a/include/engine/models/breeze_tts/generator.h b/include/engine/models/breeze_tts/generator.h index 8d5c2a8aa..abd8e7923 100644 --- a/include/engine/models/breeze_tts/generator.h +++ b/include/engine/models/breeze_tts/generator.h @@ -58,7 +58,7 @@ class BreezeGeneratorRuntime { ~BreezeGeneratorRuntime(); engine::runtime::AudioBuffer generate(const BreezeGenerationRequest & request); - BreezeSpeechCodes encode_reference(const engine::runtime::AudioBuffer & audio) const; + BreezeSpeechCodes encode_reference(const engine::runtime::AudioBuffer & audio, bool retain_graphs = true) const; void begin_stream(const BreezeGenerationRequest & request); BreezeStreamEvent next_stream_audio(size_t max_new_frames, int64_t lookahead_margin); void end_stream(); diff --git a/include/engine/models/index_tts2/session.h b/include/engine/models/index_tts2/session.h index bbcd394c1..d59dc97eb 100644 --- a/include/engine/models/index_tts2/session.h +++ b/include/engine/models/index_tts2/session.h @@ -88,8 +88,8 @@ class IndexTTS2Session final runtime::TaskSpec task_; std::shared_ptr assets_; - size_t gpt_graph_arena_bytes_ = 2048ull * 1024ull * 1024ull; - size_t s2mel_graph_arena_bytes_ = 2048ull * 1024ull * 1024ull; + size_t gpt_graph_arena_bytes_ = 32ull * 1024ull * 1024ull; + size_t s2mel_graph_arena_bytes_ = 32ull * 1024ull * 1024ull; size_t reference_graph_arena_bytes_ = 512ull * 1024ull * 1024ull; size_t emotion_text_prefill_graph_arena_bytes_ = 2048ull * 1024ull * 1024ull; size_t emotion_text_decode_graph_arena_bytes_ = 512ull * 1024ull * 1024ull; diff --git a/include/engine/models/qwen3_tts/session.h b/include/engine/models/qwen3_tts/session.h index b39742da3..e010ed3f4 100644 --- a/include/engine/models/qwen3_tts/session.h +++ b/include/engine/models/qwen3_tts/session.h @@ -58,16 +58,16 @@ class Qwen3TTSSession final runtime::TaskSpec task_; std::shared_ptr assets_; - size_t talker_graph_arena_bytes_ = 256ull * 1024ull * 1024ull; + size_t talker_graph_arena_bytes_ = 32ull * 1024ull * 1024ull; size_t speech_encoder_graph_arena_bytes_ = 32ull * 1024ull * 1024ull; size_t speech_decoder_graph_arena_bytes_ = 32ull * 1024ull * 1024ull; size_t speaker_encoder_graph_arena_bytes_ = 32ull * 1024ull * 1024ull; // No-alloc GGML context capacities for reusable constant tensor descriptors. // Generous on purpose; ConstantTensorCache fits them to the host when it has // to, so a small machine is not asked to reserve what it does not have. - size_t talker_constant_context_bytes_ = 4ull * 1024ull * 1024ull * 1024ull; - size_t code_predictor_constant_context_bytes_ = 1536ull * 1024ull * 1024ull; - size_t speech_decoder_constant_context_bytes_ = 1536ull * 1024ull * 1024ull; + size_t talker_constant_context_bytes_ = 4ull * 1024ull * 1024ull; + size_t code_predictor_constant_context_bytes_ = 4ull * 1024ull * 1024ull; + size_t speech_decoder_constant_context_bytes_ = 4ull * 1024ull * 1024ull; engine::assets::TensorStorageType talker_weight_storage_type_ = engine::assets::TensorStorageType::Native; engine::assets::TensorStorageType speech_encoder_weight_storage_type_ = engine::assets::TensorStorageType::Native; engine::assets::TensorStorageType speech_decoder_weight_storage_type_ = engine::assets::TensorStorageType::Native; diff --git a/model_specs/breeze_tts.json b/model_specs/breeze_tts.json index 7a9e9df4e..88a844cae 100644 --- a/model_specs/breeze_tts.json +++ b/model_specs/breeze_tts.json @@ -157,18 +157,18 @@ { "name": "graph_arena_mb", "type": "int", - "description": "Reusable runtime graph arena size in MiB; default 1024.", + "description": "Reusable runtime graph arena size in MiB; default 32.", "required": false, "min": 1, - "default": 1024 + "default": 32 }, { "name": "weight_context_mb", "type": "int", - "description": "Weight loading context size in MiB; default 2048.", + "description": "Weight loading context size in MiB; default 4.", "required": false, "min": 1, - "default": 2048 + "default": 4 }, { "name": "reference_cache_slots", diff --git a/model_specs/index_tts2.json b/model_specs/index_tts2.json index 45c03c917..12e3667cb 100644 --- a/model_specs/index_tts2.json +++ b/model_specs/index_tts2.json @@ -222,7 +222,7 @@ "description": "Context or graph arena size in MiB.", "required": false, "min": 1, - "default": 2048 + "default": 32 }, { "name": "s2mel_graph_arena_mb", @@ -230,7 +230,7 @@ "description": "Context or graph arena size in MiB.", "required": false, "min": 1, - "default": 2048 + "default": 32 }, { "name": "reference_graph_arena_mb", diff --git a/model_specs/qwen3_tts.json b/model_specs/qwen3_tts.json index 21fb8bd48..627bfd7f9 100644 --- a/model_specs/qwen3_tts.json +++ b/model_specs/qwen3_tts.json @@ -228,10 +228,10 @@ { "name": "talker_graph_arena_mb", "type": "int", - "description": "Talker graph arena size in MiB; default 256.", + "description": "Talker graph arena size in MiB; default 32.", "required": false, "min": 1, - "default": 256 + "default": 32 }, { "name": "speech_encoder_graph_arena_mb", @@ -260,26 +260,26 @@ { "name": "talker_constant_context_mb", "type": "int", - "description": "Talker constant tensor context size in MiB; default 4096.", + "description": "Talker constant tensor context size in MiB; default 4.", "required": false, "min": 1, - "default": 4096 + "default": 4 }, { "name": "code_predictor_constant_context_mb", "type": "int", - "description": "Code predictor constant tensor context size in MiB; default 1536.", + "description": "Code predictor constant tensor context size in MiB; default 4.", "required": false, "min": 1, - "default": 1536 + "default": 4 }, { "name": "speech_decoder_constant_context_mb", "type": "int", - "description": "Speech decoder constant tensor context size in MiB; default 1536.", + "description": "Speech decoder constant tensor context size in MiB; default 4.", "required": false, "min": 1, - "default": 1536 + "default": 4 } ], "load": [] diff --git a/src/framework/modules/vocoders/bigvgan_vocoder.cpp b/src/framework/modules/vocoders/bigvgan_vocoder.cpp index ea744483a..7b7185880 100644 --- a/src/framework/modules/vocoders/bigvgan_vocoder.cpp +++ b/src/framework/modules/vocoders/bigvgan_vocoder.cpp @@ -793,7 +793,7 @@ class BigVganRunner { } release_graph(); ggml_init_params params{ - 1024ull * 1024ull * 1024ull, + 32ull * 1024ull * 1024ull, nullptr, true, }; @@ -1055,7 +1055,7 @@ BigVganVocoderComponent BigVganVocoderComponent::load_from_tensor_source( weights->execution_context->backend(), weights->execution_context->backend_type(), "framework.bigvgan.weights", - 1024ull * 1024ull * 1024ull); + 4ull * 1024ull * 1024ull); for (const auto & tensor : source->tensors()) { weights->parameter_count += tensor_elements(tensor.shape); diff --git a/src/models/breeze_tts/generator.cpp b/src/models/breeze_tts/generator.cpp index 6fb82f449..4b52dc9f9 100644 --- a/src/models/breeze_tts/generator.cpp +++ b/src/models/breeze_tts/generator.cpp @@ -1489,8 +1489,12 @@ engine::runtime::AudioBuffer BreezeGeneratorRuntime::generate(const BreezeGenera return audio; } -BreezeSpeechCodes BreezeGeneratorRuntime::encode_reference(const engine::runtime::AudioBuffer & audio) const { - return impl_->speech_encoder->encode(audio); +BreezeSpeechCodes BreezeGeneratorRuntime::encode_reference(const engine::runtime::AudioBuffer & audio, bool retain_graphs) const { + auto codes = impl_->speech_encoder->encode(audio); + if (!retain_graphs) { + impl_->speech_encoder->release_runtime_graphs(); + } + return codes; } void BreezeGeneratorRuntime::begin_stream(const BreezeGenerationRequest & request) { diff --git a/src/models/breeze_tts/session.cpp b/src/models/breeze_tts/session.cpp index c13688fc0..4d9e5c759 100644 --- a/src/models/breeze_tts/session.cpp +++ b/src/models/breeze_tts/session.cpp @@ -185,11 +185,11 @@ BreezeTTSSession::BreezeTTSSession( const auto graph_arena_bytes = runtime::parse_size_mb_option( options.options, {"graph_arena_mb"}, - 1024ull * 1024ull * 1024ull); + 32ull * 1024ull * 1024ull); const auto weight_context_bytes = runtime::parse_size_mb_option( options.options, {"weight_context_mb"}, - 2048ull * 1024ull * 1024ull); + 4ull * 1024ull * 1024ull); const auto attention_preference = attention_preference_from_options(options); trace_attention_preference(attention_preference); const auto bf16_activations = bf16_activation_mode_from_options(options); @@ -236,7 +236,7 @@ BreezeSpeechCodes BreezeTTSSession::resolve_reference_codes(const runtime::Audio } const auto start = std::chrono::steady_clock::now(); ReferenceCacheEntry entry; - entry.codes = generator_->encode_reference(audio); + entry.codes = generator_->encode_reference(audio, reference_cache_.capacity() == 0); engine::debug::trace_log_scalar("breeze_tts.reference.frames", entry.codes.frames); engine::debug::trace_log_scalar("breeze_tts.reference.codebooks", entry.codes.code_groups); if (reference_cache_.capacity() == 0) { diff --git a/src/models/index_tts2/gpt.cpp b/src/models/index_tts2/gpt.cpp index 958ad1cfb..69fb71caf 100644 --- a/src/models/index_tts2/gpt.cpp +++ b/src/models/index_tts2/gpt.cpp @@ -1146,7 +1146,7 @@ class IndexTTS2GptRuntime::ConditioningGraph { output_dims_ = out.shape.dims[2]; ggml_set_output(output_); - graph_ = ggml_new_graph_custom(ctx_.get(), static_cast(std::max(65536, frames_ * 4096 + 8192)), false); + graph_ = ggml_new_graph_custom(ctx_.get(), 65536, false); ggml_build_forward_expand(graph_, output_); input_buffer_ = ggml_backend_alloc_ctx_tensors(input_ctx_.get(), execution_.backend()); if (input_buffer_ == nullptr) { @@ -1438,7 +1438,7 @@ class IndexTTS2GptRuntime::PrefillGraph { mel = core::reshape_tensor(ctx, core::ensure_backend_addressable_layout(ctx, mel), core::TensorShape::from_dims({1, 1, kModelDim})); auto x = modules::ConcatModule({1}).build(ctx, conds, text); x = modules::ConcatModule({1}).build(ctx, x, mel); - graph_ = ggml_new_graph_custom(ctx_.get(), static_cast(std::max(65536, prompt_steps_ * 8192)), false); + graph_ = ggml_new_graph_custom(ctx_.get(), 65536, false); for (const auto & layer : weights_->gpt_layers) { auto out = gpt2_layer_full(ctx, x, layer); x = out.output; @@ -1681,10 +1681,7 @@ class IndexTTS2GptRuntime::ForwardGraph { output_ = core::ensure_backend_addressable_layout(ctx, x).tensor; ggml_set_output(output_); - graph_ = ggml_new_graph_custom( - ctx_.get(), - static_cast(std::max(65536, (kV2ConditionTokens + text_steps_ + mel_steps_) * 8192)), - false); + graph_ = ggml_new_graph_custom(ctx_.get(), 65536, false); ggml_build_forward_expand(graph_, output_); input_buffer_ = ggml_backend_alloc_ctx_tensors(input_ctx_.get(), execution_.backend()); if (input_buffer_ == nullptr) { diff --git a/src/models/index_tts2/s2mel.cpp b/src/models/index_tts2/s2mel.cpp index a6461faa5..ae0eec10c 100644 --- a/src/models/index_tts2/s2mel.cpp +++ b/src/models/index_tts2/s2mel.cpp @@ -659,7 +659,7 @@ class IndexTTS2S2MelRuntime::GptLayerGraph { x = modules::LinearModule({128, kContentDim, true}).build(ctx, x, weights_->gpt_layer.linear2); output_ = core::ensure_backend_addressable_layout(ctx, x).tensor; ggml_set_output(output_); - graph_ = ggml_new_graph_custom(ctx_.get(), static_cast(std::max(8192, frames_ * 128)), false); + graph_ = ggml_new_graph_custom(ctx_.get(), 8192, false); ggml_build_forward_expand(graph_, output_); input_buffer_ = ggml_backend_alloc_ctx_tensors(input_ctx_.get(), execution_.backend()); if (input_buffer_ == nullptr) { @@ -795,7 +795,7 @@ class IndexTTS2S2MelRuntime::LengthRegulatorGraph { auto out = modules::MulModule{}.build(ctx, x, mask); output_ = core::ensure_backend_addressable_layout(ctx, out).tensor; ggml_set_output(output_); - graph_ = ggml_new_graph_custom(ctx_.get(), static_cast(std::max(32768, output_frames_ * 512)), false); + graph_ = ggml_new_graph_custom(ctx_.get(), 32768, false); ggml_build_forward_expand(graph_, output_); input_buffer_ = ggml_backend_alloc_ctx_tensors(input_ctx_.get(), execution_.backend()); if (input_buffer_ == nullptr) { @@ -937,7 +937,7 @@ class IndexTTS2S2MelRuntime::CfmGraph { weights_->cfm); output_ = core::ensure_backend_addressable_layout(ctx, output).tensor; ggml_set_output(output_); - graph_ = ggml_new_graph_custom(ctx_.get(), static_cast(std::max(131072, frames_ * 4096)), false); + graph_ = ggml_new_graph_custom(ctx_.get(), 65536, false); ggml_build_forward_expand(graph_, output_); input_buffer_ = ggml_backend_alloc_ctx_tensors(input_ctx_.get(), execution_.backend()); if (input_buffer_ == nullptr) { diff --git a/src/models/index_tts2/session.cpp b/src/models/index_tts2/session.cpp index c2177de3d..449d990d8 100644 --- a/src/models/index_tts2/session.cpp +++ b/src/models/index_tts2/session.cpp @@ -305,13 +305,6 @@ IndexTTS2Session::IndexTTS2Session( assets_, options.backend, conv_weight_storage_type_); - qwen_emotion_ = std::make_unique( - assets_, - execution_context(), - emotion_text_prefill_graph_arena_bytes_, - emotion_text_decode_graph_arena_bytes_, - weight_context_bytes_, - matmul_weight_storage_type_); int64_t matrix_rows = 0; for (const int64_t count : assets_->config.emo_num) { matrix_rows += count; @@ -582,6 +575,11 @@ std::vector IndexTTS2Session::resolve_emotion_vector( const bool will_evict = emotion_text_weights_cache_.capacity() > 0 && emotion_text_weights_cache_.size() >= emotion_text_weights_cache_.capacity(); + if (!qwen_emotion_) { + qwen_emotion_ = std::make_unique( + assets_, execution_context(), emotion_text_prefill_graph_arena_bytes_, + emotion_text_decode_graph_arena_bytes_, weight_context_bytes_, matmul_weight_storage_type_); + } explicit_weights = qwen_emotion_->infer(emotion_text, emotion_text_max_tokens_).values; if (mem_saver_) { qwen_emotion_->release_graphs(); diff --git a/src/models/qwen3_tts/talker.cpp b/src/models/qwen3_tts/talker.cpp index 889ac8867..4f8bf0480 100644 --- a/src/models/qwen3_tts/talker.cpp +++ b/src/models/qwen3_tts/talker.cpp @@ -16,6 +16,7 @@ #include "engine/framework/core/constant_tensor_cache.h" +#include #include #include @@ -955,13 +956,31 @@ class TalkerPrefillGraph { } hidden_output_ = decoder_out.hidden.tensor; logits_output_ = decoder_out.logits.tensor; + ggml_set_input(input_); + ggml_set_input(positions_); + ggml_set_output(positions_); + if (attention_mask_ != nullptr) { + ggml_set_input(attention_mask_); + ggml_set_output(attention_mask_); + } + ggml_set_output(hidden_output_); + if (hidden_output_->view_src != nullptr) ggml_set_output(hidden_output_->view_src); ggml_set_output(logits_output_); graph_ = ggml_new_graph_custom(ctx_.get(), 65536, false); ggml_build_forward_expand(graph_, logits_output_); + // The CPU exports every layer's K/V after execution, not just logits. + for (size_t layer = 0; layer < keys_.size(); ++layer) { + for (auto * output : {keys_[layer], values_[layer]}) { + ggml_set_output(output); + if (output->view_src != nullptr) ggml_set_output(output->view_src); + ggml_build_forward_expand(graph_, output); + } + } constants.finish_graph(); constants.ensure_uploaded(); - buffer_ = ggml_backend_alloc_ctx_tensors(ctx_.get(), weights_->backend()); - if (buffer_ == nullptr) { + graph_allocator_.reset(ggml_gallocr_new(ggml_backend_get_default_buffer_type(weights_->backend()))); + if (!graph_allocator_ || !ggml_gallocr_reserve(graph_allocator_.get(), graph_) || + !ggml_gallocr_alloc_graph(graph_allocator_.get(), graph_)) { throw std::runtime_error("failed to allocate Qwen3 talker prefill graph"); } std::vector positions(static_cast(prompt_capacity_), 0); @@ -977,9 +996,7 @@ class TalkerPrefillGraph { ~TalkerPrefillGraph() { engine::core::release_backend_graph_resources(weights_->backend(), graph_); - if (buffer_ != nullptr) { - ggml_backend_buffer_free(buffer_); - } + graph_allocator_.reset(); } bool matches(const Qwen3TalkerWeightsRuntime & weights, int64_t prompt_capacity) const { @@ -1038,7 +1055,7 @@ class TalkerPrefillGraph { std::vector keys_; std::vector values_; ggml_cgraph * graph_ = nullptr; - ggml_backend_buffer_t buffer_ = nullptr; + std::unique_ptr graph_allocator_{nullptr, ggml_gallocr_free}; }; class TalkerCachedStepGraph { @@ -1086,12 +1103,23 @@ class TalkerCachedStepGraph { step_cache_ = std::move(decoder_out.cache); hidden_output_ = decoder_out.hidden.tensor; logits_output_ = decoder_out.logits.tensor; + for (auto * input : {input_, positions_, cache_slot_, attention_mask_}) ggml_set_input(input); + ggml_set_output(hidden_output_); + if (hidden_output_->view_src != nullptr) ggml_set_output(hidden_output_->view_src); + // Persistent state must not be recycled between autoregressive steps. + for (int64_t layer = 0; layer < config.num_hidden_layers; ++layer) { + for (const auto & state : {step_cache_.key_tensor(layer), step_cache_.value_tensor(layer)}) { + ggml_set_input(state.tensor); + ggml_set_output(state.tensor); + } + } ggml_set_output(logits_output_); ggml_build_forward_expand(graph_, logits_output_); constants.finish_graph(); constants.ensure_uploaded(); - buffer_ = ggml_backend_alloc_ctx_tensors(ctx_.get(), weights_->backend()); - if (buffer_ == nullptr) { + graph_allocator_.reset(ggml_gallocr_new(ggml_backend_get_default_buffer_type(weights_->backend()))); + if (!graph_allocator_ || !ggml_gallocr_reserve(graph_allocator_.get(), graph_) || + !ggml_gallocr_alloc_graph(graph_allocator_.get(), graph_)) { throw std::runtime_error("failed to allocate Qwen3 talker cached step graph"); } attention_mask_buffer_.assign(static_cast(cache_steps_), ggml_fp32_to_fp16(-INFINITY)); @@ -1099,9 +1127,7 @@ class TalkerCachedStepGraph { ~TalkerCachedStepGraph() { engine::core::release_backend_graph_resources(weights_->backend(), graph_); - if (buffer_ != nullptr) { - ggml_backend_buffer_free(buffer_); - } + graph_allocator_.reset(); } bool can_run(const Qwen3TalkerWeightsRuntime & weights, int64_t required_capacity) const { @@ -1185,7 +1211,7 @@ class TalkerCachedStepGraph { std::vector attention_mask_buffer_; runtime::TransformerKVCache step_cache_; ggml_cgraph * graph_ = nullptr; - ggml_backend_buffer_t buffer_ = nullptr; + std::unique_ptr graph_allocator_{nullptr, ggml_gallocr_free}; CachedStepTiming last_timing_; }; diff --git a/src/models/qwen3_tts/tokenizer_speech_decoder.cpp b/src/models/qwen3_tts/tokenizer_speech_decoder.cpp index 3a1b89478..990109aaf 100644 --- a/src/models/qwen3_tts/tokenizer_speech_decoder.cpp +++ b/src/models/qwen3_tts/tokenizer_speech_decoder.cpp @@ -1226,6 +1226,7 @@ runtime::AudioBuffer Qwen3SpeechTokenizerDecoderRuntime::decode(const Qwen3Speec graph == nullptr || !graph->matches(*weights_, chunk_frames, execution_context_->backend(), threads); if (graph_rebuilt) { const auto build_start = Clock::now(); + graph.reset(); auto replacement = std::make_unique( weights_, chunk_frames, From 8a1225774159562e0dca4df1167653df250bb65a Mon Sep 17 00:00:00 2001 From: 0xShug0 <231717474+0xShug0@users.noreply.github.com> Date: Tue, 6 Oct 2026 22:36:08 -0400 Subject: [PATCH 2/2] Keep BreezeTTS reference encoder graphs cached --- docs/reports/qwen3_index_breeze_tts_optimizations.md | 6 ++---- include/engine/models/breeze_tts/generator.h | 2 +- src/models/breeze_tts/generator.cpp | 8 ++------ src/models/breeze_tts/session.cpp | 2 +- 4 files changed, 6 insertions(+), 12 deletions(-) diff --git a/docs/reports/qwen3_index_breeze_tts_optimizations.md b/docs/reports/qwen3_index_breeze_tts_optimizations.md index 2448d0d41..e2442679c 100644 --- a/docs/reports/qwen3_index_breeze_tts_optimizations.md +++ b/docs/reports/qwen3_index_breeze_tts_optimizations.md @@ -1,4 +1,4 @@ -# Qwen3-TTS, IndexTTS, and BreezeTTS Optimization Results +# Qwen3-TTS and IndexTTS Optimization Results RTX 5090, CUDA Debug build, Q8 weights, 8 CPU threads, fixed seed. Each build/model ran sequentially in a fresh server with the same mixed-length request sequence and logging enabled. Peak VRAM was sampled continuously across the sequence (5 ms target interval); warm RTF is the median of four identical long-request repeats. Positive RTF changes mean slower. @@ -9,8 +9,6 @@ RTX 5090, CUDA Debug build, Q8 weights, 8 CPU threads, fixed seed. Each build/mo | Qwen3-TTS VoiceDesign | 6,180 | 5,110 | -1,070 | 0.10334 | 0.10347 | +0.12% | | IndexTTS2, including emotion request | 6,288 | 6,290 | +2 | 0.14590 | 0.14676 | +0.60% | | IndexTTS2.5, including emotion request | 5,918 | 5,908 | -10 | 0.08688 | 0.08623 | -0.75% | -| BreezeTTS clone | 6,914 | 6,716 | -198 | 0.22636 | 0.22632 | -0.02% | -| BreezeTTS voice design | 6,696 | 6,702 | +6 | 0.27655 | 0.27631 | -0.09% | Index warm RTF above measures ordinary synthesis, not the emotion request. Before the first emotion request, the measured ordinary-only peaks were: @@ -40,4 +38,4 @@ Measured component medians in the reversed run: The initial slowdown did not reproduce consistently: Base reversed direction, CustomVoice narrowed to 0.17%, and VoiceDesign's changed cached-step compute differed by only 0.054 ms. Its total slowdown was mainly in the unchanged code predictor. These results support run-to-run timing variation rather than a demonstrated cached-step regression; a specific clock/thermal/scheduling cause was not established. They do not prove that every sub-percent effect is noise. -**Parity-safe in the tested coverage:** all 82 mixed-length A/B output WAV pairs and all 18 reversed-order Qwen3 pairs were byte-identical. This is before/current C++ parity, not a new Python parity claim. +**Parity-safe in the tested coverage:** all tested Qwen3-TTS and IndexTTS mixed-length A/B output WAV pairs and all 18 reversed-order Qwen3 pairs were byte-identical. This is before/current C++ parity, not a new Python parity claim. diff --git a/include/engine/models/breeze_tts/generator.h b/include/engine/models/breeze_tts/generator.h index abd8e7923..8d5c2a8aa 100644 --- a/include/engine/models/breeze_tts/generator.h +++ b/include/engine/models/breeze_tts/generator.h @@ -58,7 +58,7 @@ class BreezeGeneratorRuntime { ~BreezeGeneratorRuntime(); engine::runtime::AudioBuffer generate(const BreezeGenerationRequest & request); - BreezeSpeechCodes encode_reference(const engine::runtime::AudioBuffer & audio, bool retain_graphs = true) const; + BreezeSpeechCodes encode_reference(const engine::runtime::AudioBuffer & audio) const; void begin_stream(const BreezeGenerationRequest & request); BreezeStreamEvent next_stream_audio(size_t max_new_frames, int64_t lookahead_margin); void end_stream(); diff --git a/src/models/breeze_tts/generator.cpp b/src/models/breeze_tts/generator.cpp index 4b52dc9f9..6fb82f449 100644 --- a/src/models/breeze_tts/generator.cpp +++ b/src/models/breeze_tts/generator.cpp @@ -1489,12 +1489,8 @@ engine::runtime::AudioBuffer BreezeGeneratorRuntime::generate(const BreezeGenera return audio; } -BreezeSpeechCodes BreezeGeneratorRuntime::encode_reference(const engine::runtime::AudioBuffer & audio, bool retain_graphs) const { - auto codes = impl_->speech_encoder->encode(audio); - if (!retain_graphs) { - impl_->speech_encoder->release_runtime_graphs(); - } - return codes; +BreezeSpeechCodes BreezeGeneratorRuntime::encode_reference(const engine::runtime::AudioBuffer & audio) const { + return impl_->speech_encoder->encode(audio); } void BreezeGeneratorRuntime::begin_stream(const BreezeGenerationRequest & request) { diff --git a/src/models/breeze_tts/session.cpp b/src/models/breeze_tts/session.cpp index 4d9e5c759..5fba436a6 100644 --- a/src/models/breeze_tts/session.cpp +++ b/src/models/breeze_tts/session.cpp @@ -236,7 +236,7 @@ BreezeSpeechCodes BreezeTTSSession::resolve_reference_codes(const runtime::Audio } const auto start = std::chrono::steady_clock::now(); ReferenceCacheEntry entry; - entry.codes = generator_->encode_reference(audio, reference_cache_.capacity() == 0); + entry.codes = generator_->encode_reference(audio); engine::debug::trace_log_scalar("breeze_tts.reference.frames", entry.codes.frames); engine::debug::trace_log_scalar("breeze_tts.reference.codebooks", entry.codes.code_groups); if (reference_cache_.capacity() == 0) {